usearch 2.9.2 → 2.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (172) hide show
  1. package/README.md +39 -38
  2. package/javascript/{types → dist/cjs}/usearch.d.ts +90 -115
  3. package/javascript/dist/cjs/usearch.js +531 -0
  4. package/javascript/dist/esm/usearch.d.ts +212 -0
  5. package/javascript/{usearch.js → dist/esm/usearch.js} +199 -153
  6. package/package.json +25 -23
  7. package/prebuilds/darwin-arm64+x64/usearch.node +0 -0
  8. package/prebuilds/linux-arm64/usearch.node +0 -0
  9. package/prebuilds/linux-x64/usearch.node +0 -0
  10. package/prebuilds/win32-ia32/usearch.node +0 -0
  11. package/prebuilds/win32-x64/usearch.node +0 -0
  12. package/.clang-format +0 -10
  13. package/.clang-tidy +0 -60
  14. package/.cmake-format.py +0 -19
  15. package/.dockerignore +0 -93
  16. package/.gitattributes +0 -9
  17. package/.github/ISSUE_TEMPLATE/bug_report.yml +0 -90
  18. package/.github/ISSUE_TEMPLATE/config.yml +0 -5
  19. package/.github/ISSUE_TEMPLATE/feature_request.yml +0 -64
  20. package/.github/workflows/Dockerfile.libs +0 -36
  21. package/.github/workflows/prerelease.yml +0 -331
  22. package/.github/workflows/release.yml +0 -722
  23. package/.github/workflows/update_version.sh +0 -15
  24. package/.gitmodules +0 -9
  25. package/.releaserc +0 -90
  26. package/CITATION.cff +0 -11
  27. package/CMakeLists.txt +0 -324
  28. package/CODE_OF_CONDUCT.md +0 -128
  29. package/CONTRIBUTING.md +0 -361
  30. package/Cargo.toml +0 -19
  31. package/Dockerfile +0 -16
  32. package/VERSION +0 -1
  33. package/assets/unum.png +0 -0
  34. package/assets/usearch-approaches-transparent.png +0 -0
  35. package/assets/usearch-approaches-white.png +0 -0
  36. package/assets/usearch-neighbor-types.png +0 -0
  37. package/binding.gyp +0 -87
  38. package/build.gradle +0 -204
  39. package/build.rs +0 -18
  40. package/c/CMakeLists.txt +0 -25
  41. package/c/README.md +0 -34
  42. package/c/lib.cpp +0 -358
  43. package/c/test.c +0 -382
  44. package/c/usearch.h +0 -311
  45. package/cmake/clipp.cmake +0 -7
  46. package/cmake/config.cmake.in +0 -15
  47. package/cmake/emscripten.cmake +0 -8
  48. package/cmake/jemalloc.cmake +0 -62
  49. package/cmake/pkg-config.pc.in +0 -4
  50. package/cmake/usearchConfigVersion.cmake.in +0 -20
  51. package/conanfile.py +0 -26
  52. package/cpp/CMakeLists.txt +0 -14
  53. package/cpp/README.md +0 -161
  54. package/cpp/bench.cpp +0 -618
  55. package/cpp/test.cpp +0 -323
  56. package/fp16/CMakeLists.txt +0 -176
  57. package/fp16/LICENSE +0 -11
  58. package/fp16/README.md +0 -20
  59. package/fp16/bench/alt-element.cc +0 -89
  60. package/fp16/bench/from-alt-array.cc +0 -175
  61. package/fp16/bench/from-ieee-array.cc +0 -399
  62. package/fp16/bench/ieee-element.cc +0 -272
  63. package/fp16/bench/to-alt-array.cc +0 -91
  64. package/fp16/bench/to-ieee-array.cc +0 -303
  65. package/fp16/cmake/DownloadGoogleBenchmark.cmake +0 -15
  66. package/fp16/cmake/DownloadGoogleTest.cmake +0 -15
  67. package/fp16/cmake/DownloadPSimd.cmake +0 -15
  68. package/fp16/configure.py +0 -65
  69. package/fp16/confu.yaml +0 -8
  70. package/fp16/include/fp16/__init__.py +0 -0
  71. package/fp16/include/fp16/avx.py +0 -50
  72. package/fp16/include/fp16/avx2.py +0 -53
  73. package/fp16/include/fp16/bitcasts.h +0 -92
  74. package/fp16/include/fp16/fp16.h +0 -451
  75. package/fp16/include/fp16/psimd.h +0 -131
  76. package/fp16/include/fp16.h +0 -11
  77. package/fp16/jni/Android.mk +0 -6
  78. package/fp16/jni/Application.mk +0 -4
  79. package/fp16/test/alt-from-fp32-value.cc +0 -498
  80. package/fp16/test/alt-to-fp32-bits.cc +0 -263
  81. package/fp16/test/alt-to-fp32-psimd.cc +0 -145
  82. package/fp16/test/alt-to-fp32-value.cc +0 -348
  83. package/fp16/test/alt-to-fp32x2-psimd.cc +0 -245
  84. package/fp16/test/bitcasts.cc +0 -56
  85. package/fp16/test/ieee-from-fp32-value.cc +0 -523
  86. package/fp16/test/ieee-to-fp32-bits.cc +0 -333
  87. package/fp16/test/ieee-to-fp32-psimd.cc +0 -312
  88. package/fp16/test/ieee-to-fp32-value.cc +0 -420
  89. package/fp16/test/ieee-to-fp32x2-psimd.cc +0 -541
  90. package/fp16/test/peachpy/alt-xmm-to-fp32-xmm-avx.cc +0 -96
  91. package/fp16/test/peachpy/alt-xmm-to-fp32-ymm-avx2.cc +0 -96
  92. package/fp16/test/peachpy/stubs.py +0 -38
  93. package/fp16/test/tables.cc +0 -530
  94. package/fp16/test/tables.h +0 -13
  95. package/fp16/third-party/THHalf.h +0 -92
  96. package/fp16/third-party/eigen-half.h +0 -170
  97. package/fp16/third-party/float16-compressor.h +0 -80
  98. package/fp16/third-party/half.hpp +0 -2916
  99. package/fp16/third-party/npy-halffloat.h +0 -328
  100. package/include/usearch/index.hpp +0 -3836
  101. package/include/usearch/index_dense.hpp +0 -2004
  102. package/include/usearch/index_plugins.hpp +0 -2213
  103. package/javascript/lib.cpp +0 -341
  104. package/javascript/types/usearch.d.ts.map +0 -1
  105. package/javascript/usearch.test.js +0 -74
  106. package/package-ci.json +0 -10
  107. package/pyproject.toml +0 -51
  108. package/setup.py +0 -201
  109. package/simsimd/.releaserc +0 -86
  110. package/simsimd/CONTRIBUTING.md +0 -88
  111. package/simsimd/Cargo.lock +0 -709
  112. package/simsimd/Cargo.toml +0 -44
  113. package/simsimd/LICENSE +0 -201
  114. package/simsimd/README.md +0 -317
  115. package/simsimd/binding.gyp +0 -36
  116. package/simsimd/build.rs +0 -20
  117. package/simsimd/include/simsimd/binary.h +0 -245
  118. package/simsimd/include/simsimd/complex.h +0 -174
  119. package/simsimd/include/simsimd/probability.h +0 -548
  120. package/simsimd/include/simsimd/simsimd.h +0 -465
  121. package/simsimd/include/simsimd/spatial.h +0 -1267
  122. package/simsimd/include/simsimd/types.h +0 -208
  123. package/simsimd/javascript/bench.js +0 -127
  124. package/simsimd/javascript/fallback.ts +0 -193
  125. package/simsimd/javascript/lib.c +0 -102
  126. package/simsimd/javascript/node-gyp-build.d.ts +0 -1
  127. package/simsimd/javascript/simsimd.ts +0 -124
  128. package/simsimd/javascript/test.mjs +0 -153
  129. package/simsimd/javascript/tsconfig-base.json +0 -25
  130. package/simsimd/javascript/tsconfig-cjs.json +0 -8
  131. package/simsimd/javascript/tsconfig-esm.json +0 -8
  132. package/simsimd/package-lock.json +0 -1688
  133. package/simsimd/package.json +0 -61
  134. package/simsimd/rust/benches/cosine.rs +0 -27
  135. package/simsimd/rust/benches/native.rs +0 -30
  136. package/simsimd/rust/benches/sqeuclidean.rs +0 -27
  137. package/simsimd/rust/lib.c +0 -45
  138. package/simsimd/rust/lib.rs +0 -550
  139. package/sqlite/README.md +0 -178
  140. package/sqlite/lib.cpp +0 -325
  141. package/stringzilla/CONTRIBUTING.md +0 -446
  142. package/stringzilla/Cargo.lock +0 -25
  143. package/stringzilla/Cargo.toml +0 -26
  144. package/stringzilla/LICENSE +0 -201
  145. package/stringzilla/README.md +0 -1422
  146. package/stringzilla/VERSION +0 -1
  147. package/stringzilla/assets/cover-strinzilla.jpeg +0 -0
  148. package/stringzilla/assets/meme-stringzilla-v2.jpeg +0 -0
  149. package/stringzilla/assets/meme-stringzilla-v3.jpeg +0 -0
  150. package/stringzilla/binding.gyp +0 -16
  151. package/stringzilla/build.rs +0 -87
  152. package/stringzilla/c/lib.c +0 -337
  153. package/stringzilla/cli/README.md +0 -65
  154. package/stringzilla/cli/__init__.py +0 -0
  155. package/stringzilla/cli/split.py +0 -105
  156. package/stringzilla/cli/wc.py +0 -117
  157. package/stringzilla/include/stringzilla/experimental.h +0 -845
  158. package/stringzilla/include/stringzilla/module.modulemap +0 -9
  159. package/stringzilla/include/stringzilla/stringzilla.h +0 -5370
  160. package/stringzilla/include/stringzilla/stringzilla.hpp +0 -3838
  161. package/stringzilla/javascript/lib.c +0 -125
  162. package/stringzilla/javascript/stringzilla.js +0 -22
  163. package/stringzilla/package-ci.json +0 -10
  164. package/stringzilla/package.json +0 -32
  165. package/stringzilla/rust/lib.rs +0 -873
  166. package/stringzilla/swift/StringProtocol+StringZilla.swift +0 -280
  167. package/stringzilla/swift/Test.swift +0 -60
  168. package/tsconfig.json +0 -21
  169. package/wasmer.toml +0 -12
  170. package/winlibinstaller.bat +0 -42
  171. /package/{simsimd/javascript/dist-package-cjs.json → javascript/dist/cjs/package.json} +0 -0
  172. /package/{simsimd/javascript/dist-package-esm.json → javascript/dist/esm/package.json} +0 -0
@@ -1 +0,0 @@
1
- 3.5.0
@@ -1,16 +0,0 @@
1
- {
2
- "targets": [
3
- {
4
- "target_name": "stringzilla",
5
- "sources": ["javascript/lib.c"],
6
- "include_dirs": ["include"],
7
- "cflags": [
8
- "-std=c99",
9
- "-Wno-unknown-pragmas",
10
- "-Wno-maybe-uninitialized",
11
- "-Wno-cast-function-type",
12
- "-Wno-unused-function",
13
- ],
14
- }
15
- ]
16
- }
@@ -1,87 +0,0 @@
1
- use std::env;
2
-
3
- fn main() {
4
- let mut build = cc::Build::new();
5
- build
6
- .file("c/lib.c")
7
- .include("include")
8
- .warnings(false)
9
- .flag_if_supported("-std=c99")
10
- .flag_if_supported("-fPIC");
11
-
12
- // Cargo will set different environment variables that we can use to properly configure the build.
13
- // https://doc.rust-lang.org/cargo/reference/environment-variables.html#environment-variables-cargo-sets-for-build-scripts
14
- let target_arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap_or_default();
15
- let target_endian = env::var("CARGO_CFG_TARGET_ENDIAN").unwrap_or_default();
16
-
17
- // To get the operating system we can use the TARGET environment variable.
18
- // To check the list of available targets, run `rustc --print target-list`.
19
- let target = env::var("TARGET").unwrap_or_default();
20
-
21
- if target.contains("linux") {
22
- build.flag_if_supported("-fdiagnostics-color=always");
23
- build.flag_if_supported("-O3");
24
- build.flag_if_supported("-pedantic");
25
-
26
- // Set architecture-specific flags and macros
27
- if target_arch == "x86_64" {
28
- build.define("SZ_USE_X86_AVX512", "1");
29
- build.define("SZ_USE_X86_AVX2", "1");
30
- } else {
31
- build.define("SZ_USE_X86_AVX512", "0");
32
- build.define("SZ_USE_X86_AVX2", "0");
33
- }
34
-
35
- if target_arch == "aarch64" {
36
- build.flag_if_supported("-march=armv8-a+simd");
37
- build.define("SZ_USE_ARM_SVE", "1");
38
- build.define("SZ_USE_ARM_NEON", "1");
39
- } else {
40
- build.define("SZ_USE_ARM_SVE", "0");
41
- build.define("SZ_USE_ARM_NEON", "0");
42
- }
43
- } else if target.contains("darwin") {
44
- build.flag_if_supported("-fcolor-diagnostics");
45
- build.flag_if_supported("-O3");
46
- build.flag_if_supported("-pedantic");
47
-
48
- if target_arch == "x86_64" {
49
- // Assuming no AVX-512 support for Darwin as per setup.py logic
50
- build.define("SZ_USE_X86_AVX512", "0");
51
- build.define("SZ_USE_X86_AVX2", "1");
52
- } else {
53
- build.define("SZ_USE_X86_AVX512", "0");
54
- build.define("SZ_USE_X86_AVX2", "0");
55
- }
56
-
57
- if target_arch == "aarch64" {
58
- build.define("SZ_USE_ARM_SVE", "0"); // Assuming no SVE support for Darwin
59
- build.define("SZ_USE_ARM_NEON", "1");
60
- } else {
61
- build.define("SZ_USE_ARM_SVE", "0");
62
- build.define("SZ_USE_ARM_NEON", "0");
63
- }
64
- } else if target.contains("windows") {
65
- // Set architecture-specific flags and macros
66
- if target_arch == "x86_64" {
67
- build.define("SZ_USE_X86_AVX512", "1");
68
- build.define("SZ_USE_X86_AVX2", "1");
69
- } else {
70
- build.define("SZ_USE_X86_AVX512", "0");
71
- build.define("SZ_USE_X86_AVX2", "0");
72
- }
73
- }
74
-
75
- // Set endian-specific macro
76
- if target_endian == "big" {
77
- build.define("SZ_DETECT_BIG_ENDIAN", "1");
78
- } else {
79
- build.define("SZ_DETECT_BIG_ENDIAN", "0");
80
- }
81
-
82
- build.compile("stringzilla");
83
-
84
- println!("cargo:rerun-if-changed=c/lib.c");
85
- println!("cargo:rerun-if-changed=rust/lib.rs");
86
- println!("cargo:rerun-if-changed=include/stringzilla/stringzilla.h");
87
- }
@@ -1,337 +0,0 @@
1
- /**
2
- * @file lib.c
3
- * @brief StringZilla C library with dynamic backed dispatch for the most appropriate implementation.
4
- * @author Ash Vardanian
5
- * @date January 16, 2024
6
- * @copyright Copyright (c) 2024
7
- */
8
- #if defined(_WIN32) || defined(__CYGWIN__)
9
- #include <windows.h> // `DllMain`
10
- #endif
11
-
12
- // Overwrite `SZ_DYNAMIC_DISPATCH` before including StringZilla.
13
- #ifdef SZ_DYNAMIC_DISPATCH
14
- #undef SZ_DYNAMIC_DISPATCH
15
- #endif
16
- #define SZ_DYNAMIC_DISPATCH 1
17
- #include <stringzilla/stringzilla.h>
18
-
19
- #if SZ_AVOID_LIBC
20
- // If we don't have the LibC, the `malloc` definition in `stringzilla.h` will be illformed.
21
- #ifdef _MSC_VER
22
- typedef sz_size_t size_t; // Reuse the type definition we've inferred from `stringzilla.h`
23
- #else
24
- typedef __SIZE_TYPE__ size_t; // For GCC/Clang
25
- #endif
26
- int rand(void) { return 0; }
27
- void free(void *start) { sz_unused(start); }
28
- void *malloc(size_t length) {
29
- sz_unused(length);
30
- return SZ_NULL;
31
- }
32
- #endif
33
-
34
- SZ_DYNAMIC sz_capability_t sz_capabilities(void) {
35
-
36
- #if SZ_USE_X86_AVX512 || SZ_USE_X86_AVX2
37
-
38
- /// The states of 4 registers populated for a specific "cpuid" assembly call
39
- union four_registers_t {
40
- int array[4];
41
- struct separate_t {
42
- unsigned eax, ebx, ecx, edx;
43
- } named;
44
- } info1, info7;
45
-
46
- #ifdef _MSC_VER
47
- __cpuidex(info1.array, 1, 0);
48
- __cpuidex(info7.array, 7, 0);
49
- #else
50
- __asm__ __volatile__("cpuid"
51
- : "=a"(info1.named.eax), "=b"(info1.named.ebx), "=c"(info1.named.ecx), "=d"(info1.named.edx)
52
- : "a"(1), "c"(0));
53
- __asm__ __volatile__("cpuid"
54
- : "=a"(info7.named.eax), "=b"(info7.named.ebx), "=c"(info7.named.ecx), "=d"(info7.named.edx)
55
- : "a"(7), "c"(0));
56
- #endif
57
-
58
- // Check for AVX2 (Function ID 7, EBX register)
59
- // https://github.com/llvm/llvm-project/blob/50598f0ff44f3a4e75706f8c53f3380fe7faa896/clang/lib/Headers/cpuid.h#L148
60
- unsigned supports_avx2 = (info7.named.ebx & 0x00000020) != 0;
61
- // Check for AVX512F (Function ID 7, EBX register)
62
- // https://github.com/llvm/llvm-project/blob/50598f0ff44f3a4e75706f8c53f3380fe7faa896/clang/lib/Headers/cpuid.h#L155
63
- unsigned supports_avx512f = (info7.named.ebx & 0x00010000) != 0;
64
- // Check for AVX512BW (Function ID 7, EBX register)
65
- // https://github.com/llvm/llvm-project/blob/50598f0ff44f3a4e75706f8c53f3380fe7faa896/clang/lib/Headers/cpuid.h#L166
66
- unsigned supports_avx512bw = (info7.named.ebx & 0x40000000) != 0;
67
- // Check for AVX512VL (Function ID 7, EBX register)
68
- // https://github.com/llvm/llvm-project/blob/50598f0ff44f3a4e75706f8c53f3380fe7faa896/clang/lib/Headers/cpuid.h#L167C25-L167C35
69
- unsigned supports_avx512vl = (info7.named.ebx & 0x80000000) != 0;
70
- // Check for GFNI (Function ID 1, ECX register)
71
- // https://github.com/llvm/llvm-project/blob/50598f0ff44f3a4e75706f8c53f3380fe7faa896/clang/lib/Headers/cpuid.h#L171C30-L171C40
72
- unsigned supports_avx512vbmi = (info1.named.ecx & 0x00000002) != 0;
73
- // Check for GFNI (Function ID 1, ECX register)
74
- // https://github.com/llvm/llvm-project/blob/50598f0ff44f3a4e75706f8c53f3380fe7faa896/clang/lib/Headers/cpuid.h#L177C30-L177C40
75
- unsigned supports_gfni = (info1.named.ecx & 0x00000100) != 0;
76
-
77
- return (sz_capability_t)( //
78
- (sz_cap_x86_avx2_k * supports_avx2) | //
79
- (sz_cap_x86_avx512f_k * supports_avx512f) | //
80
- (sz_cap_x86_avx512vl_k * supports_avx512vl) | //
81
- (sz_cap_x86_avx512bw_k * supports_avx512bw) | //
82
- (sz_cap_x86_avx512vbmi_k * supports_avx512vbmi) | //
83
- (sz_cap_x86_gfni_k * (supports_gfni)) | //
84
- (sz_cap_serial_k));
85
-
86
- #endif // SIMSIMD_TARGET_X86
87
-
88
- #if SZ_USE_ARM_NEON || SZ_USE_ARM_SVE
89
-
90
- // Every 64-bit Arm CPU supports NEON
91
- unsigned supports_neon = 1;
92
- unsigned supports_sve = 0;
93
- unsigned supports_sve2 = 0;
94
- sz_unused(supports_sve);
95
- sz_unused(supports_sve2);
96
-
97
- return (sz_capability_t)( //
98
- (sz_cap_arm_neon_k * supports_neon) | //
99
- (sz_cap_serial_k));
100
-
101
- #endif // SIMSIMD_TARGET_ARM
102
-
103
- return sz_cap_serial_k;
104
- }
105
-
106
- typedef struct sz_implementations_t {
107
- sz_equal_t equal;
108
- sz_order_t order;
109
-
110
- sz_move_t copy;
111
- sz_move_t move;
112
- sz_fill_t fill;
113
-
114
- sz_find_byte_t find_byte;
115
- sz_find_byte_t rfind_byte;
116
- sz_find_t find;
117
- sz_find_t rfind;
118
- sz_find_set_t find_from_set;
119
- sz_find_set_t rfind_from_set;
120
-
121
- sz_edit_distance_t edit_distance;
122
- sz_alignment_score_t alignment_score;
123
- sz_hashes_t hashes;
124
-
125
- } sz_implementations_t;
126
- static sz_implementations_t sz_dispatch_table;
127
-
128
- /**
129
- * @brief Initializes a global static "virtual table" of supported backends
130
- * Run it just once to avoiding unnecessary `if`-s.
131
- */
132
- static void sz_dispatch_table_init(void) {
133
- sz_implementations_t *impl = &sz_dispatch_table;
134
- sz_capability_t caps = sz_capabilities();
135
- sz_unused(caps); //< Unused when compiling on pre-SIMD machines.
136
-
137
- impl->equal = sz_equal_serial;
138
- impl->order = sz_order_serial;
139
- impl->copy = sz_copy_serial;
140
- impl->move = sz_move_serial;
141
- impl->fill = sz_fill_serial;
142
-
143
- impl->find = sz_find_serial;
144
- impl->rfind = sz_rfind_serial;
145
- impl->find_byte = sz_find_byte_serial;
146
- impl->rfind_byte = sz_rfind_byte_serial;
147
- impl->find_from_set = sz_find_charset_serial;
148
- impl->rfind_from_set = sz_rfind_charset_serial;
149
-
150
- impl->edit_distance = sz_edit_distance_serial;
151
- impl->alignment_score = sz_alignment_score_serial;
152
- impl->hashes = sz_hashes_serial;
153
-
154
- #if SZ_USE_X86_AVX2
155
- if (caps & sz_cap_x86_avx2_k) {
156
- impl->copy = sz_copy_avx2;
157
- impl->move = sz_move_avx2;
158
- impl->fill = sz_fill_avx2;
159
- impl->find_byte = sz_find_byte_avx2;
160
- impl->rfind_byte = sz_rfind_byte_avx2;
161
- impl->find = sz_find_avx2;
162
- impl->rfind = sz_rfind_avx2;
163
- }
164
- #endif
165
-
166
- #if SZ_USE_X86_AVX512
167
- if (caps & sz_cap_x86_avx512f_k) {
168
- impl->equal = sz_equal_avx512;
169
- impl->order = sz_order_avx512;
170
- impl->copy = sz_copy_avx512;
171
- impl->move = sz_move_avx512;
172
- impl->fill = sz_fill_avx512;
173
-
174
- impl->find = sz_find_avx512;
175
- impl->rfind = sz_rfind_avx512;
176
- impl->find_byte = sz_find_byte_avx512;
177
- impl->rfind_byte = sz_rfind_byte_avx512;
178
-
179
- impl->edit_distance = sz_edit_distance_avx512;
180
- }
181
-
182
- if ((caps & sz_cap_x86_avx512f_k) && (caps & sz_cap_x86_avx512vl_k) && (caps & sz_cap_x86_gfni_k) &&
183
- (caps & sz_cap_x86_avx512bw_k) && (caps & sz_cap_x86_avx512vbmi_k)) {
184
- impl->find_from_set = sz_find_charset_avx512;
185
- impl->rfind_from_set = sz_rfind_charset_avx512;
186
- impl->alignment_score = sz_alignment_score_avx512;
187
- }
188
- #endif
189
-
190
- #if SZ_USE_ARM_NEON
191
- if (caps & sz_cap_arm_neon_k) {
192
- impl->find = sz_find_neon;
193
- impl->rfind = sz_rfind_neon;
194
- impl->find_byte = sz_find_byte_neon;
195
- impl->rfind_byte = sz_rfind_byte_neon;
196
- impl->find_from_set = sz_find_charset_neon;
197
- impl->rfind_from_set = sz_rfind_charset_neon;
198
- }
199
- #endif
200
- }
201
-
202
- #if defined(_MSC_VER)
203
- BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpReserved) {
204
- switch (fdwReason) {
205
- case DLL_PROCESS_ATTACH: sz_dispatch_table_init(); return TRUE;
206
- case DLL_THREAD_ATTACH: return TRUE;
207
- case DLL_THREAD_DETACH: return TRUE;
208
- case DLL_PROCESS_DETACH: return TRUE;
209
- }
210
- }
211
- #else
212
- __attribute__((constructor)) static void sz_dispatch_table_init_on_gcc_or_clang(void) { sz_dispatch_table_init(); }
213
- #endif
214
-
215
- SZ_DYNAMIC sz_bool_t sz_equal(sz_cptr_t a, sz_cptr_t b, sz_size_t length) {
216
- return sz_dispatch_table.equal(a, b, length);
217
- }
218
-
219
- SZ_DYNAMIC sz_ordering_t sz_order(sz_cptr_t a, sz_size_t a_length, sz_cptr_t b, sz_size_t b_length) {
220
- return sz_dispatch_table.order(a, a_length, b, b_length);
221
- }
222
-
223
- SZ_DYNAMIC void sz_copy(sz_ptr_t target, sz_cptr_t source, sz_size_t length) {
224
- sz_dispatch_table.copy(target, source, length);
225
- }
226
-
227
- SZ_DYNAMIC void sz_move(sz_ptr_t target, sz_cptr_t source, sz_size_t length) {
228
- sz_dispatch_table.move(target, source, length);
229
- }
230
-
231
- SZ_DYNAMIC void sz_fill(sz_ptr_t target, sz_size_t length, sz_u8_t value) {
232
- sz_dispatch_table.fill(target, length, value);
233
- }
234
-
235
- SZ_DYNAMIC sz_cptr_t sz_find_byte(sz_cptr_t haystack, sz_size_t h_length, sz_cptr_t needle) {
236
- return sz_dispatch_table.find_byte(haystack, h_length, needle);
237
- }
238
-
239
- SZ_DYNAMIC sz_cptr_t sz_rfind_byte(sz_cptr_t haystack, sz_size_t h_length, sz_cptr_t needle) {
240
- return sz_dispatch_table.rfind_byte(haystack, h_length, needle);
241
- }
242
-
243
- SZ_DYNAMIC sz_cptr_t sz_find(sz_cptr_t haystack, sz_size_t h_length, sz_cptr_t needle, sz_size_t n_length) {
244
- return sz_dispatch_table.find(haystack, h_length, needle, n_length);
245
- }
246
-
247
- SZ_DYNAMIC sz_cptr_t sz_rfind(sz_cptr_t haystack, sz_size_t h_length, sz_cptr_t needle, sz_size_t n_length) {
248
- return sz_dispatch_table.rfind(haystack, h_length, needle, n_length);
249
- }
250
-
251
- SZ_DYNAMIC sz_cptr_t sz_find_charset(sz_cptr_t text, sz_size_t length, sz_charset_t const *set) {
252
- return sz_dispatch_table.find_from_set(text, length, set);
253
- }
254
-
255
- SZ_DYNAMIC sz_cptr_t sz_rfind_charset(sz_cptr_t text, sz_size_t length, sz_charset_t const *set) {
256
- return sz_dispatch_table.rfind_from_set(text, length, set);
257
- }
258
-
259
- SZ_DYNAMIC sz_size_t sz_hamming_distance( //
260
- sz_cptr_t a, sz_size_t a_length, //
261
- sz_cptr_t b, sz_size_t b_length, //
262
- sz_size_t bound) {
263
- return sz_hamming_distance_serial(a, a_length, b, b_length, bound);
264
- }
265
-
266
- SZ_DYNAMIC sz_size_t sz_hamming_distance_utf8( //
267
- sz_cptr_t a, sz_size_t a_length, //
268
- sz_cptr_t b, sz_size_t b_length, //
269
- sz_size_t bound) {
270
- return sz_hamming_distance_utf8_serial(a, a_length, b, b_length, bound);
271
- }
272
-
273
- SZ_DYNAMIC sz_size_t sz_edit_distance( //
274
- sz_cptr_t a, sz_size_t a_length, //
275
- sz_cptr_t b, sz_size_t b_length, //
276
- sz_size_t bound, sz_memory_allocator_t *alloc) {
277
- return sz_dispatch_table.edit_distance(a, a_length, b, b_length, bound, alloc);
278
- }
279
-
280
- SZ_DYNAMIC sz_size_t sz_edit_distance_utf8( //
281
- sz_cptr_t a, sz_size_t a_length, //
282
- sz_cptr_t b, sz_size_t b_length, //
283
- sz_size_t bound, sz_memory_allocator_t *alloc) {
284
- return _sz_edit_distance_wagner_fisher_serial(a, a_length, b, b_length, bound, sz_true_k, alloc);
285
- }
286
-
287
- SZ_DYNAMIC sz_ssize_t sz_alignment_score(sz_cptr_t a, sz_size_t a_length, sz_cptr_t b, sz_size_t b_length,
288
- sz_error_cost_t const *subs, sz_error_cost_t gap,
289
- sz_memory_allocator_t *alloc) {
290
- return sz_dispatch_table.alignment_score(a, a_length, b, b_length, subs, gap, alloc);
291
- }
292
-
293
- SZ_DYNAMIC void sz_hashes(sz_cptr_t text, sz_size_t length, sz_size_t window_length, sz_size_t step, //
294
- sz_hash_callback_t callback, void *callback_handle) {
295
- sz_dispatch_table.hashes(text, length, window_length, step, callback, callback_handle);
296
- }
297
-
298
- SZ_DYNAMIC sz_cptr_t sz_find_char_from(sz_cptr_t h, sz_size_t h_length, sz_cptr_t n, sz_size_t n_length) {
299
- sz_charset_t set;
300
- sz_charset_init(&set);
301
- for (; n_length; ++n, --n_length) sz_charset_add(&set, *n);
302
- return sz_find_charset(h, h_length, &set);
303
- }
304
-
305
- SZ_DYNAMIC sz_cptr_t sz_find_char_not_from(sz_cptr_t h, sz_size_t h_length, sz_cptr_t n, sz_size_t n_length) {
306
- sz_charset_t set;
307
- sz_charset_init(&set);
308
- for (; n_length; ++n, --n_length) sz_charset_add(&set, *n);
309
- sz_charset_invert(&set);
310
- return sz_find_charset(h, h_length, &set);
311
- }
312
-
313
- SZ_DYNAMIC sz_cptr_t sz_rfind_char_from(sz_cptr_t h, sz_size_t h_length, sz_cptr_t n, sz_size_t n_length) {
314
- sz_charset_t set;
315
- sz_charset_init(&set);
316
- for (; n_length; ++n, --n_length) sz_charset_add(&set, *n);
317
- return sz_rfind_charset(h, h_length, &set);
318
- }
319
-
320
- SZ_DYNAMIC sz_cptr_t sz_rfind_char_not_from(sz_cptr_t h, sz_size_t h_length, sz_cptr_t n, sz_size_t n_length) {
321
- sz_charset_t set;
322
- sz_charset_init(&set);
323
- for (; n_length; ++n, --n_length) sz_charset_add(&set, *n);
324
- sz_charset_invert(&set);
325
- return sz_rfind_charset(h, h_length, &set);
326
- }
327
-
328
- sz_u64_t _sz_random_generator(void *empty_state) {
329
- sz_unused(empty_state);
330
- return (sz_u64_t)rand();
331
- }
332
-
333
- SZ_DYNAMIC void sz_generate(sz_cptr_t alphabet, sz_size_t alphabet_size, sz_ptr_t result, sz_size_t result_length,
334
- sz_random_generator_t generator, void *generator_user_data) {
335
- if (!generator) generator = _sz_random_generator;
336
- sz_generate_serial(alphabet, alphabet_size, result, result_length, generator, generator_user_data);
337
- }
@@ -1,65 +0,0 @@
1
- # SIMD-accelerated CLI utilities based on StringZilla
2
-
3
- This section of the project is pretty much a work in progress.
4
- The goal is to provide a set of command-line utilities that:
5
-
6
- - ✅ that benefit the most from SIMD instructions,
7
- - ✅ rely solely on core StringZilla functionality,
8
- - ✅ works the same on Linux, MacOS, and Windows.
9
-
10
- Other utilities are, of course, welcome to use StringZilla but may not be good candidates for this repository.
11
- To install, pull the Python package from PyPi:
12
-
13
- ```bash
14
- pip install stringzilla
15
- ```
16
-
17
- Currently implemented:
18
-
19
- - `sz_wc`: 3x faster `wc` word count.
20
- - `sz_split`: 4x faster `splt` file splitting.
21
-
22
- What other interfaces should be added?
23
- Levenshtein distances?
24
- Fuzzy search?
25
- Are there common alternatives to mimic?
26
-
27
- ## `wc`: Word Count
28
-
29
- The `wc` utility on Linux can be used to count the number of lines, words, and bytes in a file.
30
- Using SIMD-accelerated character and character-set search, StringZilla, even with slow SSDs, it can be noticeably faster.
31
-
32
- ```bash
33
- $ time wc enwik9.txt
34
- 13147025 129348346 1000000000 enwik9.txt
35
-
36
- real 0m3.562s
37
- user 0m3.470s
38
- sys 0m0.092s
39
-
40
- $ time sz_wc enwik9.txt
41
- 13147025 139132610 1000000000 enwik9.txt # Note: different word count, WIP
42
-
43
- real 0m1.165s
44
- user 0m1.121s
45
- sys 0m0.044s
46
- ```
47
-
48
- ## `split`: Split File into Smaller Ones
49
-
50
- The `split` utility on Linux can be used to split a file into smaller ones.
51
- The current prototype only splits by line counts.
52
-
53
- ```bash
54
- $ time split -l 100000 enwik9.txt ...
55
-
56
- real 0m6.424s
57
- user 0m0.179s
58
- sys 0m0.663s
59
-
60
- $ time sz_split -l 100000 enwik9.txt ...
61
-
62
- real 0m1.482s
63
- user 0m1.020s
64
- sys 0m0.460s
65
- ```
File without changes
@@ -1,105 +0,0 @@
1
- #!/usr/bin/env python3
2
-
3
- import argparse
4
- import sys
5
- import stringzilla
6
- from stringzilla import File, Str
7
-
8
-
9
- def parse_arguments():
10
- parser = argparse.ArgumentParser(
11
- description="Output pieces of FILE to PREFIXaa, PREFIXab, ...; default size is 1000 lines, and default PREFIX is 'x'."
12
- )
13
- parser.add_argument(
14
- "file", nargs="?", default="-", help='File to process, "-" for standard input'
15
- )
16
- parser.add_argument(
17
- "prefix", nargs="?", default="X", help='Output file prefix, default is "x"'
18
- )
19
- parser.add_argument(
20
- "-l",
21
- "--lines",
22
- type=int,
23
- default=1000,
24
- help="Number of lines per output file, default is 1000",
25
- )
26
- parser.add_argument(
27
- "-t",
28
- "--separator",
29
- default="\n",
30
- help="Use SEP instead of newline as the record separator; '\\0' (zero) specifies the NUL character",
31
- )
32
- parser.add_argument(
33
- "-n",
34
- "--number",
35
- type=int,
36
- default=None,
37
- help="Generate N output files based on size of input",
38
- )
39
- parser.add_argument("--version", action="version", version=stringzilla.__version__)
40
- return parser.parse_args()
41
-
42
-
43
- def split_file(file_path, lines_per_file, output_prefix, separator, number_of_files):
44
- try:
45
- if separator == "\\0":
46
- separator = "\0"
47
- if file_path == "-":
48
- file_contents = Str(sys.stdin.read())
49
- else:
50
- file_mapped = File(file_path)
51
- file_contents = Str(file_mapped)
52
-
53
- if number_of_files is not None:
54
- total_length = len(file_contents)
55
- chunk_size = total_length // number_of_files
56
- for file_part in range(number_of_files):
57
- start = file_part * chunk_size
58
- end = (
59
- start + chunk_size
60
- if file_part < number_of_files - 1
61
- else total_length
62
- )
63
- current_slice = file_contents[start:end]
64
- output_path = f"{output_prefix}{file_part}"
65
- current_slice.write_to(output_path)
66
- return
67
- current_position = 0
68
- file_part = 0
69
- newline_position = -1
70
-
71
- while current_position < len(file_contents):
72
- for _ in range(lines_per_file):
73
- newline_position = file_contents.find(separator, newline_position + 1)
74
- if newline_position == -1:
75
- break
76
-
77
- if newline_position == -1 and current_position < len(file_contents):
78
- newline_position = len(file_contents)
79
-
80
- section_length = (
81
- newline_position - current_position if newline_position != -1 else 0
82
- )
83
-
84
- if section_length > 0:
85
- current_slice = file_contents[current_position : newline_position + 1]
86
- output_path = f"{output_prefix}{file_part}"
87
- current_slice.write_to(output_path)
88
-
89
- file_part += 1
90
- current_position = newline_position + 1
91
-
92
- except FileNotFoundError:
93
- print(f"No such file: {file_path}")
94
- except Exception as e:
95
- print(f"An error occurred: {e}")
96
- print("Usage example: split.py [-l LINES] [file] [prefix]")
97
-
98
-
99
- def main():
100
- args = parse_arguments()
101
- split_file(args.file, args.lines, args.prefix, args.separator, args.number)
102
-
103
-
104
- if __name__ == "__main__":
105
- main()