usearch 2.10.3 → 2.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. package/README.md +28 -8
  2. package/binding.gyp +101 -0
  3. package/c/CMakeLists.txt +27 -0
  4. package/c/README.md +244 -0
  5. package/c/lib.cpp +459 -0
  6. package/c/test.c +386 -0
  7. package/c/usearch.h +451 -0
  8. package/cmake/clipp.cmake +7 -0
  9. package/cmake/config.cmake.in +15 -0
  10. package/cmake/emscripten.cmake +8 -0
  11. package/cmake/jemalloc.cmake +62 -0
  12. package/cmake/pkg-config.pc.in +4 -0
  13. package/cmake/usearchConfigVersion.cmake.in +20 -0
  14. package/cpp/CMakeLists.txt +14 -0
  15. package/cpp/README.md +161 -0
  16. package/cpp/bench.cpp +616 -0
  17. package/cpp/test.cpp +335 -0
  18. package/fp16/CMakeLists.txt +176 -0
  19. package/fp16/LICENSE +11 -0
  20. package/fp16/README.md +20 -0
  21. package/fp16/bench/alt-element.cc +89 -0
  22. package/fp16/bench/from-alt-array.cc +175 -0
  23. package/fp16/bench/from-ieee-array.cc +399 -0
  24. package/fp16/bench/ieee-element.cc +272 -0
  25. package/fp16/bench/to-alt-array.cc +91 -0
  26. package/fp16/bench/to-ieee-array.cc +303 -0
  27. package/fp16/cmake/DownloadGoogleBenchmark.cmake +15 -0
  28. package/fp16/cmake/DownloadGoogleTest.cmake +15 -0
  29. package/fp16/cmake/DownloadPSimd.cmake +15 -0
  30. package/fp16/configure.py +65 -0
  31. package/fp16/confu.yaml +8 -0
  32. package/fp16/include/fp16/__init__.py +0 -0
  33. package/fp16/include/fp16/avx.py +50 -0
  34. package/fp16/include/fp16/avx2.py +53 -0
  35. package/fp16/include/fp16/bitcasts.h +92 -0
  36. package/fp16/include/fp16/fp16.h +451 -0
  37. package/fp16/include/fp16/psimd.h +131 -0
  38. package/fp16/include/fp16.h +11 -0
  39. package/fp16/jni/Android.mk +6 -0
  40. package/fp16/jni/Application.mk +4 -0
  41. package/fp16/test/alt-from-fp32-value.cc +498 -0
  42. package/fp16/test/alt-to-fp32-bits.cc +263 -0
  43. package/fp16/test/alt-to-fp32-psimd.cc +145 -0
  44. package/fp16/test/alt-to-fp32-value.cc +348 -0
  45. package/fp16/test/alt-to-fp32x2-psimd.cc +245 -0
  46. package/fp16/test/bitcasts.cc +56 -0
  47. package/fp16/test/ieee-from-fp32-value.cc +523 -0
  48. package/fp16/test/ieee-to-fp32-bits.cc +333 -0
  49. package/fp16/test/ieee-to-fp32-psimd.cc +312 -0
  50. package/fp16/test/ieee-to-fp32-value.cc +420 -0
  51. package/fp16/test/ieee-to-fp32x2-psimd.cc +541 -0
  52. package/fp16/test/peachpy/alt-xmm-to-fp32-xmm-avx.cc +96 -0
  53. package/fp16/test/peachpy/alt-xmm-to-fp32-ymm-avx2.cc +96 -0
  54. package/fp16/test/peachpy/stubs.py +38 -0
  55. package/fp16/test/tables.cc +530 -0
  56. package/fp16/test/tables.h +13 -0
  57. package/fp16/third-party/THHalf.h +92 -0
  58. package/fp16/third-party/eigen-half.h +170 -0
  59. package/fp16/third-party/float16-compressor.h +80 -0
  60. package/fp16/third-party/half.hpp +2916 -0
  61. package/fp16/third-party/npy-halffloat.h +328 -0
  62. package/include/usearch/index.hpp +3838 -0
  63. package/include/usearch/index_dense.hpp +2022 -0
  64. package/include/usearch/index_plugins.hpp +2317 -0
  65. package/javascript/dist/cjs/usearch.js +5 -5
  66. package/javascript/dist/esm/usearch.js +2 -2
  67. package/javascript/dist-package-cjs.json +3 -0
  68. package/javascript/dist-package-esm.json +3 -0
  69. package/javascript/lib.cpp +350 -0
  70. package/javascript/node-gyp-build.d.ts +1 -0
  71. package/javascript/tsconfig-base.json +25 -0
  72. package/javascript/tsconfig-cjs.json +8 -0
  73. package/javascript/tsconfig-esm.json +8 -0
  74. package/javascript/usearch.test.js +84 -0
  75. package/javascript/usearch.ts +640 -0
  76. package/package.json +14 -7
  77. package/prebuilds/darwin-arm64+x64/usearch.node +0 -0
  78. package/prebuilds/linux-arm64/usearch.node +0 -0
  79. package/prebuilds/linux-x64/usearch.node +0 -0
  80. package/prebuilds/win32-ia32/usearch.node +0 -0
  81. package/prebuilds/win32-x64/usearch.node +0 -0
  82. package/simsimd/.releaserc +87 -0
  83. package/simsimd/CONTRIBUTING.md +114 -0
  84. package/simsimd/Cargo.lock +709 -0
  85. package/simsimd/Cargo.toml +44 -0
  86. package/simsimd/LICENSE +201 -0
  87. package/simsimd/README.md +708 -0
  88. package/simsimd/binding.gyp +36 -0
  89. package/simsimd/build.rs +51 -0
  90. package/simsimd/c/lib.c +110 -0
  91. package/simsimd/include/simsimd/binary.h +258 -0
  92. package/simsimd/include/simsimd/dot.h +1237 -0
  93. package/simsimd/include/simsimd/geospatial.h +37 -0
  94. package/simsimd/include/simsimd/probability.h +563 -0
  95. package/simsimd/include/simsimd/simsimd.h +1129 -0
  96. package/simsimd/include/simsimd/spatial.h +1019 -0
  97. package/simsimd/include/simsimd/types.h +309 -0
  98. package/simsimd/javascript/bench.js +127 -0
  99. package/simsimd/javascript/dist-package-cjs.json +3 -0
  100. package/simsimd/javascript/dist-package-esm.json +3 -0
  101. package/simsimd/javascript/fallback.ts +203 -0
  102. package/simsimd/javascript/lib.c +107 -0
  103. package/simsimd/javascript/node-gyp-build.d.ts +1 -0
  104. package/simsimd/javascript/simsimd.ts +135 -0
  105. package/simsimd/javascript/test.mjs +179 -0
  106. package/simsimd/javascript/tsconfig-base.json +37 -0
  107. package/simsimd/javascript/tsconfig-cjs.json +8 -0
  108. package/simsimd/javascript/tsconfig-esm.json +8 -0
  109. package/simsimd/package-lock.json +1709 -0
  110. package/simsimd/package.json +61 -0
  111. package/simsimd/rust/benches/cosine.rs +27 -0
  112. package/simsimd/rust/benches/native.rs +30 -0
  113. package/simsimd/rust/benches/sqeuclidean.rs +27 -0
  114. package/simsimd/rust/lib.rs +778 -0
@@ -0,0 +1,1237 @@
1
+ /**
2
+ * @file dot.h
3
+ * @brief SIMD-accelerated Dot Products for Real and Complex numbers.
4
+ * @author Ash Vardanian
5
+ * @date February 24, 2024
6
+ *
7
+ * Contains:
8
+ * - Dot Product for Real and Complex vectors
9
+ * - Conjugate Dot Product for Complex vectors
10
+ *
11
+ * For datatypes:
12
+ * - 64-bit IEEE floating point numbers
13
+ * - 32-bit IEEE floating point numbers
14
+ * - 16-bit IEEE floating point numbers
15
+ *
16
+ * For hardware architectures:
17
+ * - Arm (NEON, SVE?)
18
+ * - x86 (AVX2?, AVX512?)
19
+ *
20
+ * x86 intrinsics: https://www.intel.com/content/www/us/en/docs/intrinsics-guide/
21
+ * Arm intrinsics: https://developer.arm.com/architectures/instruction-sets/intrinsics/
22
+ */
23
+ #ifndef SIMSIMD_DOT_H
24
+ #define SIMSIMD_DOT_H
25
+
26
+ #include "types.h"
27
+
28
+ #ifdef __cplusplus
29
+ extern "C" {
30
+ #endif
31
+
32
+ // clang-format off
33
+
34
+ /* Serial backends for all numeric types.
35
+ * By default they use 32-bit arithmetic, unless the arguments themselves contain 64-bit floats.
36
+ * For double-precision computation check out the "*_accurate" variants of those "*_serial" functions.
37
+ */
38
+ SIMSIMD_PUBLIC void simsimd_dot_f64_serial(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* result);
39
+ SIMSIMD_PUBLIC void simsimd_dot_f64c_serial(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* results);
40
+ SIMSIMD_PUBLIC void simsimd_vdot_f64c_serial(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* results);
41
+ SIMSIMD_PUBLIC void simsimd_dot_f32_serial(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* result);
42
+ SIMSIMD_PUBLIC void simsimd_dot_f32c_serial(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* results);
43
+ SIMSIMD_PUBLIC void simsimd_vdot_f32c_serial(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* results);
44
+ SIMSIMD_PUBLIC void simsimd_dot_f16_serial(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* result);
45
+ SIMSIMD_PUBLIC void simsimd_dot_f16c_serial(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* results);
46
+ SIMSIMD_PUBLIC void simsimd_vdot_f16c_serial(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* results);
47
+
48
+ /* Double-precision serial backends for all numeric types.
49
+ * For single-precision computation check out the "*_serial" counterparts of those "*_accurate" functions.
50
+ */
51
+ SIMSIMD_PUBLIC void simsimd_dot_f32_accurate(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* result);
52
+ SIMSIMD_PUBLIC void simsimd_dot_f32c_accurate(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* results);
53
+ SIMSIMD_PUBLIC void simsimd_vdot_f32c_accurate(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* results);
54
+ SIMSIMD_PUBLIC void simsimd_dot_f16_accurate(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* result);
55
+ SIMSIMD_PUBLIC void simsimd_dot_f16c_accurate(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* results);
56
+ SIMSIMD_PUBLIC void simsimd_vdot_f16c_accurate(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* results);
57
+
58
+ /* SIMD-powered backends for Arm NEON, mostly using 32-bit arithmetic over 128-bit words.
59
+ * By far the most portable backend, covering most Arm v8 devices, over a billion phones, and almost all
60
+ * server CPUs produced before 2023.
61
+ */
62
+ SIMSIMD_PUBLIC void simsimd_dot_f32_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* result);
63
+ SIMSIMD_PUBLIC void simsimd_dot_f16_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* result);
64
+ SIMSIMD_PUBLIC void simsimd_dot_f32c_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* results);
65
+ SIMSIMD_PUBLIC void simsimd_dot_f16c_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* results);
66
+ SIMSIMD_PUBLIC void simsimd_vdot_f32c_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* results);
67
+ SIMSIMD_PUBLIC void simsimd_vdot_f16c_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* results);
68
+
69
+ /* SIMD-powered backends for Arm SVE, mostly using 32-bit arithmetic over variable-length platform-defined word sizes.
70
+ * Designed for Arm Graviton 3, Microsoft Cobalt, as well as Nvidia Grace and newer Ampere Altra CPUs.
71
+ */
72
+ SIMSIMD_PUBLIC void simsimd_dot_f16_sve(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* result);
73
+ SIMSIMD_PUBLIC void simsimd_dot_f32_sve(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* result);
74
+ SIMSIMD_PUBLIC void simsimd_dot_f64_sve(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* result);
75
+ SIMSIMD_PUBLIC void simsimd_dot_f16c_sve(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* results);
76
+ SIMSIMD_PUBLIC void simsimd_dot_f32c_sve(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* results);
77
+ SIMSIMD_PUBLIC void simsimd_dot_f64c_sve(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* results);
78
+ SIMSIMD_PUBLIC void simsimd_vdot_f16c_sve(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* results);
79
+ SIMSIMD_PUBLIC void simsimd_vdot_f32c_sve(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* results);
80
+ SIMSIMD_PUBLIC void simsimd_vdot_f64c_sve(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* results);
81
+
82
+ /* SIMD-powered backends for AVX2 CPUs of Haswell generation and newer, using 32-bit arithmetic over 256-bit words.
83
+ * First demonstrated in 2011, at least one Haswell-based processor was still being sold in 2022 — the Pentium G3420.
84
+ * Practically all modern x86 CPUs support AVX2, FMA, and F16C, making it a perfect baseline for SIMD algorithms.
85
+ * On other hand, there is no need to implement AVX2 versions of `f32` and `f64` functions, as those are
86
+ * properly vectorized by recent compilers.
87
+ */
88
+ SIMSIMD_PUBLIC void simsimd_dot_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* result);
89
+ SIMSIMD_PUBLIC void simsimd_dot_f32c_haswell(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* results);
90
+ SIMSIMD_PUBLIC void simsimd_vdot_f32c_haswell(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* results);
91
+ SIMSIMD_PUBLIC void simsimd_dot_f16c_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* results);
92
+ SIMSIMD_PUBLIC void simsimd_vdot_f16c_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* results);
93
+
94
+ /* SIMD-powered backends for various generations of AVX512 CPUs.
95
+ * Skylake is handy, as it supports masked loads and other operations, avoiding the need for the tail loop.
96
+ * Ice Lake added VNNI, VPOPCNTDQ, IFMA, VBMI, VAES, GFNI, VBMI2, BITALG, VPCLMULQDQ, and other extensions for integral operations.
97
+ * Sapphire Rapids added tiled matrix operations, but we are most interested in the new mixed-precision FMA instructions.
98
+ */
99
+ SIMSIMD_PUBLIC void simsimd_dot_f32_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* result);
100
+ SIMSIMD_PUBLIC void simsimd_dot_f32c_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* result);
101
+ SIMSIMD_PUBLIC void simsimd_vdot_f32c_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* result);
102
+ SIMSIMD_PUBLIC void simsimd_dot_f64_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* result);
103
+ SIMSIMD_PUBLIC void simsimd_dot_f64c_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* result);
104
+ SIMSIMD_PUBLIC void simsimd_vdot_f64c_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* result);
105
+
106
+ SIMSIMD_PUBLIC void simsimd_dot_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* result);
107
+ SIMSIMD_PUBLIC void simsimd_dot_f16c_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* result);
108
+ SIMSIMD_PUBLIC void simsimd_vdot_f16c_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* result);
109
+ // clang-format on
110
+
111
+ #define SIMSIMD_MAKE_DOT(name, input_type, accumulator_type, converter) \
112
+ SIMSIMD_PUBLIC void simsimd_dot_##input_type##_##name(simsimd_##input_type##_t const* a, \
113
+ simsimd_##input_type##_t const* b, simsimd_size_t n, \
114
+ simsimd_distance_t* result) { \
115
+ simsimd_##accumulator_type##_t ab = 0; \
116
+ for (simsimd_size_t i = 0; i != n; ++i) { \
117
+ simsimd_##accumulator_type##_t ai = converter(a[i]); \
118
+ simsimd_##accumulator_type##_t bi = converter(b[i]); \
119
+ ab += ai * bi; \
120
+ } \
121
+ *result = ab; \
122
+ }
123
+
124
+ #define SIMSIMD_MAKE_COMPLEX_DOT(name, input_type, accumulator_type, converter) \
125
+ SIMSIMD_PUBLIC void simsimd_dot_##input_type##c_##name(simsimd_##input_type##_t const* a, \
126
+ simsimd_##input_type##_t const* b, simsimd_size_t n, \
127
+ simsimd_distance_t* results) { \
128
+ simsimd_##accumulator_type##_t ab_real = 0, ab_imag = 0; \
129
+ for (simsimd_size_t i = 0; i + 2 <= n; i += 2) { \
130
+ simsimd_##accumulator_type##_t ar = converter(a[i]); \
131
+ simsimd_##accumulator_type##_t br = converter(b[i]); \
132
+ simsimd_##accumulator_type##_t ai = converter(a[i + 1]); \
133
+ simsimd_##accumulator_type##_t bi = converter(b[i + 1]); \
134
+ ab_real += ar * br - ai * bi; \
135
+ ab_imag += ar * bi + ai * br; \
136
+ } \
137
+ results[0] = ab_real; \
138
+ results[1] = ab_imag; \
139
+ }
140
+
141
+ #define SIMSIMD_MAKE_COMPLEX_VDOT(name, input_type, accumulator_type, converter) \
142
+ SIMSIMD_PUBLIC void simsimd_vdot_##input_type##c_##name(simsimd_##input_type##_t const* a, \
143
+ simsimd_##input_type##_t const* b, simsimd_size_t n, \
144
+ simsimd_distance_t* results) { \
145
+ simsimd_##accumulator_type##_t ab_real = 0, ab_imag = 0; \
146
+ for (simsimd_size_t i = 0; i + 2 <= n; i += 2) { \
147
+ simsimd_##accumulator_type##_t ar = converter(a[i]); \
148
+ simsimd_##accumulator_type##_t br = converter(b[i]); \
149
+ simsimd_##accumulator_type##_t ai = converter(a[i + 1]); \
150
+ simsimd_##accumulator_type##_t bi = converter(b[i + 1]); \
151
+ ab_real += ar * br + ai * bi; \
152
+ ab_imag += ar * bi - ai * br; \
153
+ } \
154
+ results[0] = ab_real; \
155
+ results[1] = ab_imag; \
156
+ }
157
+
158
+ SIMSIMD_MAKE_DOT(serial, f64, f64, SIMSIMD_IDENTIFY) // simsimd_dot_f64_serial
159
+ SIMSIMD_MAKE_COMPLEX_DOT(serial, f64, f64, SIMSIMD_IDENTIFY) // simsimd_dot_f64c_serial
160
+ SIMSIMD_MAKE_COMPLEX_VDOT(serial, f64, f64, SIMSIMD_IDENTIFY) // simsimd_vdot_f64c_serial
161
+
162
+ SIMSIMD_MAKE_DOT(serial, f32, f32, SIMSIMD_IDENTIFY) // simsimd_dot_f32_serial
163
+ SIMSIMD_MAKE_COMPLEX_DOT(serial, f32, f32, SIMSIMD_IDENTIFY) // simsimd_dot_f32c_serial
164
+ SIMSIMD_MAKE_COMPLEX_VDOT(serial, f32, f32, SIMSIMD_IDENTIFY) // simsimd_vdot_f32c_serial
165
+
166
+ SIMSIMD_MAKE_DOT(serial, f16, f32, SIMSIMD_UNCOMPRESS_F16) // simsimd_dot_f16_serial
167
+ SIMSIMD_MAKE_COMPLEX_DOT(serial, f16, f32, SIMSIMD_UNCOMPRESS_F16) // simsimd_dot_f16c_serial
168
+ SIMSIMD_MAKE_COMPLEX_VDOT(serial, f16, f32, SIMSIMD_UNCOMPRESS_F16) // simsimd_vdot_f16c_serial
169
+
170
+ SIMSIMD_MAKE_DOT(accurate, f32, f64, SIMSIMD_IDENTIFY) // simsimd_dot_f32_accurate
171
+ SIMSIMD_MAKE_COMPLEX_DOT(accurate, f32, f64, SIMSIMD_IDENTIFY) // simsimd_dot_f32c_accurate
172
+ SIMSIMD_MAKE_COMPLEX_VDOT(accurate, f32, f64, SIMSIMD_IDENTIFY) // simsimd_vdot_f32c_accurate
173
+
174
+ SIMSIMD_MAKE_DOT(accurate, f16, f64, SIMSIMD_UNCOMPRESS_F16) // simsimd_dot_f16_accurate
175
+ SIMSIMD_MAKE_COMPLEX_DOT(accurate, f16, f64, SIMSIMD_UNCOMPRESS_F16) // simsimd_dot_f16c_accurate
176
+ SIMSIMD_MAKE_COMPLEX_VDOT(accurate, f16, f64, SIMSIMD_UNCOMPRESS_F16) // simsimd_vdot_f16c_accurate
177
+
178
+ #if SIMSIMD_TARGET_ARM
179
+ #if SIMSIMD_TARGET_NEON
180
+ #pragma GCC push_options
181
+ #pragma GCC target("+simd")
182
+ #pragma clang attribute push(__attribute__((target("+simd"))), apply_to = function)
183
+
184
+ SIMSIMD_PUBLIC void simsimd_dot_f32_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
185
+ simsimd_distance_t* result) {
186
+ float32x4_t ab_vec = vdupq_n_f32(0);
187
+ simsimd_size_t i = 0;
188
+ for (; i + 4 <= n; i += 4) {
189
+ float32x4_t a_vec = vld1q_f32(a + i);
190
+ float32x4_t b_vec = vld1q_f32(b + i);
191
+ ab_vec = vfmaq_f32(ab_vec, a_vec, b_vec);
192
+ }
193
+ simsimd_f32_t ab = vaddvq_f32(ab_vec);
194
+ for (; i < n; ++i)
195
+ ab += a[i] * b[i];
196
+ *result = ab;
197
+ }
198
+
199
+ SIMSIMD_PUBLIC void simsimd_dot_f32c_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, //
200
+ simsimd_distance_t* results) {
201
+ float32x4_t ab_real_vec = vdupq_n_f32(0);
202
+ float32x4_t ab_imag_vec = vdupq_n_f32(0);
203
+ simsimd_size_t i = 0;
204
+ for (; i + 8 <= n; i += 8) {
205
+ // Unpack the input arrays into real and imaginary parts:
206
+ float32x4x2_t a_vec = vld2q_f32(a + i);
207
+ float32x4x2_t b_vec = vld2q_f32(b + i);
208
+ float32x4_t a_real_vec = a_vec.val[0];
209
+ float32x4_t a_imag_vec = a_vec.val[1];
210
+ float32x4_t b_real_vec = b_vec.val[0];
211
+ float32x4_t b_imag_vec = b_vec.val[1];
212
+
213
+ // Compute the dot product:
214
+ ab_real_vec = vfmaq_f32(ab_real_vec, a_real_vec, b_real_vec);
215
+ ab_real_vec = vfmsq_f32(ab_real_vec, a_imag_vec, b_imag_vec);
216
+ ab_imag_vec = vfmaq_f32(ab_imag_vec, a_real_vec, b_imag_vec);
217
+ ab_imag_vec = vfmaq_f32(ab_imag_vec, a_imag_vec, b_real_vec);
218
+ }
219
+
220
+ // Reduce horizontal sums:
221
+ simsimd_f32_t ab_real = vaddvq_f32(ab_real_vec);
222
+ simsimd_f32_t ab_imag = vaddvq_f32(ab_imag_vec);
223
+
224
+ // Handle the tail:
225
+ for (; i + 2 <= n; i += 2) {
226
+ simsimd_f32_t ar = a[i], ai = a[i + 1], br = b[i], bi = b[i + 1];
227
+ ab_real += ar * br - ai * bi;
228
+ ab_imag += ar * bi + ai * br;
229
+ }
230
+ results[0] = ab_real;
231
+ results[1] = ab_imag;
232
+ }
233
+
234
+ SIMSIMD_PUBLIC void simsimd_vdot_f32c_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, //
235
+ simsimd_distance_t* results) {
236
+ float32x4_t ab_real_vec = vdupq_n_f32(0);
237
+ float32x4_t ab_imag_vec = vdupq_n_f32(0);
238
+ simsimd_size_t i = 0;
239
+ for (; i + 8 <= n; i += 8) {
240
+ // Unpack the input arrays into real and imaginary parts:
241
+ float32x4x2_t a_vec = vld2q_f32(a + i);
242
+ float32x4x2_t b_vec = vld2q_f32(b + i);
243
+ float32x4_t a_real_vec = a_vec.val[0];
244
+ float32x4_t a_imag_vec = a_vec.val[1];
245
+ float32x4_t b_real_vec = b_vec.val[0];
246
+ float32x4_t b_imag_vec = b_vec.val[1];
247
+
248
+ // Compute the dot product:
249
+ ab_real_vec = vfmaq_f32(ab_real_vec, a_real_vec, b_real_vec);
250
+ ab_real_vec = vfmaq_f32(ab_real_vec, a_imag_vec, b_imag_vec);
251
+ ab_imag_vec = vfmaq_f32(ab_imag_vec, a_real_vec, b_imag_vec);
252
+ ab_imag_vec = vfmsq_f32(ab_imag_vec, a_imag_vec, b_real_vec);
253
+ }
254
+
255
+ // Reduce horizontal sums:
256
+ simsimd_f32_t ab_real = vaddvq_f32(ab_real_vec);
257
+ simsimd_f32_t ab_imag = vaddvq_f32(ab_imag_vec);
258
+
259
+ // Handle the tail:
260
+ for (; i + 2 <= n; i += 2) {
261
+ simsimd_f32_t ar = a[i], ai = a[i + 1], br = b[i], bi = b[i + 1];
262
+ ab_real += ar * br + ai * bi;
263
+ ab_imag += ar * bi - ai * br;
264
+ }
265
+ results[0] = ab_real;
266
+ results[1] = ab_imag;
267
+ }
268
+
269
+ #pragma clang attribute pop
270
+ #pragma GCC pop_options
271
+
272
+ #pragma GCC push_options
273
+ #pragma GCC target("+simd+fp16")
274
+ #pragma clang attribute push(__attribute__((target("+simd+fp16"))), apply_to = function)
275
+
276
+ SIMSIMD_PUBLIC void simsimd_dot_f16_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
277
+ simsimd_distance_t* result) {
278
+ float32x4_t ab_vec = vdupq_n_f32(0);
279
+ simsimd_size_t i = 0;
280
+ for (; i + 4 <= n; i += 4) {
281
+ float32x4_t a_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)a + i));
282
+ float32x4_t b_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)b + i));
283
+ ab_vec = vfmaq_f32(ab_vec, a_vec, b_vec);
284
+ }
285
+
286
+ // In case the software emulation for `f16` scalars is enabled, the `simsimd_uncompress_f16`
287
+ // function will run. It is extremely slow, so even for the tail, let's combine serial
288
+ // loads and stores with vectorized math.
289
+ if (i < n) {
290
+ union {
291
+ float16x4_t f16_vec;
292
+ simsimd_f16_t f16[4];
293
+ } a_padded_tail, b_padded_tail;
294
+ simsimd_size_t j = 0;
295
+ for (; i < n; ++i, ++j)
296
+ a_padded_tail.f16[j] = a[i], b_padded_tail.f16[j] = b[i];
297
+ for (; j < 4; ++j)
298
+ a_padded_tail.f16[j] = 0, b_padded_tail.f16[j] = 0;
299
+ ab_vec = vfmaq_f32(ab_vec, vcvt_f32_f16(a_padded_tail.f16_vec), vcvt_f32_f16(b_padded_tail.f16_vec));
300
+ }
301
+ *result = vaddvq_f32(ab_vec);
302
+ }
303
+
304
+ SIMSIMD_PUBLIC void simsimd_dot_f16c_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, //
305
+ simsimd_distance_t* results) {
306
+
307
+ // A nicer approach is to use `f16` arithmetic for the dot product, but that requires
308
+ // FMLA extensions available on Arm v8.3 and later. That we can also process 16 entries
309
+ // at once. That's how the original implementation worked, but compiling it was a nightmare :)
310
+ float32x4_t ab_real_vec = vdupq_n_f32(0);
311
+ float32x4_t ab_imag_vec = vdupq_n_f32(0);
312
+ simsimd_size_t i = 0;
313
+ for (; i + 8 <= n; i += 8) {
314
+ // Unpack the input arrays into real and imaginary parts.
315
+ // MSVC sadly doesn't recognize the `vld2_f16`, so we load the data as signed
316
+ // integers of the same size and reinterpret with `vreinterpret_f16_s16` afterwards.
317
+ int16x4x2_t a_vec = vld2_s16((short*)a + i);
318
+ int16x4x2_t b_vec = vld2_s16((short*)b + i);
319
+ float32x4_t a_real_vec = vcvt_f32_f16(vreinterpret_f16_s16(a_vec.val[0]));
320
+ float32x4_t a_imag_vec = vcvt_f32_f16(vreinterpret_f16_s16(a_vec.val[1]));
321
+ float32x4_t b_real_vec = vcvt_f32_f16(vreinterpret_f16_s16(b_vec.val[0]));
322
+ float32x4_t b_imag_vec = vcvt_f32_f16(vreinterpret_f16_s16(b_vec.val[1]));
323
+
324
+ // Compute the dot product:
325
+ ab_real_vec = vfmaq_f32(ab_real_vec, a_real_vec, b_real_vec);
326
+ ab_real_vec = vfmsq_f32(ab_real_vec, a_imag_vec, b_imag_vec);
327
+ ab_imag_vec = vfmaq_f32(ab_imag_vec, a_real_vec, b_imag_vec);
328
+ ab_imag_vec = vfmaq_f32(ab_imag_vec, a_imag_vec, b_real_vec);
329
+ }
330
+
331
+ // Reduce horizontal sums:
332
+ simsimd_f32_t ab_real = vaddvq_f32(ab_real_vec);
333
+ simsimd_f32_t ab_imag = vaddvq_f32(ab_imag_vec);
334
+
335
+ // Handle the tail:
336
+ for (; i + 2 <= n; i += 2) {
337
+ simsimd_f32_t ar = a[i], ai = a[i + 1], br = b[i], bi = b[i + 1];
338
+ ab_real += ar * br - ai * bi;
339
+ ab_imag += ar * bi + ai * br;
340
+ }
341
+ results[0] = ab_real;
342
+ results[1] = ab_imag;
343
+ }
344
+
345
+ SIMSIMD_PUBLIC void simsimd_vdot_f16c_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, //
346
+ simsimd_distance_t* results) {
347
+
348
+ // A nicer approach is to use `f16` arithmetic for the dot product, but that requires
349
+ // FMLA extensions available on Arm v8.3 and later. That we can also process 16 entries
350
+ // at once. That's how the original implementation worked, but compiling it was a nightmare :)
351
+ float32x4_t ab_real_vec = vdupq_n_f32(0);
352
+ float32x4_t ab_imag_vec = vdupq_n_f32(0);
353
+ simsimd_size_t i = 0;
354
+ for (; i + 8 <= n; i += 8) {
355
+ // Unpack the input arrays into real and imaginary parts.
356
+ // MSVC sadly doesn't recognize the `vld2_f16`, so we load the data as signed
357
+ // integers of the same size and reinterpret with `vreinterpret_f16_s16` afterwards.
358
+ int16x4x2_t a_vec = vld2_s16((short*)a + i);
359
+ int16x4x2_t b_vec = vld2_s16((short*)b + i);
360
+ float32x4_t a_real_vec = vcvt_f32_f16(vreinterpret_f16_s16(a_vec.val[0]));
361
+ float32x4_t a_imag_vec = vcvt_f32_f16(vreinterpret_f16_s16(a_vec.val[1]));
362
+ float32x4_t b_real_vec = vcvt_f32_f16(vreinterpret_f16_s16(b_vec.val[0]));
363
+ float32x4_t b_imag_vec = vcvt_f32_f16(vreinterpret_f16_s16(b_vec.val[1]));
364
+
365
+ // Compute the dot product:
366
+ ab_real_vec = vfmaq_f32(ab_real_vec, a_real_vec, b_real_vec);
367
+ ab_real_vec = vfmaq_f32(ab_real_vec, a_imag_vec, b_imag_vec);
368
+ ab_imag_vec = vfmaq_f32(ab_imag_vec, a_real_vec, b_imag_vec);
369
+ ab_imag_vec = vfmsq_f32(ab_imag_vec, a_imag_vec, b_real_vec);
370
+ }
371
+
372
+ // Reduce horizontal sums:
373
+ simsimd_f32_t ab_real = vaddvq_f32(ab_real_vec);
374
+ simsimd_f32_t ab_imag = vaddvq_f32(ab_imag_vec);
375
+
376
+ // Handle the tail:
377
+ for (; i + 2 <= n; i += 2) {
378
+ simsimd_f32_t ar = a[i], ai = a[i + 1], br = b[i], bi = b[i + 1];
379
+ ab_real += ar * br + ai * bi;
380
+ ab_imag += ar * bi - ai * br;
381
+ }
382
+ results[0] = ab_real;
383
+ results[1] = ab_imag;
384
+ }
385
+
386
+ #pragma clang attribute pop
387
+ #pragma GCC pop_options
388
+ #endif // SIMSIMD_TARGET_NEON
389
+
390
+ #if SIMSIMD_TARGET_SVE
391
+
392
+ #pragma GCC push_options
393
+ #pragma GCC target("+sve")
394
+ #pragma clang attribute push(__attribute__((target("+sve"))), apply_to = function)
395
+
396
+ SIMSIMD_PUBLIC void simsimd_dot_f32_sve(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
397
+ simsimd_distance_t* result) {
398
+ simsimd_size_t i = 0;
399
+ svfloat32_t ab_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
400
+ do {
401
+ svbool_t pg_vec = svwhilelt_b32((unsigned int)i, (unsigned int)n);
402
+ svfloat32_t a_vec = svld1_f32(pg_vec, a + i);
403
+ svfloat32_t b_vec = svld1_f32(pg_vec, b + i);
404
+ ab_vec = svmla_f32_x(pg_vec, ab_vec, a_vec, b_vec);
405
+ i += svcntw();
406
+ } while (i < n);
407
+ *result = svaddv_f32(svptrue_b32(), ab_vec);
408
+ }
409
+
410
+ SIMSIMD_PUBLIC void simsimd_dot_f32c_sve(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
411
+ simsimd_distance_t* results) {
412
+ simsimd_size_t i = 0;
413
+ svfloat32_t ab_real_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
414
+ svfloat32_t ab_imag_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
415
+ do {
416
+ svbool_t pg_vec = svwhilelt_b32((unsigned int)i, (unsigned int)n);
417
+ svfloat32x2_t a_vec = svld2_f32(pg_vec, a + i);
418
+ svfloat32x2_t b_vec = svld2_f32(pg_vec, b + i);
419
+ svfloat32_t a_real_vec = svget2_f32(a_vec, 0);
420
+ svfloat32_t a_imag_vec = svget2_f32(a_vec, 1);
421
+ svfloat32_t b_real_vec = svget2_f32(b_vec, 0);
422
+ svfloat32_t b_imag_vec = svget2_f32(b_vec, 1);
423
+ ab_real_vec = svmla_f32_x(pg_vec, ab_real_vec, a_real_vec, b_real_vec);
424
+ ab_real_vec = svmls_f32_x(pg_vec, ab_real_vec, a_imag_vec, b_imag_vec);
425
+ ab_imag_vec = svmla_f32_x(pg_vec, ab_imag_vec, a_real_vec, b_imag_vec);
426
+ ab_imag_vec = svmla_f32_x(pg_vec, ab_imag_vec, a_imag_vec, b_real_vec);
427
+ i += svcntw() * 2;
428
+ } while (i < n);
429
+ results[0] = svaddv_f32(svptrue_b32(), ab_real_vec);
430
+ results[1] = svaddv_f32(svptrue_b32(), ab_imag_vec);
431
+ }
432
+
433
+ SIMSIMD_PUBLIC void simsimd_vdot_f32c_sve(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
434
+ simsimd_distance_t* results) {
435
+ simsimd_size_t i = 0;
436
+ svfloat32_t ab_real_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
437
+ svfloat32_t ab_imag_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
438
+ do {
439
+ svbool_t pg_vec = svwhilelt_b32((unsigned int)i, (unsigned int)n);
440
+ svfloat32x2_t a_vec = svld2_f32(pg_vec, a + i);
441
+ svfloat32x2_t b_vec = svld2_f32(pg_vec, b + i);
442
+ svfloat32_t a_real_vec = svget2_f32(a_vec, 0);
443
+ svfloat32_t a_imag_vec = svget2_f32(a_vec, 1);
444
+ svfloat32_t b_real_vec = svget2_f32(b_vec, 0);
445
+ svfloat32_t b_imag_vec = svget2_f32(b_vec, 1);
446
+ ab_real_vec = svmla_f32_x(pg_vec, ab_real_vec, a_real_vec, b_real_vec);
447
+ ab_real_vec = svmla_f32_x(pg_vec, ab_real_vec, a_imag_vec, b_imag_vec);
448
+ ab_imag_vec = svmla_f32_x(pg_vec, ab_imag_vec, a_real_vec, b_imag_vec);
449
+ ab_imag_vec = svmls_f32_x(pg_vec, ab_imag_vec, a_imag_vec, b_real_vec);
450
+ i += svcntw() * 2;
451
+ } while (i < n);
452
+ results[0] = svaddv_f32(svptrue_b32(), ab_real_vec);
453
+ results[1] = svaddv_f32(svptrue_b32(), ab_imag_vec);
454
+ }
455
+
456
+ SIMSIMD_PUBLIC void simsimd_dot_f64_sve(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n,
457
+ simsimd_distance_t* result) {
458
+ simsimd_size_t i = 0;
459
+ svfloat64_t ab_vec = svdupq_n_f64(0.0, 0.0);
460
+ do {
461
+ svbool_t pg_vec = svwhilelt_b64((unsigned int)i, (unsigned int)n);
462
+ svfloat64_t a_vec = svld1_f64(pg_vec, a + i);
463
+ svfloat64_t b_vec = svld1_f64(pg_vec, b + i);
464
+ ab_vec = svmla_f64_x(pg_vec, ab_vec, a_vec, b_vec);
465
+ i += svcntd();
466
+ } while (i < n);
467
+ *result = svaddv_f64(svptrue_b32(), ab_vec);
468
+ }
469
+
470
+ SIMSIMD_PUBLIC void simsimd_dot_f64c_sve(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n,
471
+ simsimd_distance_t* results) {
472
+ simsimd_size_t i = 0;
473
+ svfloat64_t ab_real_vec = svdupq_n_f64(0., 0.);
474
+ svfloat64_t ab_imag_vec = svdupq_n_f64(0., 0.);
475
+ do {
476
+ svbool_t pg_vec = svwhilelt_b64((unsigned int)i, (unsigned int)n);
477
+ svfloat64x2_t a_vec = svld2_f64(pg_vec, a + i);
478
+ svfloat64x2_t b_vec = svld2_f64(pg_vec, b + i);
479
+ svfloat64_t a_real_vec = svget2_f64(a_vec, 0);
480
+ svfloat64_t a_imag_vec = svget2_f64(a_vec, 1);
481
+ svfloat64_t b_real_vec = svget2_f64(b_vec, 0);
482
+ svfloat64_t b_imag_vec = svget2_f64(b_vec, 1);
483
+ ab_real_vec = svmla_f64_x(pg_vec, ab_real_vec, a_real_vec, b_real_vec);
484
+ ab_real_vec = svmls_f64_x(pg_vec, ab_real_vec, a_imag_vec, b_imag_vec);
485
+ ab_imag_vec = svmla_f64_x(pg_vec, ab_imag_vec, a_real_vec, b_imag_vec);
486
+ ab_imag_vec = svmla_f64_x(pg_vec, ab_imag_vec, a_imag_vec, b_real_vec);
487
+ i += svcntd() * 2;
488
+ } while (i < n);
489
+ results[0] = svaddv_f64(svptrue_b64(), ab_real_vec);
490
+ results[1] = svaddv_f64(svptrue_b64(), ab_imag_vec);
491
+ }
492
+
493
+ SIMSIMD_PUBLIC void simsimd_vdot_f64c_sve(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n,
494
+ simsimd_distance_t* results) {
495
+ simsimd_size_t i = 0;
496
+ svfloat64_t ab_real_vec = svdupq_n_f64(0., 0.);
497
+ svfloat64_t ab_imag_vec = svdupq_n_f64(0., 0.);
498
+ do {
499
+ svbool_t pg_vec = svwhilelt_b64((unsigned int)i, (unsigned int)n);
500
+ svfloat64x2_t a_vec = svld2_f64(pg_vec, a + i);
501
+ svfloat64x2_t b_vec = svld2_f64(pg_vec, b + i);
502
+ svfloat64_t a_real_vec = svget2_f64(a_vec, 0);
503
+ svfloat64_t a_imag_vec = svget2_f64(a_vec, 1);
504
+ svfloat64_t b_real_vec = svget2_f64(b_vec, 0);
505
+ svfloat64_t b_imag_vec = svget2_f64(b_vec, 1);
506
+ ab_real_vec = svmla_f64_x(pg_vec, ab_real_vec, a_real_vec, b_real_vec);
507
+ ab_real_vec = svmla_f64_x(pg_vec, ab_real_vec, a_imag_vec, b_imag_vec);
508
+ ab_imag_vec = svmla_f64_x(pg_vec, ab_imag_vec, a_real_vec, b_imag_vec);
509
+ ab_imag_vec = svmls_f64_x(pg_vec, ab_imag_vec, a_imag_vec, b_real_vec);
510
+ i += svcntd() * 2;
511
+ } while (i < n);
512
+ results[0] = svaddv_f64(svptrue_b64(), ab_real_vec);
513
+ results[1] = svaddv_f64(svptrue_b64(), ab_imag_vec);
514
+ }
515
+
516
+ #pragma clang attribute pop
517
+ #pragma GCC pop_options
518
+
519
+ #pragma GCC push_options
520
+ #pragma GCC target("+sve+fp16")
521
+ #pragma clang attribute push(__attribute__((target("+sve+fp16"))), apply_to = function)
522
+
523
+ SIMSIMD_PUBLIC void simsimd_dot_f16_sve(simsimd_f16_t const* a_enum, simsimd_f16_t const* b_enum, simsimd_size_t n,
524
+ simsimd_distance_t* result) {
525
+ simsimd_size_t i = 0;
526
+ svfloat16_t ab_vec = svdupq_n_f16(0, 0, 0, 0, 0, 0, 0, 0);
527
+ simsimd_f16_for_arm_simd_t const* a = (simsimd_f16_for_arm_simd_t const*)(a_enum);
528
+ simsimd_f16_for_arm_simd_t const* b = (simsimd_f16_for_arm_simd_t const*)(b_enum);
529
+ do {
530
+ svbool_t pg_vec = svwhilelt_b16((unsigned int)i, (unsigned int)n);
531
+ svfloat16_t a_vec = svld1_f16(pg_vec, a + i);
532
+ svfloat16_t b_vec = svld1_f16(pg_vec, b + i);
533
+ ab_vec = svmla_f16_x(pg_vec, ab_vec, a_vec, b_vec);
534
+ i += svcnth();
535
+ } while (i < n);
536
+ simsimd_f16_for_arm_simd_t ab = svaddv_f16(svptrue_b16(), ab_vec);
537
+ *result = ab;
538
+ }
539
+
540
+ SIMSIMD_PUBLIC void simsimd_dot_f16c_sve(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
541
+ simsimd_distance_t* results) {
542
+ simsimd_size_t i = 0;
543
+ svfloat16_t ab_real_vec = svdupq_n_f16(0, 0, 0, 0, 0, 0, 0, 0);
544
+ svfloat16_t ab_imag_vec = svdupq_n_f16(0, 0, 0, 0, 0, 0, 0, 0);
545
+ do {
546
+ svbool_t pg_vec = svwhilelt_b16((unsigned int)i, (unsigned int)n);
547
+ svfloat16x2_t a_vec = svld2_f16(pg_vec, (simsimd_f16_for_arm_simd_t const*)a + i);
548
+ svfloat16x2_t b_vec = svld2_f16(pg_vec, (simsimd_f16_for_arm_simd_t const*)b + i);
549
+ svfloat16_t a_real_vec = svget2_f16(a_vec, 0);
550
+ svfloat16_t a_imag_vec = svget2_f16(a_vec, 1);
551
+ svfloat16_t b_real_vec = svget2_f16(b_vec, 0);
552
+ svfloat16_t b_imag_vec = svget2_f16(b_vec, 1);
553
+ ab_real_vec = svmla_f16_x(pg_vec, ab_real_vec, a_real_vec, b_real_vec);
554
+ ab_real_vec = svmls_f16_x(pg_vec, ab_real_vec, a_imag_vec, b_imag_vec);
555
+ ab_imag_vec = svmla_f16_x(pg_vec, ab_imag_vec, a_real_vec, b_imag_vec);
556
+ ab_imag_vec = svmla_f16_x(pg_vec, ab_imag_vec, a_imag_vec, b_real_vec);
557
+ i += svcnth() * 2;
558
+ } while (i < n);
559
+ results[0] = svaddv_f16(svptrue_b16(), ab_real_vec);
560
+ results[1] = svaddv_f16(svptrue_b16(), ab_imag_vec);
561
+ }
562
+
563
+ SIMSIMD_PUBLIC void simsimd_vdot_f16c_sve(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
564
+ simsimd_distance_t* results) {
565
+ simsimd_size_t i = 0;
566
+ svfloat16_t ab_real_vec = svdupq_n_f16(0, 0, 0, 0, 0, 0, 0, 0);
567
+ svfloat16_t ab_imag_vec = svdupq_n_f16(0, 0, 0, 0, 0, 0, 0, 0);
568
+ do {
569
+ svbool_t pg_vec = svwhilelt_b16((unsigned int)i, (unsigned int)n);
570
+ svfloat16x2_t a_vec = svld2_f16(pg_vec, (simsimd_f16_for_arm_simd_t const*)a + i);
571
+ svfloat16x2_t b_vec = svld2_f16(pg_vec, (simsimd_f16_for_arm_simd_t const*)b + i);
572
+ svfloat16_t a_real_vec = svget2_f16(a_vec, 0);
573
+ svfloat16_t a_imag_vec = svget2_f16(a_vec, 1);
574
+ svfloat16_t b_real_vec = svget2_f16(b_vec, 0);
575
+ svfloat16_t b_imag_vec = svget2_f16(b_vec, 1);
576
+ ab_real_vec = svmla_f16_x(pg_vec, ab_real_vec, a_real_vec, b_real_vec);
577
+ ab_real_vec = svmla_f16_x(pg_vec, ab_real_vec, a_imag_vec, b_imag_vec);
578
+ ab_imag_vec = svmla_f16_x(pg_vec, ab_imag_vec, a_real_vec, b_imag_vec);
579
+ ab_imag_vec = svmls_f16_x(pg_vec, ab_imag_vec, a_imag_vec, b_real_vec);
580
+ i += svcnth() * 2;
581
+ } while (i < n);
582
+ results[0] = svaddv_f16(svptrue_b16(), ab_real_vec);
583
+ results[1] = svaddv_f16(svptrue_b16(), ab_imag_vec);
584
+ }
585
+
586
+ #pragma clang attribute pop
587
+ #pragma GCC pop_options
588
+ #endif // SIMSIMD_TARGET_SVE
589
+ #endif // SIMSIMD_TARGET_ARM
590
+
591
+ #if SIMSIMD_TARGET_X86
592
+ #if SIMSIMD_TARGET_HASWELL
593
+ #pragma GCC push_options
594
+ #pragma GCC target("avx2", "f16c", "fma")
595
+ #pragma clang attribute push(__attribute__((target("avx2,f16c,fma"))), apply_to = function)
596
+
597
+ SIMSIMD_PUBLIC void simsimd_dot_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
598
+ simsimd_distance_t* result) {
599
+ __m256 ab_vec = _mm256_setzero_ps();
600
+ simsimd_size_t i = 0;
601
+ for (; i + 8 <= n; i += 8) {
602
+ __m256 a_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(a + i)));
603
+ __m256 b_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(b + i)));
604
+ ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
605
+ }
606
+
607
+ // In case the software emulation for `f16` scalars is enabled, the `simsimd_uncompress_f16`
608
+ // function will run. It is extremely slow, so even for the tail, let's combine serial
609
+ // loads and stores with vectorized math.
610
+ if (i < n) {
611
+ union {
612
+ __m128i f16_vec;
613
+ simsimd_f16_t f16[8];
614
+ } a_padded_tail, b_padded_tail;
615
+ simsimd_size_t j = 0;
616
+ for (; i < n; ++i, ++j)
617
+ a_padded_tail.f16[j] = a[i], b_padded_tail.f16[j] = b[i];
618
+ for (; j < 8; ++j)
619
+ a_padded_tail.f16[j] = 0, b_padded_tail.f16[j] = 0;
620
+ __m256 a_vec = _mm256_cvtph_ps(a_padded_tail.f16_vec);
621
+ __m256 b_vec = _mm256_cvtph_ps(b_padded_tail.f16_vec);
622
+ ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
623
+ }
624
+
625
+ ab_vec = _mm256_add_ps(_mm256_permute2f128_ps(ab_vec, ab_vec, 1), ab_vec);
626
+ ab_vec = _mm256_hadd_ps(ab_vec, ab_vec);
627
+ ab_vec = _mm256_hadd_ps(ab_vec, ab_vec);
628
+
629
+ simsimd_f32_t f32_result;
630
+ _mm_store_ss(&f32_result, _mm256_castps256_ps128(ab_vec));
631
+ *result = f32_result;
632
+ }
633
+
634
+ inline simsimd_f64_t _mm256_reduce_add_ps_dbl(__m256 vec) {
635
+ // Convert the lower and higher 128-bit lanes of the input vector to double precision
636
+ __m128 low_f32 = _mm256_castps256_ps128(vec);
637
+ __m128 high_f32 = _mm256_extractf128_ps(vec, 1);
638
+
639
+ // Convert single-precision (float) vectors to double-precision (double) vectors
640
+ __m256d low_f64 = _mm256_cvtps_pd(low_f32);
641
+ __m256d high_f64 = _mm256_cvtps_pd(high_f32);
642
+
643
+ // Perform the addition in double-precision
644
+ __m256d sum = _mm256_add_pd(low_f64, high_f64);
645
+
646
+ // Reduce the double-precision vector to a scalar
647
+ // Horizontal add the first and second double-precision values, and third and fourth
648
+ __m128d sum_low = _mm256_castpd256_pd128(sum);
649
+ __m128d sum_high = _mm256_extractf128_pd(sum, 1);
650
+ __m128d sum128 = _mm_add_pd(sum_low, sum_high);
651
+
652
+ // Horizontal add again to accumulate all four values into one
653
+ sum128 = _mm_hadd_pd(sum128, sum128);
654
+
655
+ // Convert the final sum to a scalar double-precision value and return
656
+ return _mm_cvtsd_f64(sum128);
657
+ }
658
+
659
+ SIMSIMD_PUBLIC void simsimd_dot_f32c_haswell(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
660
+ simsimd_distance_t* results) {
661
+
662
+ // The naive approach would be to use FMA and FMS instructions on different parts of the vectors.
663
+ // Prior to that we would need to shuffle the input vectors to separate real and imaginary parts.
664
+ // Both operations are quite expensive, and the resulting kernel would run at 2.5 GB/s.
665
+ // __m128 ab_real_vec = _mm_setzero_ps();
666
+ // __m128 ab_imag_vec = _mm_setzero_ps();
667
+ // __m256i permute_vec = _mm256_set_epi32(7, 5, 3, 1, 6, 4, 2, 0);
668
+ // simsimd_size_t i = 0;
669
+ // for (; i + 8 <= n; i += 8) {
670
+ // __m256 a_vec = _mm256_loadu_ps(a + i);
671
+ // __m256 b_vec = _mm256_loadu_ps(b + i);
672
+ // __m256 a_shuffled = _mm256_permutevar8x32_ps(a_vec, permute_vec);
673
+ // __m256 b_shuffled = _mm256_permutevar8x32_ps(b_vec, permute_vec);
674
+ // __m128 a_real_vec = _mm256_extractf128_ps(a_shuffled, 0);
675
+ // __m128 a_imag_vec = _mm256_extractf128_ps(a_shuffled, 1);
676
+ // __m128 b_real_vec = _mm256_extractf128_ps(b_shuffled, 0);
677
+ // __m128 b_imag_vec = _mm256_extractf128_ps(b_shuffled, 1);
678
+ // ab_real_vec = _mm_fmadd_ps(a_real_vec, b_real_vec, ab_real_vec);
679
+ // ab_real_vec = _mm_fnmadd_ps(a_imag_vec, b_imag_vec, ab_real_vec);
680
+ // ab_imag_vec = _mm_fmadd_ps(a_real_vec, b_imag_vec, ab_imag_vec);
681
+ // ab_imag_vec = _mm_fmadd_ps(a_imag_vec, b_real_vec, ab_imag_vec);
682
+ // }
683
+ //
684
+ // Instead, we take into account, that FMS is the same as FMA with a negative multiplier.
685
+ // To multiply a floating-point value by -1, we can use the `XOR` instruction to flip the sign bit.
686
+ // This way we can avoid the shuffling and the need for separate real and imaginary parts.
687
+ // For the imaginary part of the product, we would need to swap the real and imaginary parts of
688
+ // one of the vectors.
689
+ // Both operations are quite cheap, and the throughput doubles from 2.5 GB/s to 5 GB/s.
690
+ __m256 ab_real_vec = _mm256_setzero_ps();
691
+ __m256 ab_imag_vec = _mm256_setzero_ps();
692
+ __m256i sign_flip_vec = _mm256_set1_epi64x(0x8000000000000000);
693
+ __m256i swap_adjacent_vec = _mm256_set_epi8( //
694
+ 11, 10, 9, 8, // Points to the third f32 in 128-bit lane
695
+ 15, 14, 13, 12, // Points to the fourth f32 in 128-bit lane
696
+ 3, 2, 1, 0, // Points to the first f32 in 128-bit lane
697
+ 7, 6, 5, 4, // Points to the second f32 in 128-bit lane
698
+ 11, 10, 9, 8, // Points to the third f32 in 128-bit lane
699
+ 15, 14, 13, 12, // Points to the fourth f32 in 128-bit lane
700
+ 3, 2, 1, 0, // Points to the first f32 in 128-bit lane
701
+ 7, 6, 5, 4 // Points to the second f32 in 128-bit lane
702
+ );
703
+ simsimd_size_t i = 0;
704
+ for (; i + 8 <= n; i += 8) {
705
+ __m256 a_vec = _mm256_loadu_ps(a + i);
706
+ __m256 b_vec = _mm256_loadu_ps(b + i);
707
+ __m256 b_flipped_vec = _mm256_castsi256_ps(_mm256_xor_si256(_mm256_castps_si256(b_vec), sign_flip_vec));
708
+ __m256 b_swapped_vec = _mm256_castsi256_ps(_mm256_shuffle_epi8(_mm256_castps_si256(b_vec), swap_adjacent_vec));
709
+ ab_real_vec = _mm256_fmadd_ps(a_vec, b_flipped_vec, ab_real_vec);
710
+ ab_imag_vec = _mm256_fmadd_ps(a_vec, b_swapped_vec, ab_imag_vec);
711
+ }
712
+
713
+ // Reduce horizontal sums:
714
+ simsimd_distance_t ab_real = _mm256_reduce_add_ps_dbl(ab_real_vec);
715
+ simsimd_distance_t ab_imag = _mm256_reduce_add_ps_dbl(ab_imag_vec);
716
+
717
+ // Handle the tail:
718
+ for (; i + 2 <= n; i += 2) {
719
+ simsimd_f32_t ar = a[i], ai = a[i + 1], br = b[i], bi = b[i + 1];
720
+ ab_real += ar * br - ai * bi;
721
+ ab_imag += ar * bi + ai * br;
722
+ }
723
+ results[0] = ab_real;
724
+ results[1] = ab_imag;
725
+ }
726
+
727
+ SIMSIMD_PUBLIC void simsimd_vdot_f32c_haswell(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
728
+ simsimd_distance_t* results) {
729
+
730
+ __m256 ab_real_vec = _mm256_setzero_ps();
731
+ __m256 ab_imag_vec = _mm256_setzero_ps();
732
+ __m256i sign_flip_vec = _mm256_set1_epi64x(0x8000000000000000);
733
+ __m256i swap_adjacent_vec = _mm256_set_epi8( //
734
+ 11, 10, 9, 8, // Points to the third f32 in 128-bit lane
735
+ 15, 14, 13, 12, // Points to the fourth f32 in 128-bit lane
736
+ 3, 2, 1, 0, // Points to the first f32 in 128-bit lane
737
+ 7, 6, 5, 4, // Points to the second f32 in 128-bit lane
738
+ 11, 10, 9, 8, // Points to the third f32 in 128-bit lane
739
+ 15, 14, 13, 12, // Points to the fourth f32 in 128-bit lane
740
+ 3, 2, 1, 0, // Points to the first f32 in 128-bit lane
741
+ 7, 6, 5, 4 // Points to the second f32 in 128-bit lane
742
+ );
743
+ simsimd_size_t i = 0;
744
+ for (; i + 8 <= n; i += 8) {
745
+ __m256 a_vec = _mm256_loadu_ps(a + i);
746
+ __m256 b_vec = _mm256_loadu_ps(b + i);
747
+ ab_real_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_real_vec);
748
+ a_vec = _mm256_castsi256_ps(_mm256_xor_si256(_mm256_castps_si256(a_vec), sign_flip_vec));
749
+ b_vec = _mm256_castsi256_ps(_mm256_shuffle_epi8(_mm256_castps_si256(b_vec), swap_adjacent_vec));
750
+ ab_imag_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_imag_vec);
751
+ }
752
+
753
+ // Reduce horizontal sums:
754
+ simsimd_distance_t ab_real = _mm256_reduce_add_ps_dbl(ab_real_vec);
755
+ simsimd_distance_t ab_imag = _mm256_reduce_add_ps_dbl(ab_imag_vec);
756
+
757
+ // Handle the tail:
758
+ for (; i + 2 <= n; i += 2) {
759
+ simsimd_f32_t ar = a[i], ai = a[i + 1], br = b[i], bi = b[i + 1];
760
+ ab_real += ar * br + ai * bi;
761
+ ab_imag += ar * bi - ai * br;
762
+ }
763
+ results[0] = ab_real;
764
+ results[1] = ab_imag;
765
+ }
766
+
767
+ SIMSIMD_PUBLIC void simsimd_dot_f16c_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
768
+ simsimd_distance_t* results) {
769
+ // Ideally the implementation would load 256 bits worth of vector data at a time,
770
+ // shuffle those within a register, split in halfs, and only then upcast.
771
+ // That way, we are stepping through 32x 16-bit vector components at a time, or 16 dimensions.
772
+ // Sadly, shuffling 16-bit entries in a YMM register is hard to implement efficiently.
773
+ //
774
+ // Simpler approach is to load 128 bits at a time, upcast, and then shuffle.
775
+ // This mostly replicates the `simsimd_dot_f32c_haswell`.
776
+ __m256 ab_real_vec = _mm256_setzero_ps();
777
+ __m256 ab_imag_vec = _mm256_setzero_ps();
778
+ __m256i sign_flip_vec = _mm256_set1_epi64x(0x8000000000000000);
779
+ __m256i swap_adjacent_vec = _mm256_set_epi8( //
780
+ 11, 10, 9, 8, // Points to the third f32 in 128-bit lane
781
+ 15, 14, 13, 12, // Points to the fourth f32 in 128-bit lane
782
+ 3, 2, 1, 0, // Points to the first f32 in 128-bit lane
783
+ 7, 6, 5, 4, // Points to the second f32 in 128-bit lane
784
+ 11, 10, 9, 8, // Points to the third f32 in 128-bit lane
785
+ 15, 14, 13, 12, // Points to the fourth f32 in 128-bit lane
786
+ 3, 2, 1, 0, // Points to the first f32 in 128-bit lane
787
+ 7, 6, 5, 4 // Points to the second f32 in 128-bit lane
788
+ );
789
+ simsimd_size_t i = 0;
790
+ for (; i + 8 <= n; i += 8) {
791
+ __m256 a_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(a + i)));
792
+ __m256 b_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(b + i)));
793
+ __m256 b_flipped_vec = _mm256_castsi256_ps(_mm256_xor_si256(_mm256_castps_si256(b_vec), sign_flip_vec));
794
+ __m256 b_swapped_vec = _mm256_castsi256_ps(_mm256_shuffle_epi8(_mm256_castps_si256(b_vec), swap_adjacent_vec));
795
+ ab_real_vec = _mm256_fmadd_ps(a_vec, b_flipped_vec, ab_real_vec);
796
+ ab_imag_vec = _mm256_fmadd_ps(a_vec, b_swapped_vec, ab_imag_vec);
797
+ }
798
+
799
+ // In case the software emulation for `f16` scalars is enabled, the `simsimd_uncompress_f16`
800
+ // function will run. It is extremely slow, so even for the tail, let's combine serial
801
+ // loads and stores with vectorized math.
802
+ if (i < n) {
803
+ union {
804
+ __m128i f16_vec;
805
+ simsimd_f16_t f16[8];
806
+ } a_padded_tail, b_padded_tail;
807
+ simsimd_size_t j = 0;
808
+ for (; i < n; ++i, ++j)
809
+ a_padded_tail.f16[j] = a[i], b_padded_tail.f16[j] = b[i];
810
+ for (; j < 8; ++j)
811
+ a_padded_tail.f16[j] = 0, b_padded_tail.f16[j] = 0;
812
+ __m256 a_vec = _mm256_cvtph_ps(a_padded_tail.f16_vec);
813
+ __m256 b_vec = _mm256_cvtph_ps(b_padded_tail.f16_vec);
814
+ __m256 b_flipped_vec = _mm256_castsi256_ps(_mm256_xor_si256(_mm256_castps_si256(b_vec), sign_flip_vec));
815
+ __m256 b_swapped_vec = _mm256_castsi256_ps(_mm256_shuffle_epi8(_mm256_castps_si256(b_vec), swap_adjacent_vec));
816
+ ab_real_vec = _mm256_fmadd_ps(a_vec, b_flipped_vec, ab_real_vec);
817
+ ab_imag_vec = _mm256_fmadd_ps(a_vec, b_swapped_vec, ab_imag_vec);
818
+ }
819
+
820
+ // Reduce horizontal sums:
821
+ results[0] = _mm256_reduce_add_ps_dbl(ab_real_vec);
822
+ results[1] = _mm256_reduce_add_ps_dbl(ab_imag_vec);
823
+ }
824
+
825
+ SIMSIMD_PUBLIC void simsimd_vdot_f16c_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
826
+ simsimd_distance_t* results) {
827
+
828
+ __m256 ab_real_vec = _mm256_setzero_ps();
829
+ __m256 ab_imag_vec = _mm256_setzero_ps();
830
+ __m256i sign_flip_vec = _mm256_set1_epi64x(0x8000000000000000);
831
+ __m256i swap_adjacent_vec = _mm256_set_epi8( //
832
+ 11, 10, 9, 8, // Points to the third f32 in 128-bit lane
833
+ 15, 14, 13, 12, // Points to the fourth f32 in 128-bit lane
834
+ 3, 2, 1, 0, // Points to the first f32 in 128-bit lane
835
+ 7, 6, 5, 4, // Points to the second f32 in 128-bit lane
836
+ 11, 10, 9, 8, // Points to the third f32 in 128-bit lane
837
+ 15, 14, 13, 12, // Points to the fourth f32 in 128-bit lane
838
+ 3, 2, 1, 0, // Points to the first f32 in 128-bit lane
839
+ 7, 6, 5, 4 // Points to the second f32 in 128-bit lane
840
+ );
841
+ simsimd_size_t i = 0;
842
+ for (; i + 8 <= n; i += 8) {
843
+ __m256 a_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(a + i)));
844
+ __m256 b_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(b + i)));
845
+ ab_real_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_real_vec);
846
+ a_vec = _mm256_castsi256_ps(_mm256_xor_si256(_mm256_castps_si256(a_vec), sign_flip_vec));
847
+ b_vec = _mm256_castsi256_ps(_mm256_shuffle_epi8(_mm256_castps_si256(b_vec), swap_adjacent_vec));
848
+ ab_imag_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_imag_vec);
849
+ }
850
+
851
+ // In case the software emulation for `f16` scalars is enabled, the `simsimd_uncompress_f16`
852
+ // function will run. It is extremely slow, so even for the tail, let's combine serial
853
+ // loads and stores with vectorized math.
854
+ if (i < n) {
855
+ union {
856
+ __m128i f16_vec;
857
+ simsimd_f16_t f16[8];
858
+ } a_padded_tail, b_padded_tail;
859
+ simsimd_size_t j = 0;
860
+ for (; i < n; ++i, ++j)
861
+ a_padded_tail.f16[j] = a[i], b_padded_tail.f16[j] = b[i];
862
+ for (; j < 8; ++j)
863
+ a_padded_tail.f16[j] = 0, b_padded_tail.f16[j] = 0;
864
+ __m256 a_vec = _mm256_cvtph_ps(a_padded_tail.f16_vec);
865
+ __m256 b_vec = _mm256_cvtph_ps(b_padded_tail.f16_vec);
866
+ ab_real_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_real_vec);
867
+ a_vec = _mm256_castsi256_ps(_mm256_xor_si256(_mm256_castps_si256(a_vec), sign_flip_vec));
868
+ b_vec = _mm256_castsi256_ps(_mm256_shuffle_epi8(_mm256_castps_si256(b_vec), swap_adjacent_vec));
869
+ ab_imag_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_imag_vec);
870
+ }
871
+
872
+ // Reduce horizontal sums:
873
+ results[0] = _mm256_reduce_add_ps_dbl(ab_real_vec);
874
+ results[1] = _mm256_reduce_add_ps_dbl(ab_imag_vec);
875
+ }
876
+
877
+ #pragma clang attribute pop
878
+ #pragma GCC pop_options
879
+ #endif // SIMSIMD_TARGET_HASWELL
880
+
881
+ #if SIMSIMD_TARGET_SKYLAKE
882
+ #pragma GCC push_options
883
+ #pragma GCC target("avx512f", "avx512vl", "avx512bw", "bmi2")
884
+ #pragma clang attribute push(__attribute__((target("avx512f,avx512vl,avx512bw,bmi2"))), apply_to = function)
885
+
886
+ SIMSIMD_PUBLIC void simsimd_dot_f32_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
887
+ simsimd_distance_t* result) {
888
+ __m512 ab_vec = _mm512_setzero();
889
+ __m512 a_vec, b_vec;
890
+
891
+ simsimd_dot_f32_skylake_cycle:
892
+ if (n < 16) {
893
+ __mmask16 mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
894
+ a_vec = _mm512_maskz_loadu_ps(mask, a);
895
+ b_vec = _mm512_maskz_loadu_ps(mask, b);
896
+ n = 0;
897
+ } else {
898
+ a_vec = _mm512_loadu_ps(a);
899
+ b_vec = _mm512_loadu_ps(b);
900
+ a += 16, b += 16, n -= 16;
901
+ }
902
+ ab_vec = _mm512_fmadd_ps(a_vec, b_vec, ab_vec);
903
+ if (n)
904
+ goto simsimd_dot_f32_skylake_cycle;
905
+
906
+ *result = _mm512_reduce_add_ps(ab_vec);
907
+ }
908
+
909
+ SIMSIMD_PUBLIC void simsimd_dot_f64_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n,
910
+ simsimd_distance_t* result) {
911
+ __m512d ab_vec = _mm512_setzero_pd();
912
+ __m512d a_vec, b_vec;
913
+
914
+ simsimd_dot_f64_skylake_cycle:
915
+ if (n < 8) {
916
+ __mmask8 mask = (__mmask8)_bzhi_u32(0xFFFFFFFF, n);
917
+ a_vec = _mm512_maskz_loadu_pd(mask, a);
918
+ b_vec = _mm512_maskz_loadu_pd(mask, b);
919
+ n = 0;
920
+ } else {
921
+ a_vec = _mm512_loadu_pd(a);
922
+ b_vec = _mm512_loadu_pd(b);
923
+ a += 8, b += 8, n -= 8;
924
+ }
925
+ ab_vec = _mm512_fmadd_pd(a_vec, b_vec, ab_vec);
926
+ if (n)
927
+ goto simsimd_dot_f64_skylake_cycle;
928
+
929
+ *result = _mm512_reduce_add_pd(ab_vec);
930
+ }
931
+
932
+ SIMSIMD_PUBLIC void simsimd_dot_f32c_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
933
+ simsimd_distance_t* results) {
934
+
935
+ __m512 ab_real_vec = _mm512_setzero();
936
+ __m512 ab_imag_vec = _mm512_setzero();
937
+ __m512 a_vec;
938
+ __m512i b_vec;
939
+
940
+ // We take into account, that FMS is the same as FMA with a negative multiplier.
941
+ // To multiply a floating-point value by -1, we can use the `XOR` instruction to flip the sign bit.
942
+ // This way we can avoid the shuffling and the need for separate real and imaginary parts.
943
+ // For the imaginary part of the product, we would need to swap the real and imaginary parts of
944
+ // one of the vectors.
945
+ __m512i sign_flip_vec = _mm512_set1_epi64(0x8000000000000000);
946
+ __m512i swap_adjacent_vec = _mm512_set_epi8( //
947
+ 59, 58, 57, 56, 63, 62, 61, 60, 51, 50, 49, 48, 55, 54, 53, 52, // 4th 128-bit lane
948
+ 43, 42, 41, 40, 47, 46, 45, 44, 35, 34, 33, 32, 39, 38, 37, 36, // 3rd 128-bit lane
949
+ 27, 26, 25, 24, 31, 30, 29, 28, 19, 18, 17, 16, 23, 22, 21, 20, // 2nd 128-bit lane
950
+ 11, 10, 9, 8, 15, 14, 13, 12, 3, 2, 1, 0, 7, 6, 5, 4 // 1st 128-bit lane
951
+ );
952
+ simsimd_dot_f32c_skylake_cycle:
953
+ if (n < 16) {
954
+ __mmask16 mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
955
+ a_vec = _mm512_maskz_loadu_ps(mask, a);
956
+ b_vec = _mm512_castps_si512(_mm512_maskz_loadu_ps(mask, b));
957
+ n = 0;
958
+ } else {
959
+ a_vec = _mm512_loadu_ps(a);
960
+ b_vec = _mm512_castps_si512(_mm512_loadu_ps(b));
961
+ a += 16, b += 16, n -= 16;
962
+ }
963
+ ab_real_vec = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_xor_si512(b_vec, sign_flip_vec)), a_vec, ab_real_vec);
964
+ ab_imag_vec =
965
+ _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_shuffle_epi8(b_vec, swap_adjacent_vec)), a_vec, ab_imag_vec);
966
+ if (n)
967
+ goto simsimd_dot_f32c_skylake_cycle;
968
+
969
+ // Reduce horizontal sums:
970
+ results[0] = _mm512_reduce_add_ps(ab_real_vec);
971
+ results[1] = _mm512_reduce_add_ps(ab_imag_vec);
972
+ }
973
+
974
+ SIMSIMD_PUBLIC void simsimd_vdot_f32c_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
975
+ simsimd_distance_t* results) {
976
+
977
+ __m512 ab_real_vec = _mm512_setzero();
978
+ __m512 ab_imag_vec = _mm512_setzero();
979
+ __m512 a_vec;
980
+ __m512 b_vec;
981
+
982
+ // We take into account, that FMS is the same as FMA with a negative multiplier.
983
+ // To multiply a floating-point value by -1, we can use the `XOR` instruction to flip the sign bit.
984
+ // This way we can avoid the shuffling and the need for separate real and imaginary parts.
985
+ // For the imaginary part of the product, we would need to swap the real and imaginary parts of
986
+ // one of the vectors.
987
+ __m512i sign_flip_vec = _mm512_set1_epi64(0x8000000000000000);
988
+ __m512i swap_adjacent_vec = _mm512_set_epi8( //
989
+ 59, 58, 57, 56, 63, 62, 61, 60, 51, 50, 49, 48, 55, 54, 53, 52, // 4th 128-bit lane
990
+ 43, 42, 41, 40, 47, 46, 45, 44, 35, 34, 33, 32, 39, 38, 37, 36, // 3rd 128-bit lane
991
+ 27, 26, 25, 24, 31, 30, 29, 28, 19, 18, 17, 16, 23, 22, 21, 20, // 2nd 128-bit lane
992
+ 11, 10, 9, 8, 15, 14, 13, 12, 3, 2, 1, 0, 7, 6, 5, 4 // 1st 128-bit lane
993
+ );
994
+ simsimd_vdot_f32c_skylake_cycle:
995
+ if (n < 16) {
996
+ __mmask16 mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
997
+ a_vec = _mm512_maskz_loadu_ps(mask, a);
998
+ b_vec = _mm512_maskz_loadu_ps(mask, b);
999
+ n = 0;
1000
+ } else {
1001
+ a_vec = _mm512_loadu_ps(a);
1002
+ b_vec = _mm512_loadu_ps(b);
1003
+ a += 16, b += 16, n -= 16;
1004
+ }
1005
+ ab_real_vec = _mm512_fmadd_ps(a_vec, b_vec, ab_real_vec);
1006
+ a_vec = _mm512_castsi512_ps(_mm512_xor_si512(_mm512_castps_si512(a_vec), sign_flip_vec));
1007
+ b_vec = _mm512_castsi512_ps(_mm512_shuffle_epi8(_mm512_castps_si512(b_vec), swap_adjacent_vec));
1008
+ ab_imag_vec = _mm512_fmadd_ps(a_vec, b_vec, ab_imag_vec);
1009
+ if (n)
1010
+ goto simsimd_vdot_f32c_skylake_cycle;
1011
+
1012
+ // Reduce horizontal sums:
1013
+ results[0] = _mm512_reduce_add_ps(ab_real_vec);
1014
+ results[1] = _mm512_reduce_add_ps(ab_imag_vec);
1015
+ }
1016
+
1017
+ SIMSIMD_PUBLIC void simsimd_dot_f64c_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n,
1018
+ simsimd_distance_t* results) {
1019
+
1020
+ __m512d ab_real_vec = _mm512_setzero_pd();
1021
+ __m512d ab_imag_vec = _mm512_setzero_pd();
1022
+ __m512d a_vec;
1023
+ __m512i b_vec;
1024
+
1025
+ // We take into account, that FMS is the same as FMA with a negative multiplier.
1026
+ // To multiply a floating-point value by -1, we can use the `XOR` instruction to flip the sign bit.
1027
+ // This way we can avoid the shuffling and the need for separate real and imaginary parts.
1028
+ // For the imaginary part of the product, we would need to swap the real and imaginary parts of
1029
+ // one of the vectors.
1030
+ __m512i sign_flip_vec = _mm512_set_epi64( //
1031
+ 0x8000000000000000, 0x0000000000000000, 0x8000000000000000, 0x0000000000000000, //
1032
+ 0x8000000000000000, 0x0000000000000000, 0x8000000000000000, 0x0000000000000000 //
1033
+ );
1034
+ __m512i swap_adjacent_vec = _mm512_set_epi8( //
1035
+ 55, 54, 53, 52, 51, 50, 49, 48, 63, 62, 61, 60, 59, 58, 57, 56, // 4th 128-bit lane
1036
+ 39, 38, 37, 36, 35, 34, 33, 32, 47, 46, 45, 44, 43, 42, 41, 40, // 3rd 128-bit lane
1037
+ 23, 22, 21, 20, 19, 18, 17, 16, 31, 30, 29, 28, 27, 26, 25, 24, // 2nd 128-bit lane
1038
+ 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8 // 1st 128-bit lane
1039
+ );
1040
+ simsimd_dot_f64c_skylake_cycle:
1041
+ if (n < 8) {
1042
+ __mmask8 mask = (__mmask8)_bzhi_u32(0xFFFFFFFF, n);
1043
+ a_vec = _mm512_maskz_loadu_pd(mask, a);
1044
+ b_vec = _mm512_castpd_si512(_mm512_maskz_loadu_pd(mask, b));
1045
+ n = 0;
1046
+ } else {
1047
+ a_vec = _mm512_loadu_pd(a);
1048
+ b_vec = _mm512_castpd_si512(_mm512_loadu_pd(b));
1049
+ a += 8, b += 8, n -= 8;
1050
+ }
1051
+ ab_real_vec = _mm512_fmadd_pd(_mm512_castsi512_pd(_mm512_xor_si512(b_vec, sign_flip_vec)), a_vec, ab_real_vec);
1052
+ ab_imag_vec =
1053
+ _mm512_fmadd_pd(_mm512_castsi512_pd(_mm512_shuffle_epi8(b_vec, swap_adjacent_vec)), a_vec, ab_imag_vec);
1054
+ if (n)
1055
+ goto simsimd_dot_f64c_skylake_cycle;
1056
+
1057
+ // Reduce horizontal sums:
1058
+ results[0] = _mm512_reduce_add_pd(ab_real_vec);
1059
+ results[1] = _mm512_reduce_add_pd(ab_imag_vec);
1060
+ }
1061
+
1062
+ SIMSIMD_PUBLIC void simsimd_vdot_f64c_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n,
1063
+ simsimd_distance_t* results) {
1064
+
1065
+ __m512d ab_real_vec = _mm512_setzero_pd();
1066
+ __m512d ab_imag_vec = _mm512_setzero_pd();
1067
+ __m512d a_vec;
1068
+ __m512d b_vec;
1069
+
1070
+ // We take into account, that FMS is the same as FMA with a negative multiplier.
1071
+ // To multiply a floating-point value by -1, we can use the `XOR` instruction to flip the sign bit.
1072
+ // This way we can avoid the shuffling and the need for separate real and imaginary parts.
1073
+ // For the imaginary part of the product, we would need to swap the real and imaginary parts of
1074
+ // one of the vectors.
1075
+ __m512i sign_flip_vec = _mm512_set_epi64( //
1076
+ 0x8000000000000000, 0x0000000000000000, 0x8000000000000000, 0x0000000000000000, //
1077
+ 0x8000000000000000, 0x0000000000000000, 0x8000000000000000, 0x0000000000000000 //
1078
+ );
1079
+ __m512i swap_adjacent_vec = _mm512_set_epi8( //
1080
+ 55, 54, 53, 52, 51, 50, 49, 48, 63, 62, 61, 60, 59, 58, 57, 56, // 4th 128-bit lane
1081
+ 39, 38, 37, 36, 35, 34, 33, 32, 47, 46, 45, 44, 43, 42, 41, 40, // 3rd 128-bit lane
1082
+ 23, 22, 21, 20, 19, 18, 17, 16, 31, 30, 29, 28, 27, 26, 25, 24, // 2nd 128-bit lane
1083
+ 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8 // 1st 128-bit lane
1084
+ );
1085
+ simsimd_vdot_f64c_skylake_cycle:
1086
+ if (n < 8) {
1087
+ __mmask8 mask = (__mmask8)_bzhi_u32(0xFFFFFFFF, n);
1088
+ a_vec = _mm512_maskz_loadu_pd(mask, a);
1089
+ b_vec = _mm512_maskz_loadu_pd(mask, b);
1090
+ n = 0;
1091
+ } else {
1092
+ a_vec = _mm512_loadu_pd(a);
1093
+ b_vec = _mm512_loadu_pd(b);
1094
+ a += 8, b += 8, n -= 8;
1095
+ }
1096
+ ab_real_vec = _mm512_fmadd_pd(a_vec, b_vec, ab_real_vec);
1097
+ a_vec = _mm512_castsi512_pd(_mm512_xor_si512(_mm512_castpd_si512(a_vec), sign_flip_vec));
1098
+ b_vec = _mm512_castsi512_pd(_mm512_shuffle_epi8(_mm512_castpd_si512(b_vec), swap_adjacent_vec));
1099
+ ab_imag_vec = _mm512_fmadd_pd(a_vec, b_vec, ab_imag_vec);
1100
+ if (n)
1101
+ goto simsimd_vdot_f64c_skylake_cycle;
1102
+
1103
+ // Reduce horizontal sums:
1104
+ results[0] = _mm512_reduce_add_pd(ab_real_vec);
1105
+ results[1] = _mm512_reduce_add_pd(ab_imag_vec);
1106
+ }
1107
+
1108
+ #pragma clang attribute pop
1109
+ #pragma GCC pop_options
1110
+ #endif // SIMSIMD_TARGET_SKYLAKE
1111
+
1112
+ #if SIMSIMD_TARGET_SAPPHIRE
1113
+ #pragma GCC push_options
1114
+ #pragma GCC target("avx512f", "avx512vl", "bmi2", "avx512bw", "avx512fp16")
1115
+ #pragma clang attribute push(__attribute__((target("avx512f,avx512vl,bmi2,avx512bw,avx512fp16"))), apply_to = function)
1116
+
1117
+ SIMSIMD_PUBLIC void simsimd_dot_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
1118
+ simsimd_distance_t* result) {
1119
+ __m512h ab_vec = _mm512_setzero_ph();
1120
+ __m512i a_i16_vec, b_i16_vec;
1121
+
1122
+ simsimd_dot_f16_sapphire_cycle:
1123
+ if (n < 32) {
1124
+ __mmask32 mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
1125
+ a_i16_vec = _mm512_maskz_loadu_epi16(mask, a);
1126
+ b_i16_vec = _mm512_maskz_loadu_epi16(mask, b);
1127
+ n = 0;
1128
+ } else {
1129
+ a_i16_vec = _mm512_loadu_epi16(a);
1130
+ b_i16_vec = _mm512_loadu_epi16(b);
1131
+ a += 32, b += 32, n -= 32;
1132
+ }
1133
+ ab_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(a_i16_vec), _mm512_castsi512_ph(b_i16_vec), ab_vec);
1134
+ if (n)
1135
+ goto simsimd_dot_f16_sapphire_cycle;
1136
+
1137
+ *result = _mm512_reduce_add_ph(ab_vec);
1138
+ }
1139
+
1140
+ SIMSIMD_PUBLIC void simsimd_dot_f16c_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
1141
+ simsimd_distance_t* results) {
1142
+
1143
+ __m512h ab_real_vec = _mm512_setzero_ph();
1144
+ __m512h ab_imag_vec = _mm512_setzero_ph();
1145
+ __m512i a_vec;
1146
+ __m512i b_vec;
1147
+
1148
+ // We take into account, that FMS is the same as FMA with a negative multiplier.
1149
+ // To multiply a floating-point value by -1, we can use the `XOR` instruction to flip the sign bit.
1150
+ // This way we can avoid the shuffling and the need for separate real and imaginary parts.
1151
+ // For the imaginary part of the product, we would need to swap the real and imaginary parts of
1152
+ // one of the vectors.
1153
+ __m512i sign_flip_vec = _mm512_set1_epi32(0x80000000);
1154
+ __m512i swap_adjacent_vec = _mm512_set_epi8( //
1155
+ 61, 60, 63, 62, 57, 56, 59, 58, 53, 52, 55, 54, 49, 48, 51, 50, // 4th 128-bit lane
1156
+ 45, 44, 47, 46, 41, 40, 43, 42, 37, 36, 39, 38, 33, 32, 35, 34, // 3rd 128-bit lane
1157
+ 29, 28, 31, 30, 25, 24, 27, 26, 21, 20, 23, 22, 17, 16, 19, 18, // 2nd 128-bit lane
1158
+ 13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2 // 1st 128-bit lane
1159
+ );
1160
+
1161
+ simsimd_dot_f16c_sapphire_cycle:
1162
+ if (n < 32) {
1163
+ __mmask32 mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
1164
+ a_vec = _mm512_maskz_loadu_epi16(mask, a);
1165
+ b_vec = _mm512_maskz_loadu_epi16(mask, b);
1166
+ n = 0;
1167
+ } else {
1168
+ a_vec = _mm512_loadu_epi16(a);
1169
+ b_vec = _mm512_loadu_epi16(b);
1170
+ a += 32, b += 32, n -= 32;
1171
+ }
1172
+ ab_real_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(_mm512_xor_si512(b_vec, sign_flip_vec)),
1173
+ _mm512_castsi512_ph(a_vec), ab_real_vec);
1174
+ ab_imag_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(_mm512_shuffle_epi8(b_vec, swap_adjacent_vec)),
1175
+ _mm512_castsi512_ph(a_vec), ab_imag_vec);
1176
+ if (n)
1177
+ goto simsimd_dot_f16c_sapphire_cycle;
1178
+
1179
+ // Reduce horizontal sums:
1180
+ results[0] = _mm512_reduce_add_ph(ab_real_vec);
1181
+ results[1] = _mm512_reduce_add_ph(ab_imag_vec);
1182
+ }
1183
+
1184
+ SIMSIMD_PUBLIC void simsimd_vdot_f16c_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
1185
+ simsimd_distance_t* results) {
1186
+
1187
+ __m512h ab_real_vec = _mm512_setzero_ph();
1188
+ __m512h ab_imag_vec = _mm512_setzero_ph();
1189
+ __m512i a_vec;
1190
+ __m512i b_vec;
1191
+
1192
+ // We take into account, that FMS is the same as FMA with a negative multiplier.
1193
+ // To multiply a floating-point value by -1, we can use the `XOR` instruction to flip the sign bit.
1194
+ // This way we can avoid the shuffling and the need for separate real and imaginary parts.
1195
+ // For the imaginary part of the product, we would need to swap the real and imaginary parts of
1196
+ // one of the vectors.
1197
+ __m512i sign_flip_vec = _mm512_set1_epi32(0x80000000);
1198
+ __m512i swap_adjacent_vec = _mm512_set_epi8( //
1199
+ 61, 60, 63, 62, 57, 56, 59, 58, 53, 52, 55, 54, 49, 48, 51, 50, // 4th 128-bit lane
1200
+ 45, 44, 47, 46, 41, 40, 43, 42, 37, 36, 39, 38, 33, 32, 35, 34, // 3rd 128-bit lane
1201
+ 29, 28, 31, 30, 25, 24, 27, 26, 21, 20, 23, 22, 17, 16, 19, 18, // 2nd 128-bit lane
1202
+ 13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2 // 1st 128-bit lane
1203
+ );
1204
+
1205
+ simsimd_dot_f16c_sapphire_cycle:
1206
+ if (n < 32) {
1207
+ __mmask32 mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
1208
+ a_vec = _mm512_maskz_loadu_epi16(mask, a);
1209
+ b_vec = _mm512_maskz_loadu_epi16(mask, b);
1210
+ n = 0;
1211
+ } else {
1212
+ a_vec = _mm512_loadu_epi16(a);
1213
+ b_vec = _mm512_loadu_epi16(b);
1214
+ a += 32, b += 32, n -= 32;
1215
+ }
1216
+ ab_real_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(a_vec), _mm512_castsi512_ph(b_vec), ab_real_vec);
1217
+ a_vec = _mm512_xor_si512(a_vec, sign_flip_vec);
1218
+ b_vec = _mm512_shuffle_epi8(b_vec, swap_adjacent_vec);
1219
+ ab_imag_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(a_vec), _mm512_castsi512_ph(b_vec), ab_imag_vec);
1220
+ if (n)
1221
+ goto simsimd_dot_f16c_sapphire_cycle;
1222
+
1223
+ // Reduce horizontal sums:
1224
+ results[0] = _mm512_reduce_add_ph(ab_real_vec);
1225
+ results[1] = _mm512_reduce_add_ph(ab_imag_vec);
1226
+ }
1227
+
1228
+ #pragma clang attribute pop
1229
+ #pragma GCC pop_options
1230
+ #endif // SIMSIMD_TARGET_SAPPHIRE
1231
+ #endif // SIMSIMD_TARGET_X86
1232
+
1233
+ #ifdef __cplusplus
1234
+ }
1235
+ #endif
1236
+
1237
+ #endif