usearch 2.21.4 → 2.25.0

package/numkong/include/numkong/spatials/smef64.h

@@ -0,0 +1,483 @@
+/**
+ *  @brief Batched Spatial Distances for ARM SME-F64.
+ *  @file include/numkong/spatials/smef64.h
+ *  @author Ash Vardanian
+ *  @date February 23, 2026
+ *
+ *  @sa include/numkong/spatials.h
+ */
+#ifndef NK_SPATIALS_SMEF64_H
+#define NK_SPATIALS_SMEF64_H
+
+#if NK_TARGET_ARM64_
+#if NK_TARGET_SME
+
+#include "numkong/dots/serial.h"
+#include "numkong/reduce/sve.h" // `nk_svaddv_f64_`
+#include "numkong/dots/smef64.h"
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((target("sme,sme-f64f64"))), apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC target("+sme+sme-f64f64")
+#endif
+
+NK_PUBLIC nk_f64_t nk_dots_reduce_sumsq_f32_ssve_(nk_f32_t const *data, nk_size_t count) NK_STREAMING_ {
+    svfloat64_t accumulator_even_f64x = svdup_f64(0.0);
+    svfloat64_t accumulator_odd_f64x = svdup_f64(0.0);
+    nk_size_t const vector_length = svcntw();
+    nk_size_t const half_vector_length = svcntd();
+    for (nk_size_t i = 0; i < count; i += vector_length) {
+        svbool_t predicate_b32x = svwhilelt_b32_u64(i, count);
+        svfloat32_t values_f32x = svld1_f32(predicate_b32x, data + i);
+
+        svbool_t predicate_even_b64x = svwhilelt_b64_u64(i, count);
+        svfloat64_t values_even_f64x = svcvt_f64_f32_x(predicate_even_b64x, values_f32x);
+        accumulator_even_f64x = svmla_f64_m(predicate_even_b64x, accumulator_even_f64x, values_even_f64x,
+                                            values_even_f64x);
+
+        svbool_t predicate_odd_b64x = svwhilelt_b64_u64(i + half_vector_length, count);
+        svfloat64_t values_odd_f64x = svcvtlt_f64_f32_x(predicate_odd_b64x, values_f32x);
+        accumulator_odd_f64x = svmla_f64_m(predicate_odd_b64x, accumulator_odd_f64x, values_odd_f64x, values_odd_f64x);
+    }
+    return nk_svaddv_f64_(svptrue_b64(), accumulator_even_f64x) + nk_svaddv_f64_(svptrue_b64(), accumulator_odd_f64x);
+}
+
+NK_PUBLIC nk_f64_t nk_dots_reduce_sumsq_f64_ssve_(nk_f64_t const *data, nk_size_t count) NK_STREAMING_ {
+    svfloat64_t accumulator_f64x = svdup_f64(0.0);
+    nk_size_t const vector_length = svcntd();
+    for (nk_size_t i = 0; i < count; i += vector_length) {
+        svbool_t predicate_b64x = svwhilelt_b64_u64(i, count);
+        svfloat64_t values_f64x = svld1_f64(predicate_b64x, data + i);
+        accumulator_f64x = svmla_f64_m(predicate_b64x, accumulator_f64x, values_f64x, values_f64x);
+    }
+    return nk_svaddv_f64_(svptrue_b64(), accumulator_f64x);
+}
+
+NK_PUBLIC svfloat64_t nk_angulars_from_dot_f64x_ssvef64_(svbool_t predicate_b64x, svfloat64_t dots_f64x,
+                                                         svfloat64_t query_norm_sq_f64x,
+                                                         svfloat64_t target_norms_sq_f64x) NK_STREAMING_ {
+    svfloat64_t norms_product_f64x = svmul_f64_x(predicate_b64x, query_norm_sq_f64x, target_norms_sq_f64x);
+    svbool_t positive_norms_b64x = svcmpgt_n_f64(predicate_b64x, norms_product_f64x, 0.0);
+    svfloat64_t denom_f64x = svsqrt_f64_x(positive_norms_b64x, norms_product_f64x);
+    svfloat64_t safe_denom_f64x = svsel_f64(positive_norms_b64x, denom_f64x, svdup_n_f64(1.0));
+    svfloat64_t normalized_f64x = svdiv_f64_x(predicate_b64x, dots_f64x, safe_denom_f64x);
+    svfloat64_t angular_f64x = svsub_f64_x(predicate_b64x, svdup_n_f64(1.0), normalized_f64x);
+    angular_f64x = svsel_f64(
+        positive_norms_b64x, angular_f64x,
+        svsel_f64(svcmpeq_n_f64(predicate_b64x, dots_f64x, 0.0), svdup_n_f64(0.0), svdup_n_f64(1.0)));
+    return svmax_f64_x(predicate_b64x, angular_f64x, svdup_n_f64(0.0));
+}
+
+NK_PUBLIC svfloat64_t nk_euclideans_from_dot_f64x_ssvef64_(svbool_t predicate_b64x, svfloat64_t dots_f64x,
+                                                           svfloat64_t query_norm_sq_f64x,
+                                                           svfloat64_t target_norms_sq_f64x) NK_STREAMING_ {
+    svfloat64_t sum_sq_f64x = svadd_f64_x(predicate_b64x, query_norm_sq_f64x, target_norms_sq_f64x);
+    svfloat64_t dist_sq_f64x = svsub_f64_x(predicate_b64x, sum_sq_f64x,
+                                           svmul_f64_x(predicate_b64x, svdup_n_f64(2.0), dots_f64x));
+    dist_sq_f64x = svmax_f64_x(predicate_b64x, dist_sq_f64x, svdup_n_f64(0.0));
+    return svsqrt_f64_x(predicate_b64x, dist_sq_f64x);
+}
+
+#pragma region F32 Packed Angular
+
+static void nk_angulars_packed_f32_smef64_finalize_ssve_( //
+    nk_f32_t const *a, void const *b_packed, nk_f64_t *c, nk_size_t rows, nk_size_t columns, nk_size_t depth,
+    nk_size_t a_stride_elements, nk_size_t c_stride_elements) NK_STREAMING_ {
+
+    nk_dots_sme_packed_header_t const *header = (nk_dots_sme_packed_header_t const *)b_packed;
+    nk_f64_t const *b_norms = (nk_f64_t const *)((char const *)b_packed + header->norms_offset);
+
+    for (nk_size_t row_index = 0; row_index < rows; row_index++) {
+        nk_f32_t const *a_row = a + row_index * a_stride_elements;
+        nk_f64_t *c_row = c + row_index * c_stride_elements;
+        nk_f64_t query_norm_sq_f64 = nk_dots_reduce_sumsq_f32_ssve_(a_row, depth);
+        svfloat64_t query_norm_sq_f64x = svdup_n_f64(query_norm_sq_f64);
+
+        for (nk_size_t col_index = 0; col_index < columns; col_index += svcntd()) {
+            svbool_t predicate_b64x = svwhilelt_b64_u64(col_index, columns);
+            svfloat64_t dots_f64x = svld1_f64(predicate_b64x, c_row + col_index);
+            svfloat64_t target_norms_sq_f64x = svld1_f64(predicate_b64x, b_norms + col_index);
+            svst1_f64(predicate_b64x, c_row + col_index,
+                      nk_angulars_from_dot_f64x_ssvef64_(predicate_b64x, dots_f64x, query_norm_sq_f64x,
+                                                         target_norms_sq_f64x));
+        }
+    }
+}
+
+NK_PUBLIC void nk_angulars_packed_f32_smef64( //
+    nk_f32_t const *a, void const *b_packed, nk_f64_t *c, nk_size_t rows, nk_size_t columns, nk_size_t depth,
+    nk_size_t a_stride_in_bytes, nk_size_t c_stride_in_bytes) {
+
+    nk_size_t const a_stride_elements = a_stride_in_bytes / sizeof(nk_f32_t);
+    nk_size_t const c_stride_elements = c_stride_in_bytes / sizeof(nk_f64_t);
+
+    nk_sme_start_streaming_();
+    nk_dots_packed_f32_smef64_streaming_(a, b_packed, c, rows, columns, depth, a_stride_elements, c_stride_elements);
+    nk_angulars_packed_f32_smef64_finalize_ssve_(a, b_packed, c, rows, columns, depth, a_stride_elements,
+                                                 c_stride_elements);
+    nk_sme_stop_streaming_();
+}
+
+#pragma endregion F32 Packed Angular
+#pragma region F32 Packed Euclidean
+
+static void nk_euclideans_packed_f32_smef64_finalize_ssve_( //
+    nk_f32_t const *a, void const *b_packed, nk_f64_t *c, nk_size_t rows, nk_size_t columns, nk_size_t depth,
+    nk_size_t a_stride_elements, nk_size_t c_stride_elements) NK_STREAMING_ {
+
+    nk_dots_sme_packed_header_t const *header = (nk_dots_sme_packed_header_t const *)b_packed;
+    nk_f64_t const *b_norms = (nk_f64_t const *)((char const *)b_packed + header->norms_offset);
+
+    for (nk_size_t row_index = 0; row_index < rows; row_index++) {
+        nk_f32_t const *a_row = a + row_index * a_stride_elements;
+        nk_f64_t *c_row = c + row_index * c_stride_elements;
+        nk_f64_t query_norm_sq_f64 = nk_dots_reduce_sumsq_f32_ssve_(a_row, depth);
+        svfloat64_t query_norm_sq_f64x = svdup_n_f64(query_norm_sq_f64);
+
+        for (nk_size_t col_index = 0; col_index < columns; col_index += svcntd()) {
+            svbool_t predicate_b64x = svwhilelt_b64_u64(col_index, columns);
+            svfloat64_t dots_f64x = svld1_f64(predicate_b64x, c_row + col_index);
+            svfloat64_t target_norms_sq_f64x = svld1_f64(predicate_b64x, b_norms + col_index);
+            svst1_f64(predicate_b64x, c_row + col_index,
+                      nk_euclideans_from_dot_f64x_ssvef64_(predicate_b64x, dots_f64x, query_norm_sq_f64x,
+                                                           target_norms_sq_f64x));
+        }
+    }
+}
+
+NK_PUBLIC void nk_euclideans_packed_f32_smef64( //
+    nk_f32_t const *a, void const *b_packed, nk_f64_t *c, nk_size_t rows, nk_size_t columns, nk_size_t depth,
+    nk_size_t a_stride_in_bytes, nk_size_t c_stride_in_bytes) {
+
+    nk_size_t const a_stride_elements = a_stride_in_bytes / sizeof(nk_f32_t);
+    nk_size_t const c_stride_elements = c_stride_in_bytes / sizeof(nk_f64_t);
+
+    nk_sme_start_streaming_();
+    nk_dots_packed_f32_smef64_streaming_(a, b_packed, c, rows, columns, depth, a_stride_elements, c_stride_elements);
+    nk_euclideans_packed_f32_smef64_finalize_ssve_(a, b_packed, c, rows, columns, depth, a_stride_elements,
+                                                   c_stride_elements);
+    nk_sme_stop_streaming_();
+}
+
+#pragma endregion F32 Packed Euclidean
+#pragma region F32 Symmetric Angular
+
+static void nk_angulars_symmetric_f32_smef64_finalize_ssve_( //
+    nk_f32_t const *vectors, nk_size_t vectors_count, nk_size_t depth, nk_size_t stride_elements, nk_f64_t *result,
+    nk_size_t result_stride_elements, nk_size_t row_start, nk_size_t row_count) NK_STREAMING_ {
+    // Phase 1: cache row norms on diagonal
+    for (nk_size_t row_index = row_start; row_index < row_start + row_count; ++row_index) {
+        nk_f32_t const *row_vector = vectors + row_index * stride_elements;
+        nk_f64_t *result_row = result + row_index * result_stride_elements;
+        result_row[row_index] = nk_dots_reduce_sumsq_f32_ssve_(row_vector, depth);
+    }
+    // Phase 2: column-chunked post-processing
+    nk_f64_t column_norms[256];
+    for (nk_size_t chunk_start = 0; chunk_start < vectors_count; chunk_start += 256) {
+        nk_size_t chunk_end = chunk_start + 256 < vectors_count ? chunk_start + 256 : vectors_count;
+        for (nk_size_t col = chunk_start; col < chunk_end; ++col) {
+            nk_f32_t const *col_vector = vectors + col * stride_elements;
+            column_norms[col - chunk_start] = nk_dots_reduce_sumsq_f32_ssve_(col_vector, depth);
+        }
+        for (nk_size_t row_index = row_start; row_index < row_start + row_count; ++row_index) {
+            nk_size_t col_start = row_index + 1 > chunk_start ? row_index + 1 : chunk_start;
+            if (col_start >= chunk_end) continue;
+            nk_f64_t *result_row = result + row_index * result_stride_elements;
+            svfloat64_t query_norm_sq_f64x = svdup_n_f64(result_row[row_index]);
+            for (nk_size_t col_index = col_start; col_index < chunk_end; col_index += svcntd()) {
+                svbool_t predicate_b64x = svwhilelt_b64_u64(col_index, chunk_end);
+                svfloat64_t dots_f64x = svld1_f64(predicate_b64x, result_row + col_index);
+                svfloat64_t target_norms_sq_f64x = svld1_f64(predicate_b64x, column_norms + (col_index - chunk_start));
+                svst1_f64(predicate_b64x, result_row + col_index,
+                          nk_angulars_from_dot_f64x_ssvef64_(predicate_b64x, dots_f64x, query_norm_sq_f64x,
+                                                             target_norms_sq_f64x));
+            }
+        }
+    }
+    // Phase 3: zero diagonals
+    for (nk_size_t row_index = row_start; row_index < row_start + row_count; ++row_index)
+        result[row_index * result_stride_elements + row_index] = 0;
+}
+
+NK_PUBLIC void nk_angulars_symmetric_f32_smef64( //
+    nk_f32_t const *vectors, nk_size_t vectors_count, nk_size_t depth, nk_size_t stride_in_bytes, nk_f64_t *result,
+    nk_size_t result_stride_in_bytes, nk_size_t row_start, nk_size_t row_count) {
+
+    nk_size_t const stride_elements = stride_in_bytes / sizeof(nk_f32_t);
+    nk_size_t const result_stride_elements = result_stride_in_bytes / sizeof(nk_f64_t);
+
+    nk_sme_start_streaming_();
+    nk_dots_symmetric_f32_smef64_streaming_(vectors, vectors_count, depth, stride_elements, result,
+                                            result_stride_elements, row_start, row_count);
+    nk_angulars_symmetric_f32_smef64_finalize_ssve_(vectors, vectors_count, depth, stride_elements, result,
+                                                    result_stride_elements, row_start, row_count);
+    nk_sme_stop_streaming_();
+}
+
+#pragma endregion F32 Symmetric Angular
+#pragma region F32 Symmetric Euclidean
+
+static void nk_euclideans_symmetric_f32_smef64_finalize_ssve_( //
+    nk_f32_t const *vectors, nk_size_t vectors_count, nk_size_t depth, nk_size_t stride_elements, nk_f64_t *result,
+    nk_size_t result_stride_elements, nk_size_t row_start, nk_size_t row_count) NK_STREAMING_ {
+    // Phase 1: cache row norms on diagonal
+    for (nk_size_t row_index = row_start; row_index < row_start + row_count; ++row_index) {
+        nk_f32_t const *row_vector = vectors + row_index * stride_elements;
+        nk_f64_t *result_row = result + row_index * result_stride_elements;
+        result_row[row_index] = nk_dots_reduce_sumsq_f32_ssve_(row_vector, depth);
+    }
+    // Phase 2: column-chunked post-processing
+    nk_f64_t column_norms[256];
+    for (nk_size_t chunk_start = 0; chunk_start < vectors_count; chunk_start += 256) {
+        nk_size_t chunk_end = chunk_start + 256 < vectors_count ? chunk_start + 256 : vectors_count;
+        for (nk_size_t col = chunk_start; col < chunk_end; ++col) {
+            nk_f32_t const *col_vector = vectors + col * stride_elements;
+            column_norms[col - chunk_start] = nk_dots_reduce_sumsq_f32_ssve_(col_vector, depth);
+        }
+        for (nk_size_t row_index = row_start; row_index < row_start + row_count; ++row_index) {
+            nk_size_t col_start = row_index + 1 > chunk_start ? row_index + 1 : chunk_start;
+            if (col_start >= chunk_end) continue;
+            nk_f64_t *result_row = result + row_index * result_stride_elements;
+            svfloat64_t query_norm_sq_f64x = svdup_n_f64(result_row[row_index]);
+            for (nk_size_t col_index = col_start; col_index < chunk_end; col_index += svcntd()) {
+                svbool_t predicate_b64x = svwhilelt_b64_u64(col_index, chunk_end);
+                svfloat64_t dots_f64x = svld1_f64(predicate_b64x, result_row + col_index);
+                svfloat64_t target_norms_sq_f64x = svld1_f64(predicate_b64x, column_norms + (col_index - chunk_start));
+                svst1_f64(predicate_b64x, result_row + col_index,
+                          nk_euclideans_from_dot_f64x_ssvef64_(predicate_b64x, dots_f64x, query_norm_sq_f64x,
+                                                               target_norms_sq_f64x));
+            }
+        }
+    }
+    // Phase 3: zero diagonals
+    for (nk_size_t row_index = row_start; row_index < row_start + row_count; ++row_index)
+        result[row_index * result_stride_elements + row_index] = 0;
+}
+
+NK_PUBLIC void nk_euclideans_symmetric_f32_smef64( //
+    nk_f32_t const *vectors, nk_size_t vectors_count, nk_size_t depth, nk_size_t stride_in_bytes, nk_f64_t *result,
+    nk_size_t result_stride_in_bytes, nk_size_t row_start, nk_size_t row_count) {
+
+    nk_size_t const stride_elements = stride_in_bytes / sizeof(nk_f32_t);
+    nk_size_t const result_stride_elements = result_stride_in_bytes / sizeof(nk_f64_t);
+
+    nk_sme_start_streaming_();
+    nk_dots_symmetric_f32_smef64_streaming_(vectors, vectors_count, depth, stride_elements, result,
+                                            result_stride_elements, row_start, row_count);
+    nk_euclideans_symmetric_f32_smef64_finalize_ssve_(vectors, vectors_count, depth, stride_elements, result,
+                                                      result_stride_elements, row_start, row_count);
+    nk_sme_stop_streaming_();
+}
+
+#pragma endregion F32 Symmetric Euclidean
+#pragma region F64 Packed Angular
+
+static void nk_angulars_packed_f64_smef64_finalize_ssve_( //
+    nk_f64_t const *a, void const *b_packed, nk_f64_t *c, nk_size_t rows, nk_size_t columns, nk_size_t depth,
+    nk_size_t a_stride_elements, nk_size_t c_stride_elements) NK_STREAMING_ {
+
+    nk_dots_sme_packed_header_t const *header = (nk_dots_sme_packed_header_t const *)b_packed;
+    nk_f64_t const *b_norms = (nk_f64_t const *)((char const *)b_packed + header->norms_offset);
+
+    for (nk_size_t row_index = 0; row_index < rows; row_index++) {
+        nk_f64_t const *a_row = a + row_index * a_stride_elements;
+        nk_f64_t *c_row = c + row_index * c_stride_elements;
+        nk_f64_t query_norm_sq_f64 = nk_dots_reduce_sumsq_f64_ssve_(a_row, depth);
+        svfloat64_t query_norm_sq_f64x = svdup_n_f64(query_norm_sq_f64);
+
+        for (nk_size_t col_index = 0; col_index < columns; col_index += svcntd()) {
+            svbool_t predicate_b64x = svwhilelt_b64_u64(col_index, columns);
+            svfloat64_t dots_f64x = svld1_f64(predicate_b64x, c_row + col_index);
+            svfloat64_t target_norms_sq_f64x = svld1_f64(predicate_b64x, b_norms + col_index);
+            svst1_f64(predicate_b64x, c_row + col_index,
+                      nk_angulars_from_dot_f64x_ssvef64_(predicate_b64x, dots_f64x, query_norm_sq_f64x,
+                                                         target_norms_sq_f64x));
+        }
+    }
+}
+
+NK_PUBLIC void nk_angulars_packed_f64_smef64( //
+    nk_f64_t const *a, void const *b_packed, nk_f64_t *c, nk_size_t rows, nk_size_t columns, nk_size_t depth,
+    nk_size_t a_stride_in_bytes, nk_size_t c_stride_in_bytes) {
+
+    nk_size_t const a_stride_elements = a_stride_in_bytes / sizeof(nk_f64_t);
+    nk_size_t const c_stride_elements = c_stride_in_bytes / sizeof(nk_f64_t);
+
+    nk_sme_start_streaming_();
+    nk_dots_packed_f64_smef64_streaming_(a, b_packed, c, rows, columns, depth, a_stride_elements, c_stride_elements);
+    nk_angulars_packed_f64_smef64_finalize_ssve_(a, b_packed, c, rows, columns, depth, a_stride_elements,
+                                                 c_stride_elements);
+    nk_sme_stop_streaming_();
+}
+
+#pragma endregion F64 Packed Angular
+#pragma region F64 Packed Euclidean
+
+static void nk_euclideans_packed_f64_smef64_finalize_ssve_( //
+    nk_f64_t const *a, void const *b_packed, nk_f64_t *c, nk_size_t rows, nk_size_t columns, nk_size_t depth,
+    nk_size_t a_stride_elements, nk_size_t c_stride_elements) NK_STREAMING_ {
+
+    nk_dots_sme_packed_header_t const *header = (nk_dots_sme_packed_header_t const *)b_packed;
+    nk_f64_t const *b_norms = (nk_f64_t const *)((char const *)b_packed + header->norms_offset);
+
+    for (nk_size_t row_index = 0; row_index < rows; row_index++) {
+        nk_f64_t const *a_row = a + row_index * a_stride_elements;
+        nk_f64_t *c_row = c + row_index * c_stride_elements;
+        nk_f64_t query_norm_sq_f64 = nk_dots_reduce_sumsq_f64_ssve_(a_row, depth);
+        svfloat64_t query_norm_sq_f64x = svdup_n_f64(query_norm_sq_f64);
+
+        for (nk_size_t col_index = 0; col_index < columns; col_index += svcntd()) {
+            svbool_t predicate_b64x = svwhilelt_b64_u64(col_index, columns);
+            svfloat64_t dots_f64x = svld1_f64(predicate_b64x, c_row + col_index);
+            svfloat64_t target_norms_sq_f64x = svld1_f64(predicate_b64x, b_norms + col_index);
+            svst1_f64(predicate_b64x, c_row + col_index,
+                      nk_euclideans_from_dot_f64x_ssvef64_(predicate_b64x, dots_f64x, query_norm_sq_f64x,
+                                                           target_norms_sq_f64x));
+        }
+    }
+}
+
+NK_PUBLIC void nk_euclideans_packed_f64_smef64( //
+    nk_f64_t const *a, void const *b_packed, nk_f64_t *c, nk_size_t rows, nk_size_t columns, nk_size_t depth,
+    nk_size_t a_stride_in_bytes, nk_size_t c_stride_in_bytes) {
+
+    nk_size_t const a_stride_elements = a_stride_in_bytes / sizeof(nk_f64_t);
+    nk_size_t const c_stride_elements = c_stride_in_bytes / sizeof(nk_f64_t);
+
+    nk_sme_start_streaming_();
+    nk_dots_packed_f64_smef64_streaming_(a, b_packed, c, rows, columns, depth, a_stride_elements, c_stride_elements);
+    nk_euclideans_packed_f64_smef64_finalize_ssve_(a, b_packed, c, rows, columns, depth, a_stride_elements,
+                                                   c_stride_elements);
+    nk_sme_stop_streaming_();
+}
+
+#pragma endregion F64 Packed Euclidean
+#pragma region F64 Symmetric Angular
+
+static void nk_angulars_symmetric_f64_smef64_finalize_ssve_( //
+    nk_f64_t const *vectors, nk_size_t vectors_count, nk_size_t depth, nk_size_t stride_elements, nk_f64_t *result,
+    nk_size_t result_stride_elements, nk_size_t row_start, nk_size_t row_count) NK_STREAMING_ {
+    // Phase 1: cache row norms on diagonal
+    for (nk_size_t row_index = row_start; row_index < row_start + row_count; ++row_index) {
+        nk_f64_t const *row_vector = vectors + row_index * stride_elements;
+        nk_f64_t *result_row = result + row_index * result_stride_elements;
+        result_row[row_index] = nk_dots_reduce_sumsq_f64_ssve_(row_vector, depth);
+    }
+    // Phase 2: column-chunked post-processing
+    nk_f64_t column_norms[256];
+    for (nk_size_t chunk_start = 0; chunk_start < vectors_count; chunk_start += 256) {
+        nk_size_t chunk_end = chunk_start + 256 < vectors_count ? chunk_start + 256 : vectors_count;
+        for (nk_size_t col = chunk_start; col < chunk_end; ++col) {
+            nk_f64_t const *col_vector = vectors + col * stride_elements;
+            column_norms[col - chunk_start] = nk_dots_reduce_sumsq_f64_ssve_(col_vector, depth);
+        }
+        for (nk_size_t row_index = row_start; row_index < row_start + row_count; ++row_index) {
+            nk_size_t col_start = row_index + 1 > chunk_start ? row_index + 1 : chunk_start;
+            if (col_start >= chunk_end) continue;
+            nk_f64_t *result_row = result + row_index * result_stride_elements;
+            svfloat64_t query_norm_sq_f64x = svdup_n_f64(result_row[row_index]);
+            for (nk_size_t col_index = col_start; col_index < chunk_end; col_index += svcntd()) {
+                svbool_t predicate_b64x = svwhilelt_b64_u64(col_index, chunk_end);
+                svfloat64_t dots_f64x = svld1_f64(predicate_b64x, result_row + col_index);
+                svfloat64_t target_norms_sq_f64x = svld1_f64(predicate_b64x, column_norms + (col_index - chunk_start));
+                svst1_f64(predicate_b64x, result_row + col_index,
+                          nk_angulars_from_dot_f64x_ssvef64_(predicate_b64x, dots_f64x, query_norm_sq_f64x,
+                                                             target_norms_sq_f64x));
+            }
+        }
+    }
+    // Phase 3: zero diagonals
+    for (nk_size_t row_index = row_start; row_index < row_start + row_count; ++row_index)
+        result[row_index * result_stride_elements + row_index] = 0;
+}
+
+NK_PUBLIC void nk_angulars_symmetric_f64_smef64( //
+    nk_f64_t const *vectors, nk_size_t vectors_count, nk_size_t depth, nk_size_t stride_in_bytes, nk_f64_t *result,
+    nk_size_t result_stride_in_bytes, nk_size_t row_start, nk_size_t row_count) {
+
+    nk_size_t const stride_elements = stride_in_bytes / sizeof(nk_f64_t);
+    nk_size_t const result_stride_elements = result_stride_in_bytes / sizeof(nk_f64_t);
+
+    nk_sme_start_streaming_();
+    nk_dots_symmetric_f64_smef64_streaming_(vectors, vectors_count, depth, stride_elements, result,
+                                            result_stride_elements, row_start, row_count);
+    nk_angulars_symmetric_f64_smef64_finalize_ssve_(vectors, vectors_count, depth, stride_elements, result,
+                                                    result_stride_elements, row_start, row_count);
+    nk_sme_stop_streaming_();
+}
+
+#pragma endregion F64 Symmetric Angular
+#pragma region F64 Symmetric Euclidean
+
+static void nk_euclideans_symmetric_f64_smef64_finalize_ssve_( //
+    nk_f64_t const *vectors, nk_size_t vectors_count, nk_size_t depth, nk_size_t stride_elements, nk_f64_t *result,
+    nk_size_t result_stride_elements, nk_size_t row_start, nk_size_t row_count) NK_STREAMING_ {
+    // Phase 1: cache row norms on diagonal
+    for (nk_size_t row_index = row_start; row_index < row_start + row_count; ++row_index) {
+        nk_f64_t const *row_vector = vectors + row_index * stride_elements;
+        nk_f64_t *result_row = result + row_index * result_stride_elements;
+        result_row[row_index] = nk_dots_reduce_sumsq_f64_ssve_(row_vector, depth);
+    }
+    // Phase 2: column-chunked post-processing
+    nk_f64_t column_norms[256];
+    for (nk_size_t chunk_start = 0; chunk_start < vectors_count; chunk_start += 256) {
+        nk_size_t chunk_end = chunk_start + 256 < vectors_count ? chunk_start + 256 : vectors_count;
+        for (nk_size_t col = chunk_start; col < chunk_end; ++col) {
+            nk_f64_t const *col_vector = vectors + col * stride_elements;
+            column_norms[col - chunk_start] = nk_dots_reduce_sumsq_f64_ssve_(col_vector, depth);
+        }
+        for (nk_size_t row_index = row_start; row_index < row_start + row_count; ++row_index) {
+            nk_size_t col_start = row_index + 1 > chunk_start ? row_index + 1 : chunk_start;
+            if (col_start >= chunk_end) continue;
+            nk_f64_t *result_row = result + row_index * result_stride_elements;
+            svfloat64_t query_norm_sq_f64x = svdup_n_f64(result_row[row_index]);
+            for (nk_size_t col_index = col_start; col_index < chunk_end; col_index += svcntd()) {
+                svbool_t predicate_b64x = svwhilelt_b64_u64(col_index, chunk_end);
+                svfloat64_t dots_f64x = svld1_f64(predicate_b64x, result_row + col_index);
+                svfloat64_t target_norms_sq_f64x = svld1_f64(predicate_b64x, column_norms + (col_index - chunk_start));
+                svst1_f64(predicate_b64x, result_row + col_index,
+                          nk_euclideans_from_dot_f64x_ssvef64_(predicate_b64x, dots_f64x, query_norm_sq_f64x,
+                                                               target_norms_sq_f64x));
+            }
+        }
+    }
+    // Phase 3: zero diagonals
+    for (nk_size_t row_index = row_start; row_index < row_start + row_count; ++row_index)
+        result[row_index * result_stride_elements + row_index] = 0;
+}
+
+NK_PUBLIC void nk_euclideans_symmetric_f64_smef64( //
+    nk_f64_t const *vectors, nk_size_t vectors_count, nk_size_t depth, nk_size_t stride_in_bytes, nk_f64_t *result,
+    nk_size_t result_stride_in_bytes, nk_size_t row_start, nk_size_t row_count) {
+
+    nk_size_t const stride_elements = stride_in_bytes / sizeof(nk_f64_t);
+    nk_size_t const result_stride_elements = result_stride_in_bytes / sizeof(nk_f64_t);
+
+    nk_sme_start_streaming_();
+    nk_dots_symmetric_f64_smef64_streaming_(vectors, vectors_count, depth, stride_elements, result,
+                                            result_stride_elements, row_start, row_count);
+    nk_euclideans_symmetric_f64_smef64_finalize_ssve_(vectors, vectors_count, depth, stride_elements, result,
+                                                      result_stride_elements, row_start, row_count);
+    nk_sme_stop_streaming_();
+}
+
+#pragma endregion F64 Symmetric Euclidean
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
+#if defined(__cplusplus)
+} // extern "C"
+#endif
+
+#endif // NK_TARGET_SME
+#endif // NK_TARGET_ARM64_
+#endif // NK_SPATIALS_SMEF64_H