whisper.rn 0.4.0-rc.10 → 0.4.0-rc.11

package/cpp/ggml-cpu.cpp

@@ -2,11 +2,22 @@
 #include "ggml-backend-impl.h"
 #include "ggml-cpu.h"
 #include "ggml-cpu-aarch64.h"
+#include "ggml-cpu-traits.h"
 #include "ggml-impl.h"
+#include "amx/amx.h"
+
 #include <cctype>
 #include <string>
 #include <vector>
 
+#ifdef WSP_GGML_USE_CPU_HBM
+#include "ggml-cpu-hbm.h"
+#endif
+
+#ifdef WSP_GGML_USE_CPU_KLEIDIAI
+#include "kleidiai/kleidiai.h"
+#endif
+
 #if defined(__APPLE__)
 #include <sys/types.h>
 #include <sys/sysctl.h>
@@ -22,124 +33,26 @@
 
 // ggml-backend interface
 
-#ifdef WSP_GGML_USE_CPU_HBM
-
-// buffer type HBM
-
-#include <hbwmalloc.h>
-
-static const char * wsp_ggml_backend_cpu_hbm_buffer_type_get_name(wsp_ggml_backend_buffer_type_t buft) {
-    return "CPU_HBM";
-
-    WSP_GGML_UNUSED(buft);
-}
-
-static void wsp_ggml_backend_cpu_hbm_buffer_free_buffer(wsp_ggml_backend_buffer_t buffer) {
-    hbw_free(buffer->context);
-}
-
-static wsp_ggml_backend_buffer_t wsp_ggml_backend_cpu_hbm_buffer_type_alloc_buffer(wsp_ggml_backend_buffer_type_t buft, size_t size) {
-    void * ptr;
-    int result = hbw_posix_memalign(&ptr, wsp_ggml_backend_cpu_buffer_type_get_alignment(buft), size);
-    if (result != 0) {
-        WSP_GGML_LOG_ERROR("failed to allocate HBM buffer of size %zu\n", size);
-        return NULL;
-    }
-
-    wsp_ggml_backend_buffer_t buffer = wsp_ggml_backend_cpu_buffer_from_ptr(ptr, size);
-    buffer->buft = buft;
-    buffer->iface.free_buffer = wsp_ggml_backend_cpu_hbm_buffer_free_buffer;
-
-    return buffer;
-}
-
-wsp_ggml_backend_buffer_type_t wsp_ggml_backend_cpu_hbm_buffer_type(void) {
-    static struct wsp_ggml_backend_buffer_type wsp_ggml_backend_cpu_buffer_type_hbm = {
-        /* .iface    = */ {
-            /* .get_name         = */ wsp_ggml_backend_cpu_hbm_buffer_type_get_name,
-            /* .alloc_buffer     = */ wsp_ggml_backend_cpu_hbm_buffer_type_alloc_buffer,
-            /* .get_alignment    = */ wsp_ggml_backend_cpu_buffer_type_get_alignment,
-            /* .get_max_size     = */ NULL, // defaults to SIZE_MAX
-            /* .get_alloc_size   = */ NULL, // defaults to wsp_ggml_nbytes
-            /* .is_host          = */ wsp_ggml_backend_cpu_buffer_type_is_host,
-        },
-        /* .context  = */ NULL,
-    };
-
-    return &wsp_ggml_backend_cpu_buffer_type_hbm;
-}
-#endif
-
-// buffer type AARCH64
-
-static void wsp_ggml_backend_cpu_aarch64_buffer_init_tensor(wsp_ggml_backend_buffer_t buffer, struct wsp_ggml_tensor * tensor) {
-    tensor->extra = (void *)wsp_ggml_aarch64_get_optimal_repack_type(tensor); // NOLINT
-
-    WSP_GGML_UNUSED(buffer);
-}
-
-static void wsp_ggml_backend_cpu_aarch64_buffer_set_tensor(wsp_ggml_backend_buffer_t buffer, struct wsp_ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
-    WSP_GGML_ASSERT(offset == 0);
-    WSP_GGML_ASSERT(size == wsp_ggml_nbytes(tensor));
-
-    enum wsp_ggml_type repack_type = (enum wsp_ggml_type)(intptr_t)tensor->extra;
-
-    wsp_ggml_aarch64_repack_tensor(tensor, repack_type, data, size);
-
-    WSP_GGML_UNUSED(buffer);
-}
-
-static const char * wsp_ggml_backend_cpu_aarch64_buffer_type_get_name(wsp_ggml_backend_buffer_type_t buft) {
-    return "CPU_AARCH64";
-
-    WSP_GGML_UNUSED(buft);
-}
-
-static wsp_ggml_backend_buffer_t wsp_ggml_backend_cpu_aarch64_buffer_type_alloc_buffer(wsp_ggml_backend_buffer_type_t buft, size_t size) {
-    auto * buffer = wsp_ggml_backend_buft_alloc_buffer(wsp_ggml_backend_cpu_buffer_type(), size);
-
-    if (buffer == NULL) {
-        return NULL;
-    }
-
-    buffer->buft = buft;
-    buffer->iface.init_tensor = wsp_ggml_backend_cpu_aarch64_buffer_init_tensor;
-    buffer->iface.set_tensor = wsp_ggml_backend_cpu_aarch64_buffer_set_tensor;
-
-    return buffer;
-}
-
-wsp_ggml_backend_buffer_type_t wsp_ggml_backend_cpu_aarch64_buffer_type(void) {
-    static struct wsp_ggml_backend_buffer_type wsp_ggml_backend_cpu_buffer_type_aarch64 = {
-        /* .iface    = */ {
-            /* .get_name         = */ wsp_ggml_backend_cpu_aarch64_buffer_type_get_name,
-            /* .alloc_buffer     = */ wsp_ggml_backend_cpu_aarch64_buffer_type_alloc_buffer,
-            /* .get_alignment    = */ wsp_ggml_backend_cpu_buffer_type()->iface.get_alignment,
-            /* .get_max_size     = */ NULL, // defaults to SIZE_MAX
-            /* .get_alloc_size   = */ NULL, // defaults to wsp_ggml_nbytes
-            /* .is_host          = */ NULL,
-        },
-        /* .device  = */ wsp_ggml_backend_reg_dev_get(wsp_ggml_backend_cpu_reg(), 0),
-        /* .context = */ NULL,
-    };
-
-    return &wsp_ggml_backend_cpu_buffer_type_aarch64;
-}
-
-bool wsp_ggml_backend_cpu_buft_is_aarch64(wsp_ggml_backend_buffer_type_t buft) {
-    return buft == wsp_ggml_backend_cpu_aarch64_buffer_type();
-}
-
-static wsp_ggml_backend_buffer_type_t * wsp_ggml_backend_cpu_get_extra_bufts(wsp_ggml_backend_dev_t device) {
+std::vector<wsp_ggml_backend_buffer_type_t>& wsp_ggml_backend_cpu_get_extra_buffers_type() {
     static std::vector<wsp_ggml_backend_buffer_type_t> bufts = []() {
         std::vector<wsp_ggml_backend_buffer_type_t> bufts;
 
-#ifdef WSP_GGML_USE_CPU_HBM
-        bufts.push_back(wsp_ggml_backend_cpu_hbm_buffer_type());
+#if defined(__AMX_INT8__) && defined(__AVX512VNNI__)
+        if (wsp_ggml_backend_amx_buffer_type()) {
+            bufts.push_back(wsp_ggml_backend_amx_buffer_type());
+        }
+#endif
+
+#ifdef WSP_GGML_USE_CPU_KLEIDIAI
+        if (wsp_ggml_backend_cpu_kleidiai_buffer_type()) {
+            bufts.push_back(wsp_ggml_backend_cpu_kleidiai_buffer_type());
+        }
 #endif
 
 #ifdef WSP_GGML_USE_CPU_AARCH64
-        bufts.push_back(wsp_ggml_backend_cpu_aarch64_buffer_type());
+        if (wsp_ggml_backend_cpu_aarch64_buffer_type()) {
+            bufts.push_back(wsp_ggml_backend_cpu_aarch64_buffer_type());
+        }
 #endif
 
         bufts.push_back(NULL);
@@ -147,11 +60,22 @@ static wsp_ggml_backend_buffer_type_t * wsp_ggml_backend_cpu_get_extra_bufts(wsp
         return bufts;
     }();
 
-    return bufts.data();
+    return bufts;
+}
+
+static wsp_ggml_backend_buffer_type_t * wsp_ggml_backend_cpu_device_get_extra_buffers_type(wsp_ggml_backend_dev_t device) {
+    return wsp_ggml_backend_cpu_get_extra_buffers_type().data();
 
     WSP_GGML_UNUSED(device);
 }
 
+static bool wsp_ggml_backend_cpu_is_extra_buffer_type(wsp_ggml_backend_buffer_type_t buft) {
+    for (auto extra : wsp_ggml_backend_cpu_get_extra_buffers_type()) {
+        if (extra && extra == buft) return true;
+    }
+    return false;
+}
+
 // CPU backend - backend (stream)
 
 struct wsp_ggml_backend_cpu_context {
@@ -370,14 +294,14 @@ struct wsp_ggml_backend_cpu_device_context {
                         &hKey) == ERROR_SUCCESS) {
             DWORD cpu_brand_size = 0;
             if (RegQueryValueExA(hKey,
-                                TEXT("ProcessorNameString"),
+                                "ProcessorNameString",
                                 NULL,
                                 NULL,
                                 NULL,
                                 &cpu_brand_size) == ERROR_SUCCESS) {
                 description.resize(cpu_brand_size);
                 if (RegQueryValueExA(hKey,
-                                    TEXT("ProcessorNameString"),
+                                    "ProcessorNameString",
                                     NULL,
                                     NULL,
                                     (LPBYTE)&description[0], // NOLINT
@@ -456,14 +380,23 @@ static bool wsp_ggml_backend_cpu_device_supports_op(wsp_ggml_backend_dev_t dev,
     const struct wsp_ggml_tensor * src0 = op->src[0];
     const struct wsp_ggml_tensor * src1 = op->src[1];
 
-    if (src0 && src0->buffer && wsp_ggml_backend_cpu_buft_is_aarch64(src0->buffer->buft)) {
-        if (op->op != WSP_GGML_OP_MUL_MAT || src0->type != WSP_GGML_TYPE_Q4_0 || wsp_ggml_aarch64_get_optimal_repack_type(src0) == WSP_GGML_TYPE_Q4_0) {
-            return false;
+    if (op->op == WSP_GGML_OP_NONE || op->op == WSP_GGML_OP_RESHAPE || op->op == WSP_GGML_OP_VIEW || op->op == WSP_GGML_OP_PERMUTE || op->op == WSP_GGML_OP_TRANSPOSE) {
+        return true;
+    }
+
+    // extra_buffer_op?
+    for (auto extra : wsp_ggml_backend_cpu_get_extra_buffers_type()) {
+        if (extra) {
+            auto buf_extra = (ggml::cpu::extra_buffer_type*) extra->context;
+            if (buf_extra && buf_extra->supports_op(dev, op)) {
+                return true;
+            }
         }
     }
 
-    for (int i = 1; i < WSP_GGML_MAX_SRC; i++) {
-        if (op->src[i] && op->src[i]->buffer && wsp_ggml_backend_cpu_buft_is_aarch64(op->src[i]->buffer->buft)) {
+    // the other case need host buffer.
+    for (int i = 0; i < WSP_GGML_MAX_SRC; i++) {
+        if (op->src[i] && op->src[i]->buffer && !wsp_ggml_backend_buft_is_host(op->src[i]->buffer->buft)) {
             return false;
         }
     }
@@ -471,28 +404,37 @@ static bool wsp_ggml_backend_cpu_device_supports_op(wsp_ggml_backend_dev_t dev,
     switch (op->op) {
         case WSP_GGML_OP_CPY:
             return
+                op->type != WSP_GGML_TYPE_IQ3_XXS &&
+                op->type != WSP_GGML_TYPE_IQ3_S   &&
                 op->type != WSP_GGML_TYPE_IQ2_XXS &&
                 op->type != WSP_GGML_TYPE_IQ2_XS  &&
+                op->type != WSP_GGML_TYPE_IQ2_S   &&
                 op->type != WSP_GGML_TYPE_IQ1_S   &&
                 op->type != WSP_GGML_TYPE_IQ1_M; // missing type_traits.from_float
         case WSP_GGML_OP_MUL_MAT:
             return src1->type == WSP_GGML_TYPE_F32 || src1->type == wsp_ggml_get_type_traits_cpu(src0->type)->vec_dot_type;
-        case WSP_GGML_OP_ROPE_BACK:
-            return op->src[2] == NULL && (op->op_params[2] & 4) == 0;
+        case WSP_GGML_OP_SOFT_MAX_BACK: {
+            if (op->src[0]->type != WSP_GGML_TYPE_F32 || op->src[1]->type != WSP_GGML_TYPE_F32) {
+                return false;
+            }
+            float max_bias = 0.0f;
+
+            memcpy(&max_bias, (const float *) op->op_params + 1, sizeof(float));
+
+            return max_bias == 0.0f;
+        }
         case WSP_GGML_OP_IM2COL_BACK:
             return src0->type == WSP_GGML_TYPE_F32 && src1->type == WSP_GGML_TYPE_F32;
         case WSP_GGML_OP_OUT_PROD:
-            return (src0->type == WSP_GGML_TYPE_F32 || wsp_ggml_is_quantized(src0->type)) && src1->type == WSP_GGML_TYPE_F32;
+            return (src0->type == WSP_GGML_TYPE_F32 || (wsp_ggml_is_quantized(src0->type) && src0->ne[2] == src1->ne[2] && src0->ne[3] == src1->ne[3])) &&
+                src1->type == WSP_GGML_TYPE_F32 && op->type == WSP_GGML_TYPE_F32;
         default:
             return true;
     }
-
-    WSP_GGML_UNUSED(dev);
 }
 
 static bool wsp_ggml_backend_cpu_device_supports_buft(wsp_ggml_backend_dev_t dev, wsp_ggml_backend_buffer_type_t buft) {
-    return wsp_ggml_backend_buft_is_host(buft) || wsp_ggml_backend_cpu_buft_is_aarch64(buft);
-
+    return wsp_ggml_backend_buft_is_host(buft) || wsp_ggml_backend_cpu_is_extra_buffer_type(buft);
     WSP_GGML_UNUSED(dev);
 }
 
@@ -541,16 +483,12 @@ static wsp_ggml_backend_dev_t wsp_ggml_backend_cpu_reg_get_device(wsp_ggml_backe
     return &wsp_ggml_backend_cpu_device;
 }
 
-struct wsp_ggml_backend_feature {
-    const char * name;
-    const char * value;
-};
-
-// Not used yet
 // This is intended to replace the the wsp_ggml_cpu_has_* functions when loading the CPU backend dynamically,
-// and additionally to allow other backends to expose their own list of features that applications can query using the same API.
+// and additionally to allow other backends to expose their own list of features that applications can query using the same API
 static wsp_ggml_backend_feature * wsp_ggml_backend_cpu_get_features(wsp_ggml_backend_reg_t reg) {
     static std::vector<wsp_ggml_backend_feature> features = []() {
+        wsp_ggml_cpu_init();
+
         std::vector<wsp_ggml_backend_feature> features;
         if (wsp_ggml_cpu_has_sse3()) {
             features.push_back({ "SSE3", "1" });
@@ -561,6 +499,9 @@ static wsp_ggml_backend_feature * wsp_ggml_backend_cpu_get_features(wsp_ggml_bac
         if (wsp_ggml_cpu_has_avx()) {
             features.push_back({ "AVX", "1" });
         }
+        if (wsp_ggml_cpu_has_avx_vnni()) {
+            features.push_back({ "AVX_VNNI", "1" });
+        }
         if (wsp_ggml_cpu_has_avx2()) {
             features.push_back({ "AVX2", "1" });
         }
@@ -570,9 +511,6 @@ static wsp_ggml_backend_feature * wsp_ggml_backend_cpu_get_features(wsp_ggml_bac
         if (wsp_ggml_cpu_has_fma()) {
             features.push_back({ "FMA", "1" });
         }
-        if (wsp_ggml_cpu_has_avx_vnni()) {
-            features.push_back({ "AVX_VNNI", "1" });
-        }
         if (wsp_ggml_cpu_has_avx512()) {
             features.push_back({ "AVX512", "1" });
         }
@@ -603,22 +541,46 @@ static wsp_ggml_backend_feature * wsp_ggml_backend_cpu_get_features(wsp_ggml_bac
         if (wsp_ggml_cpu_has_sve()) {
             features.push_back({ "SVE", "1" });
         }
+        if (wsp_ggml_cpu_has_dotprod()) {
+            features.push_back({ "DOTPROD", "1" });
+        }
         if (wsp_ggml_cpu_get_sve_cnt() > 0) {
             static std::string sve_cnt = std::to_string(wsp_ggml_cpu_get_sve_cnt());
             features.push_back({ "SVE_CNT", sve_cnt.c_str() });
         }
+        if (wsp_ggml_cpu_has_sme()) {
+            features.push_back({ "SME", "1" });
+        }
         if (wsp_ggml_cpu_has_riscv_v()) {
             features.push_back({ "RISCV_V", "1" });
         }
         if (wsp_ggml_cpu_has_vsx()) {
             features.push_back({ "VSX", "1" });
         }
+        if (wsp_ggml_cpu_has_vxe()) {
+            features.push_back({ "VXE", "1" });
+        }
         if (wsp_ggml_cpu_has_wasm_simd()) {
             features.push_back({ "WASM_SIMD", "1" });
         }
         if (wsp_ggml_cpu_has_llamafile()) {
             features.push_back({ "LLAMAFILE", "1" });
         }
+    #ifdef WSP_GGML_USE_ACCELERATE
+        features.push_back({ "ACCELERATE", "1" });
+    #endif
+    #ifdef WSP_GGML_USE_CPU_HBM
+        features.push_back({ "CPU_HBM", "1" });
+    #endif
+    #ifdef WSP_GGML_USE_OPENMP
+        features.push_back({ "OPENMP", "1" });
+    #endif
+    #ifdef WSP_GGML_USE_CPU_KLEIDIAI
+        features.push_back({ "KLEIDIAI", "1" });
+    #endif
+    #ifdef WSP_GGML_USE_CPU_AARCH64
+        features.push_back({ "AARCH64_REPACK", "1" });
+    #endif
 
         features.push_back({ nullptr, nullptr });
 
@@ -632,10 +594,35 @@ static wsp_ggml_backend_feature * wsp_ggml_backend_cpu_get_features(wsp_ggml_bac
 
 static void * wsp_ggml_backend_cpu_get_proc_address(wsp_ggml_backend_reg_t reg, const char * name) {
     if (strcmp(name, "wsp_ggml_backend_set_n_threads") == 0) {
-        return (void *)wsp_ggml_backend_cpu_set_n_threads;
+        wsp_ggml_backend_set_n_threads_t fct = wsp_ggml_backend_cpu_set_n_threads;
+        return (void *)fct;
     }
     if (strcmp(name, "wsp_ggml_backend_dev_get_extra_bufts") == 0) {
-        return (void *)wsp_ggml_backend_cpu_get_extra_bufts;
+        wsp_ggml_backend_dev_get_extra_bufts_t fct = wsp_ggml_backend_cpu_device_get_extra_buffers_type;
+        return (void *)fct;
+    }
+    if (strcmp(name, "wsp_ggml_backend_get_features") == 0) {
+        return (void *)wsp_ggml_backend_cpu_get_features;
+    }
+    if (strcmp(name, "wsp_ggml_backend_set_abort_callback") == 0) {
+        return (void *)wsp_ggml_backend_cpu_set_abort_callback;
+    }
+    if (strcmp(name, "wsp_ggml_backend_cpu_numa_init") == 0) {
+        return (void *)wsp_ggml_numa_init;
+    }
+    if (strcmp(name, "wsp_ggml_backend_cpu_is_numa") == 0) {
+        return (void *)wsp_ggml_is_numa;
+    }
+
+    // threadpool - TODO:  move to ggml-base
+    if (strcmp(name, "wsp_ggml_threadpool_new") == 0) {
+        return (void *)wsp_ggml_threadpool_new;
+    }
+    if (strcmp(name, "wsp_ggml_threadpool_free") == 0) {
+        return (void *)wsp_ggml_threadpool_free;
+    }
+    if (strcmp(name, "wsp_ggml_backend_cpu_set_threadpool") == 0) {
+        return (void *)wsp_ggml_backend_cpu_set_threadpool;
     }
 
     return NULL;
@@ -655,9 +642,12 @@ wsp_ggml_backend_reg_t wsp_ggml_backend_cpu_reg(void) {
     wsp_ggml_cpu_init();
 
     static struct wsp_ggml_backend_reg wsp_ggml_backend_cpu_reg = {
-        /* .iface   = */ wsp_ggml_backend_cpu_reg_i,
-        /* .context = */ NULL,
+        /* .api_version = */ WSP_GGML_BACKEND_API_VERSION,
+        /* .iface       = */ wsp_ggml_backend_cpu_reg_i,
+        /* .context     = */ NULL,
     };
 
     return &wsp_ggml_backend_cpu_reg;
 }
+
+WSP_GGML_BACKEND_DL_IMPL(wsp_ggml_backend_cpu_reg)

package/cpp/ggml-cpu.h

@@ -7,31 +7,8 @@
 extern "C" {
 #endif
 
-    // Scheduling priorities
-    enum wsp_ggml_sched_priority {
-        WSP_GGML_SCHED_PRIO_NORMAL,
-        WSP_GGML_SCHED_PRIO_MEDIUM,
-        WSP_GGML_SCHED_PRIO_HIGH,
-        WSP_GGML_SCHED_PRIO_REALTIME
-    };
-
-    // Threadpool params
-    // Use wsp_ggml_threadpool_params_default() or wsp_ggml_threadpool_params_init() to populate the defaults
-    struct wsp_ggml_threadpool_params {
-        bool                cpumask[WSP_GGML_MAX_N_THREADS]; // mask of cpu cores (all-zeros means use default affinity settings)
-        int                 n_threads;                   // number of threads
-        enum wsp_ggml_sched_priority prio;                   // thread priority
-        uint32_t            poll;                        // polling level (0 - no polling, 100 - aggressive polling)
-        bool                strict_cpu;                  // strict cpu placement
-        bool                paused;                      // start in paused state
-    };
-
-    struct wsp_ggml_threadpool;     // forward declaration, see ggml.c
-
-    typedef struct wsp_ggml_threadpool * wsp_ggml_threadpool_t;
-
     // the compute plan that needs to be prepared for wsp_ggml_graph_compute()
-    // since https://github.com/ggerganov/ggml/issues/287
+    // since https://github.com/ggml-org/ggml/issues/287
     struct wsp_ggml_cplan {
         size_t    work_size; // size of work buffer, calculated by `wsp_ggml_graph_plan()`
         uint8_t * work_data; // work buffer, to be allocated by caller before calling to `wsp_ggml_graph_compute()`
@@ -75,14 +52,11 @@ extern "C" {
     WSP_GGML_BACKEND_API float   wsp_ggml_get_f32_nd(const struct wsp_ggml_tensor * tensor, int i0, int i1, int i2, int i3);
     WSP_GGML_BACKEND_API void    wsp_ggml_set_f32_nd(const struct wsp_ggml_tensor * tensor, int i0, int i1, int i2, int i3, float value);
 
-    WSP_GGML_BACKEND_API struct wsp_ggml_threadpool_params wsp_ggml_threadpool_params_default(int n_threads);
-    WSP_GGML_BACKEND_API void                          wsp_ggml_threadpool_params_init   (struct wsp_ggml_threadpool_params * p, int n_threads);
-    WSP_GGML_BACKEND_API bool                          wsp_ggml_threadpool_params_match  (const struct wsp_ggml_threadpool_params * p0, const struct wsp_ggml_threadpool_params * p1);
-    WSP_GGML_BACKEND_API struct wsp_ggml_threadpool *      wsp_ggml_threadpool_new          (struct wsp_ggml_threadpool_params  * params);
-    WSP_GGML_BACKEND_API void                          wsp_ggml_threadpool_free         (struct wsp_ggml_threadpool * threadpool);
-    WSP_GGML_BACKEND_API int                           wsp_ggml_threadpool_get_n_threads(struct wsp_ggml_threadpool * threadpool);
-    WSP_GGML_BACKEND_API void                          wsp_ggml_threadpool_pause        (struct wsp_ggml_threadpool * threadpool);
-    WSP_GGML_BACKEND_API void                          wsp_ggml_threadpool_resume       (struct wsp_ggml_threadpool * threadpool);
+    WSP_GGML_BACKEND_API struct wsp_ggml_threadpool *      wsp_ggml_threadpool_new           (struct wsp_ggml_threadpool_params  * params);
+    WSP_GGML_BACKEND_API void                          wsp_ggml_threadpool_free          (struct wsp_ggml_threadpool * threadpool);
+    WSP_GGML_BACKEND_API int                           wsp_ggml_threadpool_get_n_threads (struct wsp_ggml_threadpool * threadpool);
+    WSP_GGML_BACKEND_API void                          wsp_ggml_threadpool_pause         (struct wsp_ggml_threadpool * threadpool);
+    WSP_GGML_BACKEND_API void                          wsp_ggml_threadpool_resume        (struct wsp_ggml_threadpool * threadpool);
 
     // wsp_ggml_graph_plan() has to be called before wsp_ggml_graph_compute()
     // when plan.work_size > 0, caller must allocate memory for plan.work_data
@@ -104,10 +78,10 @@ extern "C" {
     WSP_GGML_BACKEND_API int wsp_ggml_cpu_has_sse3       (void);
     WSP_GGML_BACKEND_API int wsp_ggml_cpu_has_ssse3      (void);
     WSP_GGML_BACKEND_API int wsp_ggml_cpu_has_avx        (void);
+    WSP_GGML_BACKEND_API int wsp_ggml_cpu_has_avx_vnni   (void);
     WSP_GGML_BACKEND_API int wsp_ggml_cpu_has_avx2       (void);
     WSP_GGML_BACKEND_API int wsp_ggml_cpu_has_f16c       (void);
     WSP_GGML_BACKEND_API int wsp_ggml_cpu_has_fma        (void);
-    WSP_GGML_BACKEND_API int wsp_ggml_cpu_has_avx_vnni   (void);
     WSP_GGML_BACKEND_API int wsp_ggml_cpu_has_avx512     (void);
     WSP_GGML_BACKEND_API int wsp_ggml_cpu_has_avx512_vbmi(void);
     WSP_GGML_BACKEND_API int wsp_ggml_cpu_has_avx512_vnni(void);
@@ -117,35 +91,28 @@ extern "C" {
     WSP_GGML_BACKEND_API int wsp_ggml_cpu_has_neon       (void);
     WSP_GGML_BACKEND_API int wsp_ggml_cpu_has_arm_fma    (void);
     WSP_GGML_BACKEND_API int wsp_ggml_cpu_has_fp16_va    (void);
+    WSP_GGML_BACKEND_API int wsp_ggml_cpu_has_dotprod    (void);
     WSP_GGML_BACKEND_API int wsp_ggml_cpu_has_matmul_int8(void);
     WSP_GGML_BACKEND_API int wsp_ggml_cpu_has_sve        (void);
     WSP_GGML_BACKEND_API int wsp_ggml_cpu_get_sve_cnt    (void);  // sve vector length in bytes
+    WSP_GGML_BACKEND_API int wsp_ggml_cpu_has_sme        (void);
     // other
     WSP_GGML_BACKEND_API int wsp_ggml_cpu_has_riscv_v    (void);
     WSP_GGML_BACKEND_API int wsp_ggml_cpu_has_vsx        (void);
+    WSP_GGML_BACKEND_API int wsp_ggml_cpu_has_vxe        (void);
     WSP_GGML_BACKEND_API int wsp_ggml_cpu_has_wasm_simd  (void);
     WSP_GGML_BACKEND_API int wsp_ggml_cpu_has_llamafile  (void);
 
     // Internal types and functions exposed for tests and benchmarks
 
-    typedef void (*wsp_ggml_from_float_to_mat_t)
-                                     (const float * WSP_GGML_RESTRICT x, void * WSP_GGML_RESTRICT y, int64_t nr, int64_t k, int64_t bs);
     typedef void (*wsp_ggml_vec_dot_t)  (int n, float * WSP_GGML_RESTRICT s, size_t bs, const void * WSP_GGML_RESTRICT x, size_t bx,
                                        const void * WSP_GGML_RESTRICT y, size_t by, int nrc);
-    typedef void (*wsp_ggml_gemv_t)     (int n, float * WSP_GGML_RESTRICT s, size_t bs, const void * WSP_GGML_RESTRICT x,
-                                       const void * WSP_GGML_RESTRICT y, int nr, int nc);
-    typedef void (*wsp_ggml_gemm_t)     (int n, float * WSP_GGML_RESTRICT s, size_t bs, const void * WSP_GGML_RESTRICT x,
-                                       const void * WSP_GGML_RESTRICT y, int nr, int nc);
 
     struct wsp_ggml_type_traits_cpu {
         wsp_ggml_from_float_t        from_float;
-        wsp_ggml_from_float_to_mat_t from_float_to_mat;
         wsp_ggml_vec_dot_t           vec_dot;
         enum wsp_ggml_type           vec_dot_type;
         int64_t                  nrows; // number of rows to process simultaneously
-        int64_t                  ncols; // number of columns to process simultaneously
-        wsp_ggml_gemv_t              gemv;
-        wsp_ggml_gemm_t              gemm;
     };
 
     WSP_GGML_BACKEND_API const struct wsp_ggml_type_traits_cpu * wsp_ggml_get_type_traits_cpu(enum wsp_ggml_type type);
@@ -165,13 +132,6 @@ extern "C" {
 
     WSP_GGML_BACKEND_API wsp_ggml_backend_reg_t wsp_ggml_backend_cpu_reg(void);
 
-#ifdef WSP_GGML_USE_CPU_HBM
-    WSP_GGML_BACKEND_API wsp_ggml_backend_buffer_type_t wsp_ggml_backend_cpu_hbm_buffer_type(void);
-#endif
-
-    WSP_GGML_BACKEND_API wsp_ggml_backend_buffer_type_t wsp_ggml_backend_cpu_aarch64_buffer_type(void);
-    WSP_GGML_BACKEND_API bool wsp_ggml_backend_cpu_buft_is_aarch64(wsp_ggml_backend_buffer_type_t buft);
-
 #ifdef __cplusplus
 }
 #endif

package/cpp/ggml-impl.h

@@ -3,6 +3,8 @@
 // GGML internal header
 
 #include "ggml.h"
+#include "gguf.h"
+
 #include <assert.h>
 #include <math.h>
 #include <stdlib.h> // load `stdlib.h` before other headers to work around MinGW bug: https://sourceforge.net/p/mingw-w64/bugs/192/
@@ -14,7 +16,7 @@
 #include <arm_sve.h>
 #endif // __ARM_FEATURE_SVE
 
-#if defined(__ARM_NEON)
+#if defined(__ARM_NEON) && !defined(__CUDACC__) && !defined(__MUSACC__)
 // if YCM cannot find <arm_neon.h>, make a symbolic link to it, for example:
 //
 //   $ ln -sfn /Library/Developer/CommandLineTools/usr/lib/clang/13.1.6/include/arm_neon.h ./src/
@@ -30,11 +32,13 @@
 extern "C" {
 #endif
 
-#undef MIN
-#undef MAX
+#ifndef MIN
+#    define MIN(a, b) ((a) < (b) ? (a) : (b))
+#endif
 
-#define MIN(a, b) ((a) < (b) ? (a) : (b))
-#define MAX(a, b) ((a) > (b) ? (a) : (b))
+#ifndef MAX
+#    define MAX(a, b) ((a) > (b) ? (a) : (b))
+#endif
 
 // required for mmap as gguf only guarantees 32-byte alignment
 #define TENSOR_ALIGNMENT 32
@@ -72,8 +76,8 @@ static inline int wsp_ggml_up(int n, int m) {
 //
 
 WSP_GGML_ATTRIBUTE_FORMAT(2, 3)
-void wsp_ggml_log_internal        (enum wsp_ggml_log_level level, const char * format, ...);
-void wsp_ggml_log_callback_default(enum wsp_ggml_log_level level, const char * text, void * user_data);
+WSP_GGML_API void wsp_ggml_log_internal        (enum wsp_ggml_log_level level, const char * format, ...);
+WSP_GGML_API void wsp_ggml_log_callback_default(enum wsp_ggml_log_level level, const char * text, void * user_data);
 
 #define WSP_GGML_LOG(...)       wsp_ggml_log_internal(WSP_GGML_LOG_LEVEL_NONE , __VA_ARGS__)
 #define WSP_GGML_LOG_INFO(...)  wsp_ggml_log_internal(WSP_GGML_LOG_LEVEL_INFO , __VA_ARGS__)
@@ -295,24 +299,27 @@ struct wsp_ggml_cgraph {
     enum wsp_ggml_cgraph_eval_order order;
 };
 
+// returns a slice of cgraph with nodes [i0, i1)
+// the slice does not have leafs or gradients
+// if you need the gradients, get them from the original graph
 struct wsp_ggml_cgraph wsp_ggml_graph_view(struct wsp_ggml_cgraph * cgraph, int i0, int i1);
 
 // Memory allocation
 
-void * wsp_ggml_aligned_malloc(size_t size);
-void wsp_ggml_aligned_free(void * ptr, size_t size);
+WSP_GGML_API void * wsp_ggml_aligned_malloc(size_t size);
+WSP_GGML_API void wsp_ggml_aligned_free(void * ptr, size_t size);
 
 // FP16 to FP32 conversion
 
 #if defined(__ARM_NEON)
-    #ifdef _MSC_VER
+    #if defined(_MSC_VER) || (defined(__CUDACC__) && __CUDACC_VER_MAJOR__ <= 11)
         typedef uint16_t wsp_ggml_fp16_internal_t;
     #else
         typedef __fp16 wsp_ggml_fp16_internal_t;
     #endif
 #endif
 
-#if defined(__ARM_NEON) && !defined(_MSC_VER)
+#if defined(__ARM_NEON) && !defined(_MSC_VER) && !(defined(__CUDACC__) && __CUDACC_VER_MAJOR__ <= 11)
     #define WSP_GGML_COMPUTE_FP16_TO_FP32(x) wsp_ggml_compute_fp16_to_fp32(x)
     #define WSP_GGML_COMPUTE_FP32_TO_FP16(x) wsp_ggml_compute_fp32_to_fp16(x)
 
@@ -549,3 +556,12 @@ static inline wsp_ggml_bf16_t wsp_ggml_compute_fp32_to_bf16(float s) {
 #ifdef __cplusplus
 }
 #endif
+
+#ifdef __cplusplus
+#include <vector>
+
+// expose GGUF internals for test code
+WSP_GGML_API size_t wsp_gguf_type_size(enum wsp_gguf_type type);
+WSP_GGML_API struct wsp_gguf_context * wsp_gguf_init_from_file_impl(FILE * file, struct wsp_gguf_init_params params);
+WSP_GGML_API void wsp_gguf_write_to_buf(const struct wsp_gguf_context * ctx, std::vector<int8_t> & buf, bool only_meta);
+#endif // __cplusplus

package/cpp/ggml-metal-impl.h

@@ -102,6 +102,21 @@ typedef struct {
     uint64_t nb3;
 } wsp_ggml_metal_kargs_cpy;
 
+typedef struct {
+    int64_t  ne10;
+    int64_t  ne11;
+    int64_t  ne12;
+    uint64_t nb10;
+    uint64_t nb11;
+    uint64_t nb12;
+    uint64_t nb13;
+    uint64_t nb1;
+    uint64_t nb2;
+    uint64_t nb3;
+    uint64_t offs;
+    bool     inplace;
+} wsp_ggml_metal_kargs_set;
+
 typedef struct {
     int32_t  ne00;
     int32_t  ne01;
@@ -192,6 +207,30 @@ typedef struct {
     int16_t  r3;
 } wsp_ggml_metal_kargs_mul_mv;
 
+typedef struct {
+    int32_t  ne00;
+    int32_t  ne01;
+    int32_t  ne02;
+    uint64_t nb00;
+    uint64_t nb01;
+    uint64_t nb02;
+    uint64_t nb03;
+    int32_t  ne10;
+    int32_t  ne11;
+    int32_t  ne12;
+    uint64_t nb10;
+    uint64_t nb11;
+    uint64_t nb12;
+    uint64_t nb13;
+    int32_t  ne0;
+    int32_t  ne1;
+    int16_t  r2;
+    int16_t  r3;
+    int16_t  nsg;
+    int16_t  nxpsg;
+    int16_t  r1ptg;
+} wsp_ggml_metal_kargs_mul_mv_ext;
+
 typedef struct {
     int32_t  nei0;
     int32_t  nei1;