whisper.rn 0.4.0-rc.9 → 0.4.0

package/cpp/ggml-backend.h

@@ -3,6 +3,20 @@
 #include "ggml.h"
 #include "ggml-alloc.h"
 
+#ifdef WSP_GGML_BACKEND_SHARED
+#    if defined(_WIN32) && !defined(__MINGW32__)
+#        ifdef WSP_GGML_BACKEND_BUILD
+#            define WSP_GGML_BACKEND_API __declspec(dllexport) extern
+#        else
+#            define WSP_GGML_BACKEND_API __declspec(dllimport) extern
+#        endif
+#    else
+#        define WSP_GGML_BACKEND_API __attribute__ ((visibility ("default"))) extern
+#    endif
+#else
+#    define WSP_GGML_BACKEND_API extern
+#endif
+
 #ifdef  __cplusplus
 extern "C" {
 #endif
@@ -24,7 +38,7 @@ extern "C" {
     WSP_GGML_API wsp_ggml_backend_buffer_t wsp_ggml_backend_buft_alloc_buffer  (wsp_ggml_backend_buffer_type_t buft, size_t size);
     WSP_GGML_API size_t                wsp_ggml_backend_buft_get_alignment (wsp_ggml_backend_buffer_type_t buft);
     WSP_GGML_API size_t                wsp_ggml_backend_buft_get_max_size  (wsp_ggml_backend_buffer_type_t buft);
-    WSP_GGML_API size_t                wsp_ggml_backend_buft_get_alloc_size(wsp_ggml_backend_buffer_type_t buft, struct wsp_ggml_tensor * tensor);
+    WSP_GGML_API size_t                wsp_ggml_backend_buft_get_alloc_size(wsp_ggml_backend_buffer_type_t buft, const struct wsp_ggml_tensor * tensor);
     WSP_GGML_API bool                  wsp_ggml_backend_buft_is_host       (wsp_ggml_backend_buffer_type_t buft);
     WSP_GGML_API wsp_ggml_backend_dev_t    wsp_ggml_backend_buft_get_device    (wsp_ggml_backend_buffer_type_t buft);
 
@@ -42,10 +56,10 @@ extern "C" {
     WSP_GGML_API void                           wsp_ggml_backend_buffer_free          (wsp_ggml_backend_buffer_t buffer);
     WSP_GGML_API void *                         wsp_ggml_backend_buffer_get_base      (wsp_ggml_backend_buffer_t buffer);
     WSP_GGML_API size_t                         wsp_ggml_backend_buffer_get_size      (wsp_ggml_backend_buffer_t buffer);
-    WSP_GGML_API void                           wsp_ggml_backend_buffer_init_tensor   (wsp_ggml_backend_buffer_t buffer, struct wsp_ggml_tensor * tensor);
+    WSP_GGML_API enum wsp_ggml_status               wsp_ggml_backend_buffer_init_tensor   (wsp_ggml_backend_buffer_t buffer, struct wsp_ggml_tensor * tensor);
     WSP_GGML_API size_t                         wsp_ggml_backend_buffer_get_alignment (wsp_ggml_backend_buffer_t buffer);
     WSP_GGML_API size_t                         wsp_ggml_backend_buffer_get_max_size  (wsp_ggml_backend_buffer_t buffer);
-    WSP_GGML_API size_t                         wsp_ggml_backend_buffer_get_alloc_size(wsp_ggml_backend_buffer_t buffer, struct wsp_ggml_tensor * tensor);
+    WSP_GGML_API size_t                         wsp_ggml_backend_buffer_get_alloc_size(wsp_ggml_backend_buffer_t buffer, const struct wsp_ggml_tensor * tensor);
     WSP_GGML_API void                           wsp_ggml_backend_buffer_clear         (wsp_ggml_backend_buffer_t buffer, uint8_t value);
     WSP_GGML_API bool                           wsp_ggml_backend_buffer_is_host       (wsp_ggml_backend_buffer_t buffer);
     WSP_GGML_API void                           wsp_ggml_backend_buffer_set_usage     (wsp_ggml_backend_buffer_t buffer, enum wsp_ggml_backend_buffer_usage usage);
@@ -72,7 +86,7 @@ extern "C" {
     WSP_GGML_API void wsp_ggml_backend_tensor_set_async(wsp_ggml_backend_t backend,       struct wsp_ggml_tensor * tensor, const void * data, size_t offset, size_t size);
     WSP_GGML_API void wsp_ggml_backend_tensor_get_async(wsp_ggml_backend_t backend, const struct wsp_ggml_tensor * tensor,       void * data, size_t offset, size_t size);
 
-    // "offset" refers to the offset of the tensor data for setting/getting data
+    // "offset" refers to the offset in tensor->data for setting/getting data
     WSP_GGML_API void wsp_ggml_backend_tensor_set(      struct wsp_ggml_tensor * tensor, const void * data, size_t offset, size_t size);
     WSP_GGML_API void wsp_ggml_backend_tensor_get(const struct wsp_ggml_tensor * tensor,       void * data, size_t offset, size_t size);
     WSP_GGML_API void wsp_ggml_backend_tensor_memset(   struct wsp_ggml_tensor * tensor,     uint8_t value, size_t offset, size_t size);
@@ -114,11 +128,12 @@ extern "C" {
     //
 
     enum wsp_ggml_backend_dev_type {
+        // CPU device using system memory
         WSP_GGML_BACKEND_DEVICE_TYPE_CPU,
+        // GPU device using dedicated memory
         WSP_GGML_BACKEND_DEVICE_TYPE_GPU,
-        // devices with full capabilities (excludes backends such as BLAS that only support matrix multiplication)
-        WSP_GGML_BACKEND_DEVICE_TYPE_CPU_FULL,
-        WSP_GGML_BACKEND_DEVICE_TYPE_GPU_FULL
+        // accelerator devices intended to be used together with the CPU backend (e.g. BLAS or AMX)
+        WSP_GGML_BACKEND_DEVICE_TYPE_ACCEL
     };
 
     // functionality supported by the device
@@ -167,15 +182,29 @@ extern "C" {
     WSP_GGML_API wsp_ggml_backend_dev_t wsp_ggml_backend_reg_dev_get(wsp_ggml_backend_reg_t reg, size_t index);
     WSP_GGML_API void *             wsp_ggml_backend_reg_get_proc_address(wsp_ggml_backend_reg_t reg, const char * name);
 
-
-    // Functions that may be obtained using wsp_ggml_backend_reg_get_proc_address
-    typedef wsp_ggml_backend_buffer_type_t (*wsp_ggml_backend_split_buffer_type_t)(const float *);
-    typedef void (*wsp_ggml_backend_set_n_threads_t)(wsp_ggml_backend_t, int);
+    // Common functions that may be obtained using wsp_ggml_backend_reg_get_proc_address
+
+    // Split buffer type for tensor parallelism
+    typedef wsp_ggml_backend_buffer_type_t   (*wsp_ggml_backend_split_buffer_type_t)(int main_device, const float * tensor_split);
+    // Set the number of threads for the backend
+    typedef void                         (*wsp_ggml_backend_set_n_threads_t)(wsp_ggml_backend_t backend, int n_threads);
+    // Get additional buffer types provided by the device (returns a NULL-terminated array)
+    typedef wsp_ggml_backend_buffer_type_t * (*wsp_ggml_backend_dev_get_extra_bufts_t)(wsp_ggml_backend_dev_t device);
+    // Set the abort callback for the backend
+    typedef void                         (*wsp_ggml_backend_set_abort_callback_t)(wsp_ggml_backend_t backend, wsp_ggml_abort_callback abort_callback, void * abort_callback_data);
+    // Get a list of feature flags supported by the backend (returns a NULL-terminated array)
+    struct wsp_ggml_backend_feature {
+        const char * name;
+        const char * value;
+    };
+    typedef struct wsp_ggml_backend_feature * (*wsp_ggml_backend_get_features_t)(wsp_ggml_backend_reg_t reg);
 
     //
     // Backend registry
     //
 
+    WSP_GGML_API void wsp_ggml_backend_device_register(wsp_ggml_backend_dev_t device);
+
     // Backend (reg) enumeration
     WSP_GGML_API size_t             wsp_ggml_backend_reg_count(void);
     WSP_GGML_API wsp_ggml_backend_reg_t wsp_ggml_backend_reg_get(size_t index);
@@ -192,9 +221,17 @@ extern "C" {
     WSP_GGML_API wsp_ggml_backend_t wsp_ggml_backend_init_by_name(const char * name, const char * params);
     // = wsp_ggml_backend_dev_init(wsp_ggml_backend_dev_by_type(type), params)
     WSP_GGML_API wsp_ggml_backend_t wsp_ggml_backend_init_by_type(enum wsp_ggml_backend_dev_type type, const char * params);
-    // = wsp_ggml_backend_dev_init(wsp_ggml_backend_dev_by_type(GPU_FULL) OR wsp_ggml_backend_dev_by_type(CPU_FULL), NULL)
+    // = wsp_ggml_backend_dev_init(wsp_ggml_backend_dev_by_type(GPU) OR wsp_ggml_backend_dev_by_type(CPU), NULL)
     WSP_GGML_API wsp_ggml_backend_t wsp_ggml_backend_init_best(void);
 
+    // Load a backend from a dynamic library and register it
+    WSP_GGML_API wsp_ggml_backend_reg_t wsp_ggml_backend_load(const char * path);
+    // Unload a backend if loaded dynamically and unregister it
+    WSP_GGML_API void               wsp_ggml_backend_unload(wsp_ggml_backend_reg_t reg);
+    // Load all known backends from dynamic libraries
+    WSP_GGML_API void               wsp_ggml_backend_load_all(void);
+    WSP_GGML_API void               wsp_ggml_backend_load_all_from_path(const char * dir_path);
+
     //
     // Backend scheduler
     //
@@ -211,7 +248,7 @@ extern "C" {
         // preferrably to run on the same backend as the buffer
         wsp_ggml_backend_buffer_set_usage(buf_weights, WSP_GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
 
-        sched = wsp_ggml_backend_sched_new({backend_gpu, backend_gpu2, backend_cpu}, NULL, num_backends, WSP_GGML_DEFAULT_GRAPH_SIZE, false);
+        sched = wsp_ggml_backend_sched_new({backend_gpu, backend_gpu2, backend_cpu}, NULL, num_backends, WSP_GGML_DEFAULT_GRAPH_SIZE, false, true);
 
         // initialize buffers from a max size graph (optional)
         reserve_graph = build_graph(sched, max_batch_size);
@@ -223,14 +260,20 @@ extern "C" {
         wsp_ggml_backend_sched_reserve(sched, reserve_graph);
 
         // compute
-        graph = build_graph(sched);
-        wsp_ggml_backend_sched_graph_compute(sched, graph);
+        graph = build_graph(sched); // the graph and its tensors are single-use in terms of allocation, multi-use in terms of computation
+        for (int i = 0; i < 10; ++i) {
+            wsp_ggml_backend_sched_graph_compute(sched, graph); // on the first iteration the graph is allocated automatically
+        }
 
         // if there are graph inputs:
-        wsp_ggml_backend_sched_reset(sched);
-        wsp_ggml_backend_sched_alloc_graph(sched, graph);
-        wsp_ggml_backend_tensor_set(input_tensor, ...);
-        wsp_ggml_backend_sched_graph_compute(sched, graph);
+        graph = build_graph(sched); // get a new graph that is not allocated (the metadata for the old graph is freed once wsp_ggml_free is called)
+        wsp_ggml_backend_sched_reset(sched); // clear the allocation of the previous graph
+        wsp_ggml_backend_sched_alloc_graph(sched, graph); // explicitly allocate the new graph but do not execute it
+        wsp_ggml_backend_tensor_set(input_tensor, ...); // copy data to the newly allocated graph tensors
+        wsp_ggml_backend_sched_graph_compute(sched, graph); // execute the graph
+
+        // as an alternative to the above it is also possible to assign the inputs to a dedicated context and
+        // allocate them statically via wsp_ggml_backend_alloc_ctx_tensors
     }
     */
 
@@ -245,8 +288,8 @@ extern "C" {
     //
     typedef bool (*wsp_ggml_backend_sched_eval_callback)(struct wsp_ggml_tensor * t, bool ask, void * user_data);
 
-    // Initialize a backend scheduler
-    WSP_GGML_API wsp_ggml_backend_sched_t wsp_ggml_backend_sched_new(wsp_ggml_backend_t * backends, wsp_ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size, bool parallel);
+    // Initialize a backend scheduler, backends with low index are given priority over backends with high index
+    WSP_GGML_API wsp_ggml_backend_sched_t wsp_ggml_backend_sched_new(wsp_ggml_backend_t * backends, wsp_ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size, bool parallel, bool op_offload);
     WSP_GGML_API void                 wsp_ggml_backend_sched_free(wsp_ggml_backend_sched_t sched);
 
     // Initialize backend buffers from a measure graph
@@ -270,7 +313,9 @@ extern "C" {
     WSP_GGML_API enum wsp_ggml_status     wsp_ggml_backend_sched_graph_compute_async(wsp_ggml_backend_sched_t sched, struct wsp_ggml_cgraph * graph);
     WSP_GGML_API void                 wsp_ggml_backend_sched_synchronize(wsp_ggml_backend_sched_t sched);
 
-    // Reset all assignments and allocators - must be called before changing the node backends
+    // Reset all assignments and allocators - must be called before changing the node backends or allocating a new graph.
+    // This in effect deallocates all tensors that were previously allocated and leaves them with dangling pointers.
+    // The correct way to use this API is to discard the deallocated tensors and create new ones.
     WSP_GGML_API void                 wsp_ggml_backend_sched_reset(wsp_ggml_backend_sched_t sched);
 
     // Set a callback to be called for each resulting node during graph compute
@@ -297,30 +342,13 @@ extern "C" {
     WSP_GGML_API bool wsp_ggml_backend_compare_graph_backend(wsp_ggml_backend_t backend1, wsp_ggml_backend_t backend2, struct wsp_ggml_cgraph * graph, wsp_ggml_backend_eval_callback callback, void * user_data);
 
     // Tensor initialization
-    WSP_GGML_API void wsp_ggml_backend_tensor_alloc(wsp_ggml_backend_buffer_t buffer, struct wsp_ggml_tensor * tensor, void * addr);
-    WSP_GGML_API void wsp_ggml_backend_view_init(struct wsp_ggml_tensor * tensor);
+    WSP_GGML_API enum wsp_ggml_status wsp_ggml_backend_tensor_alloc(wsp_ggml_backend_buffer_t buffer, struct wsp_ggml_tensor * tensor, void * addr);
+    WSP_GGML_API enum wsp_ggml_status wsp_ggml_backend_view_init(struct wsp_ggml_tensor * tensor);
 
-    //
-    // CPU backend
-    //
-
-    WSP_GGML_API wsp_ggml_backend_t wsp_ggml_backend_cpu_init(void);
-
-    WSP_GGML_API bool wsp_ggml_backend_is_cpu                (wsp_ggml_backend_t backend);
-    WSP_GGML_API void wsp_ggml_backend_cpu_set_n_threads     (wsp_ggml_backend_t backend_cpu, int n_threads);
-    WSP_GGML_API void wsp_ggml_backend_cpu_set_threadpool    (wsp_ggml_backend_t backend_cpu, wsp_ggml_threadpool_t threadpool);
-    WSP_GGML_API void wsp_ggml_backend_cpu_set_abort_callback(wsp_ggml_backend_t backend_cpu, wsp_ggml_abort_callback abort_callback, void * abort_callback_data);
-
-    // Create a backend buffer from an existing pointer
+    // CPU buffer types are always available
     WSP_GGML_API wsp_ggml_backend_buffer_t      wsp_ggml_backend_cpu_buffer_from_ptr(void * ptr, size_t size);
     WSP_GGML_API wsp_ggml_backend_buffer_type_t wsp_ggml_backend_cpu_buffer_type(void);
 
-    WSP_GGML_API wsp_ggml_backend_reg_t wsp_ggml_backend_cpu_reg(void);
-
-#ifdef WSP_GGML_USE_CPU_HBM
-    WSP_GGML_API wsp_ggml_backend_buffer_type_t wsp_ggml_backend_cpu_hbm_buffer_type(void);
-#endif
-
 #ifdef  __cplusplus
 }
 #endif

package/cpp/ggml-common.h

@@ -6,7 +6,20 @@
 typedef uint16_t wsp_ggml_half;
 typedef uint32_t wsp_ggml_half2;
 
-#define WSP_GGML_COMMON_AGGR
+#define WSP_GGML_COMMON_AGGR_U
+#define WSP_GGML_COMMON_AGGR_S
+
+#define WSP_GGML_COMMON_DECL
+#elif defined(WSP_GGML_COMMON_DECL_CPP)
+#include <cstdint>
+
+typedef uint16_t wsp_ggml_half;
+typedef uint32_t wsp_ggml_half2;
+
+// std-c++ allow anonymous unions but some compiler warn on it
+#define WSP_GGML_COMMON_AGGR_U data
+// std-c++ do not allow it.
+#define WSP_GGML_COMMON_AGGR_S data
 
 #define WSP_GGML_COMMON_DECL
 #elif defined(WSP_GGML_COMMON_DECL_METAL)
@@ -15,7 +28,8 @@ typedef uint32_t wsp_ggml_half2;
 typedef half  wsp_ggml_half;
 typedef half2 wsp_ggml_half2;
 
-#define WSP_GGML_COMMON_AGGR
+#define WSP_GGML_COMMON_AGGR_U
+#define WSP_GGML_COMMON_AGGR_S
 
 #define WSP_GGML_COMMON_DECL
 #elif defined(WSP_GGML_COMMON_DECL_CUDA)
@@ -29,7 +43,8 @@ typedef half2 wsp_ggml_half2;
 typedef half  wsp_ggml_half;
 typedef half2 wsp_ggml_half2;
 
-#define WSP_GGML_COMMON_AGGR data
+#define WSP_GGML_COMMON_AGGR_U
+#define WSP_GGML_COMMON_AGGR_S data
 
 #define WSP_GGML_COMMON_DECL
 #elif defined(WSP_GGML_COMMON_DECL_HIP)
@@ -39,7 +54,8 @@ typedef half2 wsp_ggml_half2;
 typedef half  wsp_ggml_half;
 typedef half2 wsp_ggml_half2;
 
-#define WSP_GGML_COMMON_AGGR data
+#define WSP_GGML_COMMON_AGGR_U
+#define WSP_GGML_COMMON_AGGR_S data
 
 #define WSP_GGML_COMMON_DECL
 #elif defined(WSP_GGML_COMMON_DECL_SYCL)
@@ -49,7 +65,8 @@ typedef half2 wsp_ggml_half2;
 typedef sycl::half  wsp_ggml_half;
 typedef sycl::half2 wsp_ggml_half2;
 
-#define WSP_GGML_COMMON_AGGR data
+#define WSP_GGML_COMMON_AGGR_U
+#define WSP_GGML_COMMON_AGGR_S data
 
 #define WSP_GGML_COMMON_DECL
 #endif
@@ -141,6 +158,12 @@ typedef sycl::half2 wsp_ggml_half2;
 
 #endif // WSP_GGML_COMMON_DECL_CUDA || WSP_GGML_COMMON_DECL_HIP
 
+#ifdef _MSC_VER
+#define WSP_GGML_EXTENSION
+#else // _MSC_VER
+#define WSP_GGML_EXTENSION __extension__
+#endif // _MSC_VER
+
 #define QK4_0 32
 typedef struct {
     wsp_ggml_half d;           // delta
@@ -150,13 +173,13 @@ static_assert(sizeof(block_q4_0) == sizeof(wsp_ggml_half) + QK4_0 / 2, "wrong q4
 
 #define QK4_1 32
 typedef struct {
-    union {
+    WSP_GGML_EXTENSION union {
         struct {
             wsp_ggml_half d; // delta
             wsp_ggml_half m; // min
-        } WSP_GGML_COMMON_AGGR;
+        } WSP_GGML_COMMON_AGGR_S;
         wsp_ggml_half2 dm;
-    };
+    } WSP_GGML_COMMON_AGGR_U;
     uint8_t qs[QK4_1 / 2]; // nibbles / quants
 } block_q4_1;
 static_assert(sizeof(block_q4_1) == 2 * sizeof(wsp_ggml_half) + QK4_1 / 2, "wrong q4_1 block size/padding");
@@ -171,13 +194,13 @@ static_assert(sizeof(block_q5_0) == sizeof(wsp_ggml_half) + sizeof(uint32_t) + Q
 
 #define QK5_1 32
 typedef struct {
-    union {
+    WSP_GGML_EXTENSION union {
         struct {
             wsp_ggml_half d; // delta
             wsp_ggml_half m; // min
-        } WSP_GGML_COMMON_AGGR;
+        } WSP_GGML_COMMON_AGGR_S;
         wsp_ggml_half2 dm;
-    };
+    } WSP_GGML_COMMON_AGGR_U;
     uint8_t qh[4];         // 5-th bit of quants
     uint8_t qs[QK5_1 / 2]; // nibbles / quants
 } block_q5_1;
@@ -192,41 +215,17 @@ static_assert(sizeof(block_q8_0) == sizeof(wsp_ggml_half) + QK8_0, "wrong q8_0 b
 
 #define QK8_1 32
 typedef struct {
-    union {
+    WSP_GGML_EXTENSION union {
         struct {
             wsp_ggml_half d; // delta
             wsp_ggml_half s; // d * sum(qs[i])
-        } WSP_GGML_COMMON_AGGR;
+        } WSP_GGML_COMMON_AGGR_S;
         wsp_ggml_half2 ds;
-    };
+    } WSP_GGML_COMMON_AGGR_U;
     int8_t qs[QK8_1]; // quants
 } block_q8_1;
 static_assert(sizeof(block_q8_1) == 2*sizeof(wsp_ggml_half) + QK8_1, "wrong q8_1 block size/padding");
 
-typedef struct {
-    wsp_ggml_half d[4];        // deltas for 4 q4_0 blocks
-    uint8_t qs[QK4_0 * 2]; // nibbles / quants for 4 q4_0 blocks
-} block_q4_0x4;
-static_assert(sizeof(block_q4_0x4) == 4 * sizeof(wsp_ggml_half) + QK4_0 * 2, "wrong q4_0x4 block size/padding");
-
-typedef struct {
-    wsp_ggml_half d[8];        // deltas for 8 q4_0 blocks
-    uint8_t qs[QK4_0 * 4]; // nibbles / quants for 8 q4_0 blocks
-} block_q4_0x8;
-static_assert(sizeof(block_q4_0x8) == 8 * sizeof(wsp_ggml_half) + QK4_0 * 4, "wrong q4_0x8 block size/padding");
-
-typedef struct {
-    wsp_ggml_half d[4];        // deltas for 4 q8_0 blocks
-    int8_t qs[QK8_0 * 4];  // quants for 4 q8_0 blocks
-} block_q8_0x4;
-static_assert(sizeof(block_q8_0x4) == 4 * sizeof(wsp_ggml_half) + QK8_0 * 4, "wrong q8_0x4 block size/padding");
-
-typedef struct {
-    wsp_ggml_half d[8];        // deltas for 8 q8_0 blocks
-    int8_t qs[QK8_0 * 8];  // quants for 8 q8_0 blocks
-} block_q8_0x8;
-static_assert(sizeof(block_q8_0x8) == 8 * sizeof(wsp_ggml_half) + QK8_0 * 8, "wrong q8_0x8 block size/padding");
-
 //
 // Ternary quantization
 //
@@ -257,13 +256,13 @@ static_assert(sizeof(block_tq2_0) == sizeof(wsp_ggml_half) + QK_K / 4, "wrong tq
 typedef struct {
     uint8_t scales[QK_K/16]; // scales and mins, quantized with 4 bits
     uint8_t qs[QK_K/4];      // quants
-    union {
+    WSP_GGML_EXTENSION union {
         struct {
             wsp_ggml_half d;    // super-block scale for quantized scales
             wsp_ggml_half dmin; // super-block scale for quantized mins
-        } WSP_GGML_COMMON_AGGR;
+        } WSP_GGML_COMMON_AGGR_S;
         wsp_ggml_half2 dm;
-    };
+    } WSP_GGML_COMMON_AGGR_U;
 } block_q2_K;
 static_assert(sizeof(block_q2_K) == 2*sizeof(wsp_ggml_half) + QK_K/16 + QK_K/4, "wrong q2_K block size/padding");
 
@@ -284,13 +283,13 @@ static_assert(sizeof(block_q3_K) == sizeof(wsp_ggml_half) + QK_K / 4 + QK_K / 8
 // weight is represented as x = a * q + b
 // Effectively 4.5 bits per weight
 typedef struct {
-    union {
+    WSP_GGML_EXTENSION union {
         struct {
             wsp_ggml_half d;    // super-block scale for quantized scales
             wsp_ggml_half dmin; // super-block scale for quantized mins
-        } WSP_GGML_COMMON_AGGR;
+        } WSP_GGML_COMMON_AGGR_S;
         wsp_ggml_half2 dm;
-    };
+    } WSP_GGML_COMMON_AGGR_U;
     uint8_t scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits
     uint8_t qs[QK_K/2];           // 4--bit quants
 } block_q4_K;
@@ -301,13 +300,13 @@ static_assert(sizeof(block_q4_K) == 2*sizeof(wsp_ggml_half) + K_SCALE_SIZE + QK_
 // weight is represented as x = a * q + b
 // Effectively 5.5 bits per weight
 typedef struct {
-    union {
+    WSP_GGML_EXTENSION union {
         struct {
             wsp_ggml_half d;    // super-block scale for quantized scales
             wsp_ggml_half dmin; // super-block scale for quantized mins
-        } WSP_GGML_COMMON_AGGR;
+        } WSP_GGML_COMMON_AGGR_S;
         wsp_ggml_half2 dm;
-    };
+    } WSP_GGML_COMMON_AGGR_U;
     uint8_t scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits
     uint8_t qh[QK_K/8];           // quants, high bit
     uint8_t qs[QK_K/2];           // quants, low 4 bits
@@ -431,6 +430,13 @@ static_assert(sizeof(block_iq4_xs) == sizeof(wsp_ggml_half) + sizeof(uint16_t) +
 #define WSP_GGML_TABLE_BEGIN(type, name, size) static const type name[size] = {
 #define WSP_GGML_TABLE_END() };
 
+#define WSP_GGML_COMMON_IMPL
+#elif defined(WSP_GGML_COMMON_IMPL_CPP)
+#include <cstdint>
+
+#define WSP_GGML_TABLE_BEGIN(type, name, size) static const type name[size] = {
+#define WSP_GGML_TABLE_END() };
+
 #define WSP_GGML_COMMON_IMPL
 #elif defined(WSP_GGML_COMMON_IMPL_METAL)
 #include <metal_stdlib>
@@ -473,7 +479,6 @@ WSP_GGML_TABLE_BEGIN(uint8_t, ksigns_iq2xs, 128)
     240, 113, 114, 243, 116, 245, 246, 119, 120, 249, 250, 123, 252, 125, 126, 255,
 WSP_GGML_TABLE_END()
 
-//#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics
 WSP_GGML_TABLE_BEGIN(uint64_t, ksigns64, 128)
     0x0000000000000000, 0xff000000000000ff, 0xff0000000000ff00, 0x000000000000ffff,
     0xff00000000ff0000, 0x0000000000ff00ff, 0x0000000000ffff00, 0xff00000000ffffff,
@@ -508,7 +513,6 @@ WSP_GGML_TABLE_BEGIN(uint64_t, ksigns64, 128)
     0x00ffffffff000000, 0xffffffffff0000ff, 0xffffffffff00ff00, 0x00ffffffff00ffff,
     0xffffffffffff0000, 0x00ffffffffff00ff, 0x00ffffffffffff00, 0xffffffffffffffff,
 WSP_GGML_TABLE_END()
-//#endif
 
 
 WSP_GGML_TABLE_BEGIN(uint64_t, iq2xxs_grid, 256)
@@ -1070,6 +1074,10 @@ WSP_GGML_TABLE_BEGIN(uint32_t, iq3s_grid, 512)
     0x0f090307, 0x0f090501, 0x0f090b01, 0x0f0b0505, 0x0f0b0905, 0x0f0d0105, 0x0f0d0703, 0x0f0f0101,
 WSP_GGML_TABLE_END()
 
+WSP_GGML_TABLE_BEGIN(int8_t, kvalues_iq4nl, 16)
+    -127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113,
+WSP_GGML_TABLE_END()
+
 #define NGRID_IQ1S 2048
 #define IQ1S_DELTA 0.125f
 #define IQ1M_DELTA 0.125f

package/cpp/ggml-cpp.h

@@ -0,0 +1,39 @@
+#pragma once
+
+#ifndef __cplusplus
+#error "This header is for C++ only"
+#endif
+
+#include "ggml.h"
+#include "ggml-alloc.h"
+#include "ggml-backend.h"
+#include "gguf.h"
+#include <memory>
+
+// Smart pointers for ggml types
+
+// ggml
+
+struct wsp_ggml_context_deleter { void operator()(wsp_ggml_context * ctx) { wsp_ggml_free(ctx); } };
+struct wsp_gguf_context_deleter { void operator()(wsp_gguf_context * ctx) { wsp_gguf_free(ctx); } };
+
+typedef std::unique_ptr<wsp_ggml_context, wsp_ggml_context_deleter> wsp_ggml_context_ptr;
+typedef std::unique_ptr<wsp_gguf_context, wsp_gguf_context_deleter> wsp_gguf_context_ptr;
+
+// ggml-alloc
+
+struct wsp_ggml_gallocr_deleter { void operator()(wsp_ggml_gallocr_t galloc) { wsp_ggml_gallocr_free(galloc); } };
+
+typedef std::unique_ptr<wsp_ggml_gallocr, wsp_ggml_gallocr_deleter> wsp_ggml_gallocr_ptr;
+
+// ggml-backend
+
+struct wsp_ggml_backend_deleter        { void operator()(wsp_ggml_backend_t backend)       { wsp_ggml_backend_free(backend); } };
+struct wsp_ggml_backend_buffer_deleter { void operator()(wsp_ggml_backend_buffer_t buffer) { wsp_ggml_backend_buffer_free(buffer); } };
+struct wsp_ggml_backend_event_deleter  { void operator()(wsp_ggml_backend_event_t event)   { wsp_ggml_backend_event_free(event); } };
+struct wsp_ggml_backend_sched_deleter  { void operator()(wsp_ggml_backend_sched_t sched)   { wsp_ggml_backend_sched_free(sched); } };
+
+typedef std::unique_ptr<wsp_ggml_backend,        wsp_ggml_backend_deleter>        wsp_ggml_backend_ptr;
+typedef std::unique_ptr<wsp_ggml_backend_buffer, wsp_ggml_backend_buffer_deleter> wsp_ggml_backend_buffer_ptr;
+typedef std::unique_ptr<wsp_ggml_backend_event,  wsp_ggml_backend_event_deleter>  wsp_ggml_backend_event_ptr;
+typedef std::unique_ptr<wsp_ggml_backend_sched,  wsp_ggml_backend_sched_deleter>  wsp_ggml_backend_sched_ptr;