whisper.rn 0.4.0-rc.1 → 0.4.0-rc.10

package/cpp/ggml-alloc.h

@@ -6,20 +6,70 @@
 extern "C" {
 #endif
 
+typedef struct wsp_ggml_backend_buffer_type * wsp_ggml_backend_buffer_type_t;
+typedef struct      wsp_ggml_backend_buffer * wsp_ggml_backend_buffer_t;
+typedef struct             wsp_ggml_backend * wsp_ggml_backend_t;
 
-WSP_GGML_API struct wsp_ggml_allocr * wsp_ggml_allocr_new(void * data, size_t size, size_t alignment);
-WSP_GGML_API struct wsp_ggml_allocr * wsp_ggml_allocr_new_measure(size_t alignment);
+// Tensor allocator
+struct wsp_ggml_tallocr {
+    wsp_ggml_backend_buffer_t buffer;
+    void * base;
+    size_t alignment;
+    size_t offset;
+};
 
-// tell the allocator to parse nodes following the order described in the list
-// you should call this if your graph are optimized to execute out-of-order
-WSP_GGML_API void   wsp_ggml_allocr_set_parse_seq(struct wsp_ggml_allocr * alloc, const int * list, int n);
+WSP_GGML_API struct wsp_ggml_tallocr wsp_ggml_tallocr_new(wsp_ggml_backend_buffer_t buffer);
+WSP_GGML_API void                wsp_ggml_tallocr_alloc(struct wsp_ggml_tallocr * talloc, struct wsp_ggml_tensor * tensor);
 
-WSP_GGML_API void   wsp_ggml_allocr_free(struct wsp_ggml_allocr * alloc);
-WSP_GGML_API bool   wsp_ggml_allocr_is_measure(struct wsp_ggml_allocr * alloc);
-WSP_GGML_API void   wsp_ggml_allocr_reset(struct wsp_ggml_allocr * alloc);
-WSP_GGML_API void   wsp_ggml_allocr_alloc(struct wsp_ggml_allocr * alloc, struct wsp_ggml_tensor * tensor);
-WSP_GGML_API size_t wsp_ggml_allocr_alloc_graph(struct wsp_ggml_allocr * alloc, struct wsp_ggml_cgraph * graph);
+// Graph allocator
+/*
+  Example usage:
+    wsp_ggml_gallocr_t galloc = wsp_ggml_gallocr_new(wsp_ggml_backend_cpu_buffer_type());
 
+    // optional: create a worst-case graph and reserve the buffers to avoid reallocations
+    wsp_ggml_gallocr_reserve(galloc, build_graph(max_batch));
+
+    // allocate the graph
+    struct wsp_ggml_cgraph * graph = build_graph(batch);
+    wsp_ggml_gallocr_alloc_graph(galloc, graph);
+
+    printf("compute buffer size: %zu bytes\n", wsp_ggml_gallocr_get_buffer_size(galloc, 0));
+
+    // evaluate the graph
+    wsp_ggml_backend_graph_compute(backend, graph);
+*/
+
+// special tensor flags for use with the graph allocator:
+//   wsp_ggml_set_input(): all input tensors are allocated at the beginning of the graph in non-overlapping addresses
+//   wsp_ggml_set_output(): output tensors are never freed and never overwritten
+
+typedef struct wsp_ggml_gallocr * wsp_ggml_gallocr_t;
+
+WSP_GGML_API wsp_ggml_gallocr_t wsp_ggml_gallocr_new(wsp_ggml_backend_buffer_type_t buft);
+WSP_GGML_API wsp_ggml_gallocr_t wsp_ggml_gallocr_new_n(wsp_ggml_backend_buffer_type_t * bufts, int n_bufs);
+WSP_GGML_API void           wsp_ggml_gallocr_free(wsp_ggml_gallocr_t galloc);
+
+// pre-allocate buffers from a measure graph - does not allocate or modify the graph
+// call with a worst-case graph to avoid buffer reallocations
+// not strictly required for single buffer usage: wsp_ggml_gallocr_alloc_graph will reallocate the buffers automatically if needed
+// returns false if the buffer allocation failed
+WSP_GGML_API bool wsp_ggml_gallocr_reserve(wsp_ggml_gallocr_t galloc, struct wsp_ggml_cgraph * graph);
+WSP_GGML_API bool wsp_ggml_gallocr_reserve_n(
+    wsp_ggml_gallocr_t galloc,
+    struct wsp_ggml_cgraph * graph,
+    const int * node_buffer_ids,
+    const int * leaf_buffer_ids);
+
+// automatic reallocation if the topology changes when using a single buffer
+// returns false if using multiple buffers and a re-allocation is needed (call wsp_ggml_gallocr_reserve_n first to set the node buffers)
+WSP_GGML_API bool wsp_ggml_gallocr_alloc_graph(wsp_ggml_gallocr_t galloc, struct wsp_ggml_cgraph * graph);
+
+WSP_GGML_API size_t wsp_ggml_gallocr_get_buffer_size(wsp_ggml_gallocr_t galloc, int buffer_id);
+
+// Utils
+// Create a buffer and allocate all the tensors in a wsp_ggml_context
+WSP_GGML_API struct wsp_ggml_backend_buffer * wsp_ggml_backend_alloc_ctx_tensors_from_buft(struct wsp_ggml_context * ctx, wsp_ggml_backend_buffer_type_t buft);
+WSP_GGML_API struct wsp_ggml_backend_buffer * wsp_ggml_backend_alloc_ctx_tensors(struct wsp_ggml_context * ctx, wsp_ggml_backend_t backend);
 
 #ifdef  __cplusplus
 }

package/cpp/ggml-backend-impl.h

@@ -0,0 +1,216 @@
+#pragma once
+
+// ggml-backend internal header
+
+#include "ggml-backend.h"
+
+#ifdef  __cplusplus
+extern "C" {
+#endif
+
+    //
+    // Backend buffer type
+    //
+
+    struct wsp_ggml_backend_buffer_type_i {
+        const char *          (*get_name)      (wsp_ggml_backend_buffer_type_t buft);
+        // allocate a buffer of this type
+        wsp_ggml_backend_buffer_t (*alloc_buffer)  (wsp_ggml_backend_buffer_type_t buft, size_t size);
+        // tensor alignment
+        size_t                (*get_alignment) (wsp_ggml_backend_buffer_type_t buft);
+        // (optional) max buffer size that can be allocated (defaults to SIZE_MAX)
+        size_t                (*get_max_size)  (wsp_ggml_backend_buffer_type_t buft);
+        // (optional) data size needed to allocate the tensor, including padding (defaults to wsp_ggml_nbytes)
+        size_t                (*get_alloc_size)(wsp_ggml_backend_buffer_type_t buft, const struct wsp_ggml_tensor * tensor);
+        // (optional) check if tensor data is in host memory and uses standard ggml tensor layout (defaults to false)
+        bool                  (*is_host)       (wsp_ggml_backend_buffer_type_t buft);
+    };
+
+    struct wsp_ggml_backend_buffer_type {
+        struct wsp_ggml_backend_buffer_type_i  iface;
+        wsp_ggml_backend_dev_t device;
+        void * context;
+    };
+
+    //
+    // Backend buffer
+    //
+
+    struct wsp_ggml_backend_buffer_i {
+        // (optional) free the buffer
+        void         (*free_buffer)  (wsp_ggml_backend_buffer_t buffer);
+        // base address of the buffer
+        void *       (*get_base)     (wsp_ggml_backend_buffer_t buffer);
+        // (optional) initialize a tensor in the buffer (eg. add tensor extras)
+        void         (*init_tensor)  (wsp_ggml_backend_buffer_t buffer, struct wsp_ggml_tensor * tensor);
+        // tensor data access
+        void         (*memset_tensor)(wsp_ggml_backend_buffer_t buffer,       struct wsp_ggml_tensor * tensor,     uint8_t value, size_t offset, size_t size);
+        void         (*set_tensor)   (wsp_ggml_backend_buffer_t buffer,       struct wsp_ggml_tensor * tensor, const void * data, size_t offset, size_t size);
+        void         (*get_tensor)   (wsp_ggml_backend_buffer_t buffer, const struct wsp_ggml_tensor * tensor,       void * data, size_t offset, size_t size);
+        // (optional) tensor copy: dst is in the buffer, src may be in any buffer, including buffers from a different backend (return false if not supported)
+        bool         (*cpy_tensor)   (wsp_ggml_backend_buffer_t buffer, const struct wsp_ggml_tensor * src, struct wsp_ggml_tensor * dst);
+        // clear the entire buffer
+        void         (*clear)        (wsp_ggml_backend_buffer_t buffer, uint8_t value);
+        // (optional) reset any internal state due to tensor initialization, such as tensor extras
+        void         (*reset)        (wsp_ggml_backend_buffer_t buffer);
+    };
+
+    struct wsp_ggml_backend_buffer {
+        struct wsp_ggml_backend_buffer_i  iface;
+        wsp_ggml_backend_buffer_type_t    buft;
+        void * context;
+        size_t size;
+        enum wsp_ggml_backend_buffer_usage usage;
+    };
+
+    wsp_ggml_backend_buffer_t wsp_ggml_backend_buffer_init(
+                   wsp_ggml_backend_buffer_type_t buft,
+            struct wsp_ggml_backend_buffer_i      iface,
+                   void *                     context,
+                   size_t                     size);
+
+    // do not use directly, use wsp_ggml_backend_tensor_copy instead
+    bool wsp_ggml_backend_buffer_copy_tensor(const struct wsp_ggml_tensor * src, struct wsp_ggml_tensor * dst);
+
+    // multi-buffer
+    // buffer that contains a collection of buffers
+    wsp_ggml_backend_buffer_t wsp_ggml_backend_multi_buffer_alloc_buffer(wsp_ggml_backend_buffer_t * buffers, size_t n_buffers);
+    bool                  wsp_ggml_backend_buffer_is_multi_buffer(wsp_ggml_backend_buffer_t buffer);
+    void                  wsp_ggml_backend_multi_buffer_set_usage(wsp_ggml_backend_buffer_t buffer, enum wsp_ggml_backend_buffer_usage usage);
+
+    //
+    // Backend (stream)
+    //
+
+    struct wsp_ggml_backend_i {
+        const char * (*get_name)(wsp_ggml_backend_t backend);
+
+        void (*free)(wsp_ggml_backend_t backend);
+
+        // (optional) asynchronous tensor data access
+        void (*set_tensor_async)(wsp_ggml_backend_t backend,       struct wsp_ggml_tensor * tensor, const void * data, size_t offset, size_t size);
+        void (*get_tensor_async)(wsp_ggml_backend_t backend, const struct wsp_ggml_tensor * tensor,       void * data, size_t offset, size_t size);
+        bool (*cpy_tensor_async)(wsp_ggml_backend_t backend_src, wsp_ggml_backend_t backend_dst, const struct wsp_ggml_tensor * src, struct wsp_ggml_tensor * dst);
+
+        // (optional) complete all pending operations (required if the backend supports async operations)
+        void (*synchronize)(wsp_ggml_backend_t backend);
+
+        // (optional) graph plans (not used currently)
+        // compute graph with a plan
+        wsp_ggml_backend_graph_plan_t (*graph_plan_create) (wsp_ggml_backend_t backend, const struct wsp_ggml_cgraph * cgraph);
+        void                      (*graph_plan_free)   (wsp_ggml_backend_t backend, wsp_ggml_backend_graph_plan_t plan);
+        // update the plan with a new graph - this should be faster than creating a new plan when the graph has the same topology
+        void                      (*graph_plan_update) (wsp_ggml_backend_t backend, wsp_ggml_backend_graph_plan_t plan, const struct wsp_ggml_cgraph * cgraph);
+        // compute the graph with the plan
+        enum wsp_ggml_status          (*graph_plan_compute)(wsp_ggml_backend_t backend, wsp_ggml_backend_graph_plan_t plan);
+
+        // compute graph (always async if supported by the backend)
+        enum wsp_ggml_status          (*graph_compute)     (wsp_ggml_backend_t backend, struct wsp_ggml_cgraph * cgraph);
+
+        // (optional) event synchronization
+        // record an event on this stream
+        void (*event_record)(wsp_ggml_backend_t backend, wsp_ggml_backend_event_t event);
+        // wait for an event on on a different stream
+        void (*event_wait)  (wsp_ggml_backend_t backend, wsp_ggml_backend_event_t event);
+    };
+
+    struct wsp_ggml_backend {
+        wsp_ggml_guid_t guid;
+        struct wsp_ggml_backend_i iface;
+        wsp_ggml_backend_dev_t device;
+        void * context;
+    };
+
+    struct wsp_ggml_backend_event {
+        struct wsp_ggml_backend_device * device;
+        void * context;
+    };
+
+    //
+    // Backend device
+    //
+
+    // Note: if additional properties are needed, we should add a struct with all of them
+    //       the current functions to obtain the properties can remain, since they are more convenient for often used properties
+    struct wsp_ggml_backend_device_i {
+        // device name: short identifier for this device, such as "CPU" or "CUDA0"
+        const char * (*get_name)(wsp_ggml_backend_dev_t dev);
+
+        // device description: short informative description of the device, could be the model name
+        const char * (*get_description)(wsp_ggml_backend_dev_t dev);
+
+        // device memory in bytes
+        void         (*get_memory)(wsp_ggml_backend_dev_t dev, size_t * free, size_t * total);
+
+        // device type
+        enum wsp_ggml_backend_dev_type (*get_type)(wsp_ggml_backend_dev_t dev);
+
+        // device properties
+        void (*get_props)(wsp_ggml_backend_dev_t dev, struct wsp_ggml_backend_dev_props * props);
+
+        // backend (stream) initialization
+        wsp_ggml_backend_t (*init_backend)(wsp_ggml_backend_dev_t dev, const char * params);
+
+        // preferred buffer type
+        wsp_ggml_backend_buffer_type_t (*get_buffer_type)(wsp_ggml_backend_dev_t dev);
+
+        // (optional) host buffer type (in system memory, typically this is a pinned memory buffer for faster transfers between host and device)
+        wsp_ggml_backend_buffer_type_t (*get_host_buffer_type)(wsp_ggml_backend_dev_t dev);
+
+        // (optional) buffer from pointer: create a buffer from a host pointer (useful for memory mapped models and importing data from other libraries)
+        wsp_ggml_backend_buffer_t (*buffer_from_host_ptr)(wsp_ggml_backend_dev_t dev, void * ptr, size_t size, size_t max_tensor_size);
+
+        // check if the backend can compute an operation
+        bool (*supports_op)(wsp_ggml_backend_dev_t dev, const struct wsp_ggml_tensor * op);
+
+        // check if the backend can use tensors allocated in a buffer type
+        bool (*supports_buft)(wsp_ggml_backend_dev_t dev, wsp_ggml_backend_buffer_type_t buft);
+
+        // (optional) check if the backend wants to run an operation, even if the weights are allocated in an incompatible buffer
+        // these should be expensive operations that may benefit from running on this backend instead of the CPU backend
+        bool (*offload_op)(wsp_ggml_backend_dev_t dev, const struct wsp_ggml_tensor * op);
+
+        // (optional) event synchronization
+        wsp_ggml_backend_event_t (*event_new)         (wsp_ggml_backend_dev_t dev);
+        void                 (*event_free)        (wsp_ggml_backend_dev_t dev, wsp_ggml_backend_event_t event);
+        void                 (*event_synchronize) (wsp_ggml_backend_dev_t dev, wsp_ggml_backend_event_t event);
+    };
+
+    struct wsp_ggml_backend_device {
+        struct wsp_ggml_backend_device_i iface;
+        wsp_ggml_backend_reg_t reg;
+        void * context;
+    };
+
+    //
+    // Backend (reg)
+    //
+
+    struct wsp_ggml_backend_reg_i {
+        const char * (*get_name)(wsp_ggml_backend_reg_t reg);
+
+        // enumerate available devices
+        size_t             (*get_device_count)(wsp_ggml_backend_reg_t reg);
+        wsp_ggml_backend_dev_t (*get_device)(wsp_ggml_backend_reg_t reg, size_t index);
+
+        // (optional) get a pointer to a function in the backend
+        // backends can add custom functions that are not part of the standard ggml-backend interface
+        void * (*get_proc_address)(wsp_ggml_backend_reg_t reg, const char * name);
+    };
+
+    struct wsp_ggml_backend_reg {
+        // int api_version; // TODO: for dynamic loading
+        struct wsp_ggml_backend_reg_i iface;
+        void * context;
+    };
+
+
+    // Internal backend registry API
+    void wsp_ggml_backend_register(wsp_ggml_backend_reg_t reg);
+    void wsp_ggml_backend_device_register(wsp_ggml_backend_dev_t device);
+    // TODO: backends can be loaded as a dynamic library, in which case it needs to export this function
+    // typedef wsp_ggml_backend_register_t * (*wsp_ggml_backend_init)(void);
+
+#ifdef  __cplusplus
+}
+#endif

package/cpp/ggml-backend-reg.cpp

@@ -0,0 +1,204 @@
+#include "ggml-backend-impl.h"
+#include "ggml-backend.h"
+#include "ggml-cpu.h"
+#include "ggml-impl.h"
+#include <cstring>
+#include <vector>
+
+// Backend registry
+
+#ifdef WSP_GGML_USE_CUDA
+#include "ggml-cuda.h"
+#endif
+
+#ifdef WSP_GGML_USE_METAL
+#include <TargetConditionals.h>
+
+#if !TARGET_OS_SIMULATOR
+#include "ggml-metal.h"
+#endif
+
+#endif
+
+#ifdef WSP_GGML_USE_SYCL
+#include "ggml-sycl.h"
+#endif
+
+#ifdef WSP_GGML_USE_VULKAN
+#include "ggml-vulkan.h"
+#endif
+
+#ifdef WSP_GGML_USE_BLAS
+#include "ggml-blas.h"
+#endif
+
+#ifdef WSP_GGML_USE_RPC
+#include "ggml-rpc.h"
+#endif
+
+#ifdef WSP_GGML_USE_AMX
+#  include "ggml-amx.h"
+#endif
+
+#ifdef WSP_GGML_USE_CANN
+#include "ggml-cann.h"
+#endif
+
+#ifdef WSP_GGML_USE_KOMPUTE
+#include "ggml-kompute.h"
+#endif
+
+struct wsp_ggml_backend_registry {
+    std::vector<wsp_ggml_backend_reg_t> backends;
+    std::vector<wsp_ggml_backend_dev_t> devices;
+
+    wsp_ggml_backend_registry() {
+#ifdef WSP_GGML_USE_CUDA
+        register_backend(wsp_ggml_backend_cuda_reg());
+#endif
+#ifdef WSP_GGML_USE_METAL
+
+#if !TARGET_OS_SIMULATOR
+        register_backend(wsp_ggml_backend_metal_reg());
+#endif
+
+#endif
+#ifdef WSP_GGML_USE_SYCL
+        register_backend(wsp_ggml_backend_sycl_reg());
+#endif
+#ifdef WSP_GGML_USE_VULKAN
+        register_backend(wsp_ggml_backend_vk_reg());
+#endif
+#ifdef WSP_GGML_USE_CANN
+        register_backend(wsp_ggml_backend_cann_reg());
+#endif
+#ifdef WSP_GGML_USE_BLAS
+        register_backend(wsp_ggml_backend_blas_reg());
+#endif
+#ifdef WSP_GGML_USE_RPC
+        register_backend(wsp_ggml_backend_rpc_reg());
+#endif
+#ifdef WSP_GGML_USE_AMX
+        register_backend(wsp_ggml_backend_amx_reg());
+#endif
+#ifdef WSP_GGML_USE_KOMPUTE
+        register_backend(wsp_ggml_backend_kompute_reg());
+#endif
+
+        register_backend(wsp_ggml_backend_cpu_reg());
+    }
+
+    void register_backend(wsp_ggml_backend_reg_t reg) {
+        if (!reg) {
+            return;
+        }
+
+#ifndef NDEBUG
+        WSP_GGML_LOG_DEBUG("%s: registered backend %s (%zu devices)\n",
+            __func__, wsp_ggml_backend_reg_name(reg), wsp_ggml_backend_reg_dev_count(reg));
+#endif
+        backends.push_back(reg);
+        for (size_t i = 0; i < wsp_ggml_backend_reg_dev_count(reg); i++) {
+            register_device(wsp_ggml_backend_reg_dev_get(reg, i));
+        }
+    }
+
+    void register_device(wsp_ggml_backend_dev_t device) {
+#ifndef NDEBUG
+        WSP_GGML_LOG_DEBUG("%s: registered device %s (%s)\n", __func__, wsp_ggml_backend_dev_name(device), wsp_ggml_backend_dev_description(device));
+#endif
+        devices.push_back(device);
+    }
+};
+
+static wsp_ggml_backend_registry & get_reg() {
+    static wsp_ggml_backend_registry reg;
+    return reg;
+}
+
+// Internal API
+void wsp_ggml_backend_register(wsp_ggml_backend_reg_t reg) {
+    get_reg().register_backend(reg);
+}
+
+void wsp_ggml_backend_device_register(wsp_ggml_backend_dev_t device) {
+    get_reg().register_device(device);
+}
+
+// Backend (reg) enumeration
+size_t wsp_ggml_backend_reg_count() {
+    return get_reg().backends.size();
+}
+
+wsp_ggml_backend_reg_t wsp_ggml_backend_reg_get(size_t index) {
+    WSP_GGML_ASSERT(index < wsp_ggml_backend_reg_count());
+    return get_reg().backends[index];
+}
+
+wsp_ggml_backend_reg_t wsp_ggml_backend_reg_by_name(const char * name) {
+    for (size_t i = 0; i < wsp_ggml_backend_reg_count(); i++) {
+        wsp_ggml_backend_reg_t reg = wsp_ggml_backend_reg_get(i);
+        if (std::strcmp(wsp_ggml_backend_reg_name(reg), name) == 0) {
+            return reg;
+        }
+    }
+    return NULL;
+}
+
+// Device enumeration
+size_t wsp_ggml_backend_dev_count() {
+    return get_reg().devices.size();
+}
+
+wsp_ggml_backend_dev_t wsp_ggml_backend_dev_get(size_t index) {
+    WSP_GGML_ASSERT(index < wsp_ggml_backend_dev_count());
+    return get_reg().devices[index];
+}
+
+wsp_ggml_backend_dev_t wsp_ggml_backend_dev_by_name(const char * name) {
+    for (size_t i = 0; i < wsp_ggml_backend_dev_count(); i++) {
+        wsp_ggml_backend_dev_t dev = wsp_ggml_backend_dev_get(i);
+        if (strcmp(wsp_ggml_backend_dev_name(dev), name) == 0) {
+            return dev;
+        }
+    }
+    return NULL;
+}
+
+wsp_ggml_backend_dev_t wsp_ggml_backend_dev_by_type(enum wsp_ggml_backend_dev_type type) {
+    for (size_t i = 0; i < wsp_ggml_backend_dev_count(); i++) {
+        wsp_ggml_backend_dev_t dev = wsp_ggml_backend_dev_get(i);
+        if (wsp_ggml_backend_dev_type(dev) == type) {
+            return dev;
+        }
+    }
+    return NULL;
+}
+
+// Convenience functions
+wsp_ggml_backend_t wsp_ggml_backend_init_by_name(const char * name, const char * params) {
+    wsp_ggml_backend_dev_t dev = wsp_ggml_backend_dev_by_name(name);
+    if (!dev) {
+        return NULL;
+    }
+    return wsp_ggml_backend_dev_init(dev, params);
+}
+
+wsp_ggml_backend_t wsp_ggml_backend_init_by_type(enum wsp_ggml_backend_dev_type type, const char * params) {
+    wsp_ggml_backend_dev_t dev = wsp_ggml_backend_dev_by_type(type);
+    if (!dev) {
+        return NULL;
+    }
+    return wsp_ggml_backend_dev_init(dev, params);
+}
+
+wsp_ggml_backend_t wsp_ggml_backend_init_best(void) {
+    wsp_ggml_backend_dev_t dev = wsp_ggml_backend_dev_by_type(WSP_GGML_BACKEND_DEVICE_TYPE_GPU);
+    if (!dev) {
+        dev = wsp_ggml_backend_dev_by_type(WSP_GGML_BACKEND_DEVICE_TYPE_CPU);
+    }
+    if (!dev) {
+        return NULL;
+    }
+    return wsp_ggml_backend_dev_init(dev, NULL);
+}