whisper.rn 0.4.0-rc.7 → 0.4.0-rc.9

package/cpp/ggml-impl.h

@@ -1,23 +1,32 @@
 #pragma once
 
-#include "ggml.h"
-
 // GGML internal header
 
+#include "ggml.h"
+
 #include <assert.h>
-#include <stddef.h>
+#include <stdlib.h> // load `stdlib.h` before other headers to work around MinGW bug: https://sourceforge.net/p/mingw-w64/bugs/192/
 #include <stdbool.h>
-#include <string.h> // memcpy
-#include <math.h>   // fabsf
+#include <stdint.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+#undef MIN
+#undef MAX
+
+#define MIN(a, b) ((a) < (b) ? (a) : (b))
+#define MAX(a, b) ((a) > (b) ? (a) : (b))
+
+// required for mmap as gguf only guarantees 32-byte alignment
+#define TENSOR_ALIGNMENT 32
+
 // static_assert should be a #define, but if it's not,
 // fall back to the _Static_assert C11 keyword.
 // if C99 - static_assert is noop
 // ref: https://stackoverflow.com/a/53923785/4039976
+#ifndef __cplusplus
 #ifndef static_assert
 #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201100L)
 #define static_assert(cond, msg) _Static_assert(cond, msg)
@@ -25,218 +34,175 @@ extern "C" {
 #define static_assert(cond, msg) struct global_scope_noop_trick
 #endif
 #endif
-
-// __FMA__ and __F16C__ are not defined in MSVC, however they are implied with AVX2/AVX512
-#if defined(_MSC_VER) && (defined(__AVX2__) || defined(__AVX512F__))
-#ifndef __FMA__
-#define __FMA__
-#endif
-#ifndef __F16C__
-#define __F16C__
-#endif
-#ifndef __SSE3__
-#define __SSE3__
-#endif
 #endif
 
-// 16-bit float
-// on Arm, we use __fp16
-// on x86, we use uint16_t
-#if defined(__ARM_NEON) && !defined(_MSC_VER)
-
-// if YCM cannot find <arm_neon.h>, make a symbolic link to it, for example:
 //
-//   $ ln -sfn /Library/Developer/CommandLineTools/usr/lib/clang/13.1.6/include/arm_neon.h ./src/
+// logging
 //
-#include <arm_neon.h>
 
-#define WSP_GGML_COMPUTE_FP16_TO_FP32(x) ((float) (x))
-#define WSP_GGML_COMPUTE_FP32_TO_FP16(x) (x)
+WSP_GGML_ATTRIBUTE_FORMAT(2, 3)
+void wsp_ggml_log_internal        (enum wsp_ggml_log_level level, const char * format, ...);
+void wsp_ggml_log_callback_default(enum wsp_ggml_log_level level, const char * text, void * user_data);
 
-#define WSP_GGML_FP16_TO_FP32(x) ((float) (x))
-#define WSP_GGML_FP32_TO_FP16(x) (x)
+#define WSP_GGML_LOG(...)       wsp_ggml_log_internal(WSP_GGML_LOG_LEVEL_NONE , __VA_ARGS__)
+#define WSP_GGML_LOG_INFO(...)  wsp_ggml_log_internal(WSP_GGML_LOG_LEVEL_INFO , __VA_ARGS__)
+#define WSP_GGML_LOG_WARN(...)  wsp_ggml_log_internal(WSP_GGML_LOG_LEVEL_WARN , __VA_ARGS__)
+#define WSP_GGML_LOG_ERROR(...) wsp_ggml_log_internal(WSP_GGML_LOG_LEVEL_ERROR, __VA_ARGS__)
+#define WSP_GGML_LOG_DEBUG(...) wsp_ggml_log_internal(WSP_GGML_LOG_LEVEL_DEBUG, __VA_ARGS__)
+#define WSP_GGML_LOG_CONT(...)  wsp_ggml_log_internal(WSP_GGML_LOG_LEVEL_CONT , __VA_ARGS__)
 
-#else
+// bitset
 
-#ifdef __wasm_simd128__
-#include <wasm_simd128.h>
-#else
-#ifdef __POWER9_VECTOR__
-#include <altivec.h>
-#undef bool
-#define bool _Bool
-#else
-#if defined(_MSC_VER) || defined(__MINGW32__)
-#include <intrin.h>
-#else
-#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__) || defined(__SSSE3__) || defined(__SSE3__)
-#if !defined(__riscv)
-#include <immintrin.h>
-#endif
-#endif
-#endif
-#endif
-#endif
+typedef uint32_t wsp_ggml_bitset_t;
 
-#ifdef __riscv_v_intrinsic
-#include <riscv_vector.h>
-#endif
+static_assert(sizeof(wsp_ggml_bitset_t) == 4, "bitset_t constants must be updated");
+#define BITSET_SHR 5 // log2(sizeof(wsp_ggml_bitset_t)*8)
+#define BITSET_MASK (sizeof(wsp_ggml_bitset_t)*8 - 1)
 
-#ifdef __F16C__
+static size_t wsp_ggml_bitset_size(size_t n) {
+    return (n + BITSET_MASK) >> BITSET_SHR;
+}
 
-#ifdef _MSC_VER
-#define WSP_GGML_COMPUTE_FP16_TO_FP32(x) _mm_cvtss_f32(_mm_cvtph_ps(_mm_cvtsi32_si128(x)))
-#define WSP_GGML_COMPUTE_FP32_TO_FP16(x) _mm_extract_epi16(_mm_cvtps_ph(_mm_set_ss(x), 0), 0)
-#else
-#define WSP_GGML_COMPUTE_FP16_TO_FP32(x) _cvtsh_ss(x)
-#define WSP_GGML_COMPUTE_FP32_TO_FP16(x) _cvtss_sh(x, 0)
-#endif
+static inline bool wsp_ggml_bitset_get(const wsp_ggml_bitset_t * bitset, size_t i) {
+    return !!(bitset[i >> BITSET_SHR] & (1u << (i & BITSET_MASK)));
+}
 
-#elif defined(__POWER9_VECTOR__)
-
-#define WSP_GGML_COMPUTE_FP16_TO_FP32(x) wsp_ggml_compute_fp16_to_fp32(x)
-#define WSP_GGML_COMPUTE_FP32_TO_FP16(x) wsp_ggml_compute_fp32_to_fp16(x)
-/* the inline asm below is about 12% faster than the lookup method */
-#define WSP_GGML_FP16_TO_FP32(x) WSP_GGML_COMPUTE_FP16_TO_FP32(x)
-#define WSP_GGML_FP32_TO_FP16(x) WSP_GGML_COMPUTE_FP32_TO_FP16(x)
-
-static inline float wsp_ggml_compute_fp16_to_fp32(wsp_ggml_fp16_t h) {
-    register float f;
-    register double d;
-    __asm__(
-        "mtfprd %0,%2\n"
-        "xscvhpdp %0,%0\n"
-        "frsp %1,%0\n" :
-        /* temp */ "=d"(d),
-        /* out */  "=f"(f):
-        /* in */   "r"(h));
-    return f;
+static inline void wsp_ggml_bitset_set(wsp_ggml_bitset_t * bitset, size_t i) {
+    bitset[i >> BITSET_SHR] |= (1u << (i & BITSET_MASK));
 }
 
-static inline wsp_ggml_fp16_t wsp_ggml_compute_fp32_to_fp16(float f) {
-    register double d;
-    register wsp_ggml_fp16_t r;
-    __asm__( /* xscvdphp can work on double or single precision */
-        "xscvdphp %0,%2\n"
-        "mffprd %1,%0\n" :
-        /* temp */ "=d"(d),
-        /* out */  "=r"(r):
-        /* in */   "f"(f));
-    return r;
+static inline void wsp_ggml_bitset_clear(wsp_ggml_bitset_t * bitset, size_t i) {
+    bitset[i >> BITSET_SHR] &= ~(1u << (i & BITSET_MASK));
 }
 
-#else
+// hash set
 
-// FP16 <-> FP32
-// ref: https://github.com/Maratyszcza/FP16
+#define WSP_GGML_HASHSET_FULL ((size_t)-1)
+#define WSP_GGML_HASHSET_ALREADY_EXISTS ((size_t)-2)
 
-static inline float fp32_from_bits(uint32_t w) {
-    union {
-        uint32_t as_bits;
-        float as_value;
-    } fp32;
-    fp32.as_bits = w;
-    return fp32.as_value;
-}
+struct wsp_ggml_hash_set {
+    size_t size;
+    wsp_ggml_bitset_t * used;       // whether or not the keys are in use i.e. set
+    struct wsp_ggml_tensor ** keys; // actual tensors in the set, keys[i] is only defined if wsp_ggml_bitset_get(used, i)
+};
 
-static inline uint32_t fp32_to_bits(float f) {
-    union {
-        float as_value;
-        uint32_t as_bits;
-    } fp32;
-    fp32.as_value = f;
-    return fp32.as_bits;
-}
+struct wsp_ggml_hash_set wsp_ggml_hash_set_new(size_t size);
+void                 wsp_ggml_hash_set_free(struct wsp_ggml_hash_set * hash_set);
 
-static inline float wsp_ggml_compute_fp16_to_fp32(wsp_ggml_fp16_t h) {
-    const uint32_t w = (uint32_t) h << 16;
-    const uint32_t sign = w & UINT32_C(0x80000000);
-    const uint32_t two_w = w + w;
+// returns the minimum size for a hash set that can hold min_sz elements
+size_t wsp_ggml_hash_size(size_t min_sz);
 
-    const uint32_t exp_offset = UINT32_C(0xE0) << 23;
-#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) || defined(__GNUC__) && !defined(__STRICT_ANSI__)
-    const float exp_scale = 0x1.0p-112f;
-#else
-    const float exp_scale = fp32_from_bits(UINT32_C(0x7800000));
-#endif
-    const float normalized_value = fp32_from_bits((two_w >> 4) + exp_offset) * exp_scale;
+// remove all elements from the hash set
+void wsp_ggml_hash_set_reset(struct wsp_ggml_hash_set * hash_set);
 
-    const uint32_t magic_mask = UINT32_C(126) << 23;
-    const float magic_bias = 0.5f;
-    const float denormalized_value = fp32_from_bits((two_w >> 17) | magic_mask) - magic_bias;
+// returns true if key is in the hash set
+static bool wsp_ggml_hash_contains(const struct wsp_ggml_hash_set * hash_set, struct wsp_ggml_tensor * key);
 
-    const uint32_t denormalized_cutoff = UINT32_C(1) << 27;
-    const uint32_t result = sign |
-        (two_w < denormalized_cutoff ? fp32_to_bits(denormalized_value) : fp32_to_bits(normalized_value));
-    return fp32_from_bits(result);
-}
+// returns WSP_GGML_HASHSET_FULL if table is full, otherwise the current index of the key or where it should be inserted
+static size_t wsp_ggml_hash_find(const struct wsp_ggml_hash_set * hash_set, struct wsp_ggml_tensor * key);
 
-static inline wsp_ggml_fp16_t wsp_ggml_compute_fp32_to_fp16(float f) {
-#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) || defined(__GNUC__) && !defined(__STRICT_ANSI__)
-    const float scale_to_inf = 0x1.0p+112f;
-    const float scale_to_zero = 0x1.0p-110f;
-#else
-    const float scale_to_inf = fp32_from_bits(UINT32_C(0x77800000));
-    const float scale_to_zero = fp32_from_bits(UINT32_C(0x08800000));
-#endif
-    float base = (fabsf(f) * scale_to_inf) * scale_to_zero;
-
-    const uint32_t w = fp32_to_bits(f);
-    const uint32_t shl1_w = w + w;
-    const uint32_t sign = w & UINT32_C(0x80000000);
-    uint32_t bias = shl1_w & UINT32_C(0xFF000000);
-    if (bias < UINT32_C(0x71000000)) {
-        bias = UINT32_C(0x71000000);
-    }
+// returns WSP_GGML_HASHSET_ALREADY_EXISTS if key already exists, index otherwise, asserts if table is full
+static size_t wsp_ggml_hash_insert(struct wsp_ggml_hash_set * hash_set, struct wsp_ggml_tensor * key);
+
+// return index, asserts if table is full
+static size_t wsp_ggml_hash_find_or_insert(struct wsp_ggml_hash_set * hash_set, struct wsp_ggml_tensor * key);
 
-    base = fp32_from_bits((bias >> 1) + UINT32_C(0x07800000)) + base;
-    const uint32_t bits = fp32_to_bits(base);
-    const uint32_t exp_bits = (bits >> 13) & UINT32_C(0x00007C00);
-    const uint32_t mantissa_bits = bits & UINT32_C(0x00000FFF);
-    const uint32_t nonsign = exp_bits + mantissa_bits;
-    return (sign >> 16) | (shl1_w > UINT32_C(0xFF000000) ? UINT16_C(0x7E00) : nonsign);
+// hash function for wsp_ggml_tensor
+static inline size_t wsp_ggml_hash(const struct wsp_ggml_tensor * p) {
+    // the last 4 bits are always zero due to alignment
+    return (size_t)(uintptr_t)p >> 4;
 }
 
-#define WSP_GGML_COMPUTE_FP16_TO_FP32(x) wsp_ggml_compute_fp16_to_fp32(x)
-#define WSP_GGML_COMPUTE_FP32_TO_FP16(x) wsp_ggml_compute_fp32_to_fp16(x)
+static size_t wsp_ggml_hash_find(const struct wsp_ggml_hash_set * hash_set, struct wsp_ggml_tensor * key) {
+    size_t h = wsp_ggml_hash(key) % hash_set->size;
+
+    // linear probing
+    size_t i = h;
+    while (wsp_ggml_bitset_get(hash_set->used, i) && hash_set->keys[i] != key) {
+        i = (i + 1) % hash_set->size;
+        if (i == h) {
+            // visited all hash table entries -> not found
+            return WSP_GGML_HASHSET_FULL;
+        }
+    }
+    return i;
+}
 
-#endif // __F16C__
+static bool wsp_ggml_hash_contains(const struct wsp_ggml_hash_set * hash_set, struct wsp_ggml_tensor * key) {
+    size_t i = wsp_ggml_hash_find(hash_set, key);
+    return i != WSP_GGML_HASHSET_FULL && wsp_ggml_bitset_get(hash_set->used, i);
+}
 
-#endif // __ARM_NEON
+static size_t wsp_ggml_hash_insert(struct wsp_ggml_hash_set * hash_set, struct wsp_ggml_tensor * key) {
+    size_t h = wsp_ggml_hash(key) % hash_set->size;
+
+    // linear probing
+    size_t i = h;
+    do {
+        if (!wsp_ggml_bitset_get(hash_set->used, i)) {
+            wsp_ggml_bitset_set(hash_set->used, i);
+            hash_set->keys[i] = key;
+            return i;
+        }
+        if (hash_set->keys[i] == key) {
+            return WSP_GGML_HASHSET_ALREADY_EXISTS;
+        }
+        i = (i + 1) % hash_set->size;
+    } while (i != h);
+
+    // visited all hash table entries -> not found
+    WSP_GGML_ABORT("fatal error");
+}
 
-// precomputed f32 table for f16 (256 KB)
-// defined in ggml.c, initialized in wsp_ggml_init()
-extern float wsp_ggml_table_f32_f16[1 << 16];
+static size_t wsp_ggml_hash_find_or_insert(struct wsp_ggml_hash_set * hash_set, struct wsp_ggml_tensor * key) {
+    size_t h = wsp_ggml_hash(key) % hash_set->size;
+
+    // linear probing
+    size_t i = h;
+    do {
+        if (!wsp_ggml_bitset_get(hash_set->used, i)) {
+            wsp_ggml_bitset_set(hash_set->used, i);
+            hash_set->keys[i] = key;
+            return i;
+        }
+        if (hash_set->keys[i] == key) {
+            return i;
+        }
+        i = (i + 1) % hash_set->size;
+    } while (i != h);
+
+    // visited all hash table entries -> not found
+    WSP_GGML_ABORT("fatal error");
+}
 
-// On ARM NEON, it's quicker to directly convert x -> x instead of calling into wsp_ggml_lookup_fp16_to_fp32,
-// so we define WSP_GGML_FP16_TO_FP32 and WSP_GGML_FP32_TO_FP16 elsewhere for NEON.
-// This is also true for POWER9.
-#if !defined(WSP_GGML_FP16_TO_FP32) || !defined(WSP_GGML_FP32_TO_FP16)
+// computation graph
 
-inline static float wsp_ggml_lookup_fp16_to_fp32(wsp_ggml_fp16_t f) {
-    uint16_t s;
-    memcpy(&s, &f, sizeof(uint16_t));
-    return wsp_ggml_table_f32_f16[s];
-}
+enum wsp_ggml_cgraph_eval_order {
+    WSP_GGML_CGRAPH_EVAL_ORDER_LEFT_TO_RIGHT = 0,
+    WSP_GGML_CGRAPH_EVAL_ORDER_RIGHT_TO_LEFT,
+    WSP_GGML_CGRAPH_EVAL_ORDER_COUNT
+};
 
-#define WSP_GGML_FP16_TO_FP32(x) wsp_ggml_lookup_fp16_to_fp32(x)
-#define WSP_GGML_FP32_TO_FP16(x) WSP_GGML_COMPUTE_FP32_TO_FP16(x)
+struct wsp_ggml_cgraph {
+    int size;
+    int n_nodes;
+    int n_leafs;
 
-#endif
+    struct wsp_ggml_tensor ** nodes;
+    struct wsp_ggml_tensor ** grads;
+    struct wsp_ggml_tensor ** leafs;
 
-#define WSP_GGML_HASHTABLE_FULL ((size_t)-1)
-#define WSP_GGML_HASHTABLE_ALREADY_EXISTS ((size_t)-2)
+    struct wsp_ggml_hash_set visited_hash_set;
 
-bool   wsp_ggml_hash_contains      (const struct wsp_ggml_hash_set hash_set, struct wsp_ggml_tensor * key);
+    enum wsp_ggml_cgraph_eval_order order;
+};
 
-// returns WSP_GGML_HASHTABLE_FULL if table is full, otherwise the current index of the key or where it should be inserted
-size_t wsp_ggml_hash_find          (const struct wsp_ggml_hash_set hash_set, struct wsp_ggml_tensor * key);
+struct wsp_ggml_cgraph wsp_ggml_graph_view(struct wsp_ggml_cgraph * cgraph, int i0, int i1);
 
-// returns WSP_GGML_HASHTABLE_ALREADY_EXISTS if key already exists, index otherwise, asserts if table is full
-size_t wsp_ggml_hash_insert        (      struct wsp_ggml_hash_set hash_set, struct wsp_ggml_tensor * key);
+// Memory allocation
 
-// return index, asserts if table is full
-size_t wsp_ggml_hash_find_or_insert(      struct wsp_ggml_hash_set hash_set, struct wsp_ggml_tensor * key);
+void * wsp_ggml_aligned_malloc(size_t size);
+void wsp_ggml_aligned_free(void * ptr, size_t size);
 
 #ifdef __cplusplus
 }

package/cpp/ggml-metal.h

@@ -1,7 +1,9 @@
+// Note: this description is outdated
+//
 // An interface allowing to compute wsp_ggml_cgraph with Metal
 //
 // This is a fully functional interface that extends ggml with GPU support for Apple devices.
-// A similar interface can be created for other GPU backends (e.g. Vulkan, CUDA, OpenCL, etc.)
+// A similar interface can be created for other GPU backends (e.g. Vulkan, CUDA, etc.)
 //
 // How it works?
 //
@@ -25,10 +27,6 @@
 #include <stddef.h>
 #include <stdbool.h>
 
-// max memory buffers that can be mapped to the device
-#define WSP_GGML_METAL_MAX_BUFFERS 64
-#define WSP_GGML_METAL_MAX_COMMAND_BUFFERS 32
-
 struct wsp_ggml_tensor;
 struct wsp_ggml_cgraph;
 
@@ -36,59 +34,6 @@ struct wsp_ggml_cgraph;
 extern "C" {
 #endif
 
-//
-// internal API
-// temporary exposed to user-code
-//
-
-struct wsp_ggml_metal_context;
-
-void wsp_ggml_metal_log_set_callback(wsp_ggml_log_callback log_callback, void * user_data);
-
-// number of command buffers to use
-struct wsp_ggml_metal_context * wsp_ggml_metal_init(int n_cb);
-void wsp_ggml_metal_free(struct wsp_ggml_metal_context * ctx);
-
-void * wsp_ggml_metal_host_malloc(size_t n);
-void   wsp_ggml_metal_host_free  (void * data);
-
-// set the number of command buffers to use
-void wsp_ggml_metal_set_n_cb(struct wsp_ggml_metal_context * ctx, int n_cb);
-
-// creates a mapping between a host memory buffer and a device memory buffer
-// - make sure to map all buffers used in the graph before calling wsp_ggml_metal_graph_compute
-// - the mapping is used during computation to determine the arguments of the compute kernels
-// - you don't need to keep the host memory buffer allocated as it is never accessed by Metal
-// - max_size specifies the maximum size of a tensor and is used to create shared views such
-//   that it is guaranteed that the tensor will fit in at least one of the views
-//
-bool wsp_ggml_metal_add_buffer(
-        struct wsp_ggml_metal_context * ctx,
-                       const char * name,
-                             void * data,
-                           size_t   size,
-                           size_t   max_size);
-
-// set data from host memory into the device
-void wsp_ggml_metal_set_tensor(struct wsp_ggml_metal_context * ctx, struct wsp_ggml_tensor * t);
-
-// get data from the device into host memory
-void wsp_ggml_metal_get_tensor(struct wsp_ggml_metal_context * ctx, struct wsp_ggml_tensor * t);
-
-// try to find operations that can be run concurrently in the graph
-// you should run it again if the topology of your graph changes
-void wsp_ggml_metal_graph_find_concurrency(struct wsp_ggml_metal_context * ctx, struct wsp_ggml_cgraph * gf, bool check_mem);
-
-// if the graph has been optimized for concurrently dispatch, return length of the concur_list if optimized
-int wsp_ggml_metal_if_optimized(struct wsp_ggml_metal_context * ctx);
-
-// output the concur_list for wsp_ggml_alloc
-int * wsp_ggml_metal_get_concur_list(struct wsp_ggml_metal_context * ctx);
-
-// same as wsp_ggml_graph_compute but uses Metal
-// creates gf->n_threads command buffers in parallel
-void wsp_ggml_metal_graph_compute(struct wsp_ggml_metal_context * ctx, struct wsp_ggml_cgraph * gf);
-
 //
 // backend API
 // user-code should use only these functions
@@ -98,7 +43,12 @@ WSP_GGML_API wsp_ggml_backend_t wsp_ggml_backend_metal_init(void);
 
 WSP_GGML_API bool wsp_ggml_backend_is_metal(wsp_ggml_backend_t backend);
 
-WSP_GGML_API void wsp_ggml_backend_metal_set_n_cb(wsp_ggml_backend_t backend, int n_cb);
+WSP_GGML_DEPRECATED(
+        WSP_GGML_API wsp_ggml_backend_buffer_t wsp_ggml_backend_metal_buffer_from_ptr(void * data, size_t size, size_t max_size),
+        "obsoleted by the new device interface - https://github.com/ggerganov/llama.cpp/pull/9713");
+
+WSP_GGML_API void wsp_ggml_backend_metal_set_abort_callback(wsp_ggml_backend_t backend, wsp_ggml_abort_callback abort_callback, void * user_data);
+
 WSP_GGML_API wsp_ggml_backend_buffer_type_t wsp_ggml_backend_metal_buffer_type(void);
 
 // helper to check if the device supports a specific family
@@ -106,7 +56,11 @@ WSP_GGML_API wsp_ggml_backend_buffer_type_t wsp_ggml_backend_metal_buffer_type(v
 // ref: https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf
 WSP_GGML_API bool wsp_ggml_backend_metal_supports_family(wsp_ggml_backend_t backend, int family);
 
+// capture all command buffers committed the next time `wsp_ggml_backend_graph_compute` is called
+WSP_GGML_API void wsp_ggml_backend_metal_capture_next_compute(wsp_ggml_backend_t backend);
+
+WSP_GGML_API wsp_ggml_backend_reg_t wsp_ggml_backend_metal_reg(void);
+
 #ifdef __cplusplus
 }
 #endif
-