whisper.rn 0.5.2 → 0.5.4

package/cpp/ggml.c

@@ -943,6 +943,7 @@ static const char * WSP_GGML_OP_NAME[WSP_GGML_OP_COUNT] = {
     "COS",
     "SUM",
     "SUM_ROWS",
+    "CUMSUM",
     "MEAN",
     "ARGMAX",
     "COUNT_EQUAL",
@@ -998,6 +999,8 @@ static const char * WSP_GGML_OP_NAME[WSP_GGML_OP_COUNT] = {
     "TIMESTEP_EMBEDDING",
     "ARGSORT",
     "LEAKY_RELU",
+    "TRI",
+    "FILL",
 
     "FLASH_ATTN_EXT",
     "FLASH_ATTN_BACK",
@@ -1010,6 +1013,7 @@ static const char * WSP_GGML_OP_NAME[WSP_GGML_OP_COUNT] = {
     "RWKV_WKV6",
     "GATED_LINEAR_ATTN",
     "RWKV_WKV7",
+    "SOLVE_TRI",
 
     "UNARY",
 
@@ -1027,7 +1031,7 @@ static const char * WSP_GGML_OP_NAME[WSP_GGML_OP_COUNT] = {
     "GLU",
 };
 
-static_assert(WSP_GGML_OP_COUNT == 90, "WSP_GGML_OP_COUNT != 90");
+static_assert(WSP_GGML_OP_COUNT == 94, "WSP_GGML_OP_COUNT != 94");
 
 static const char * WSP_GGML_OP_SYMBOL[WSP_GGML_OP_COUNT] = {
     "none",
@@ -1047,6 +1051,7 @@ static const char * WSP_GGML_OP_SYMBOL[WSP_GGML_OP_COUNT] = {
     "cos(x)",
     "Σx",
     "Σx_k",
+    "cumsum(x)",
     "Σx/n",
     "argmax(x)",
     "count_equal(x)",
@@ -1102,6 +1107,8 @@ static const char * WSP_GGML_OP_SYMBOL[WSP_GGML_OP_COUNT] = {
     "timestep_embedding(timesteps, dim, max_period)",
     "argsort(x)",
     "leaky_relu(x)",
+    "tri(x)",
+    "fill(x, c)",
 
     "flash_attn_ext(x)",
     "flash_attn_back(x)",
@@ -1114,6 +1121,7 @@ static const char * WSP_GGML_OP_SYMBOL[WSP_GGML_OP_COUNT] = {
     "rwkv_wkv6(k, v, r, tf, td, s)",
     "gated_linear_attn(k, v, q, gate, s)",
     "rwkv_wkv7(r, w, k, v, a, b, s)",
+    "A X = B, A triangular, solve X",
 
     "unary(x)",
 
@@ -1131,7 +1139,7 @@ static const char * WSP_GGML_OP_SYMBOL[WSP_GGML_OP_COUNT] = {
     "glu(x)",
 };
 
-static_assert(WSP_GGML_OP_COUNT == 90, "WSP_GGML_OP_COUNT != 90");
+static_assert(WSP_GGML_OP_COUNT == 94, "WSP_GGML_OP_COUNT != 94");
 
 static_assert(WSP_GGML_OP_POOL_COUNT == 2, "WSP_GGML_OP_POOL_COUNT != 2");
 
@@ -1150,6 +1158,8 @@ static const char * WSP_GGML_UNARY_OP_NAME[WSP_GGML_UNARY_OP_COUNT] = {
     "HARDSWISH",
     "HARDSIGMOID",
     "EXP",
+    "EXPM1",
+    "SOFTPLUS",
     "GELU_ERF",
     "XIELU",
     "FLOOR",
@@ -1158,7 +1168,7 @@ static const char * WSP_GGML_UNARY_OP_NAME[WSP_GGML_UNARY_OP_COUNT] = {
     "TRUNC",
 };
 
-static_assert(WSP_GGML_UNARY_OP_COUNT == 20, "WSP_GGML_UNARY_OP_COUNT != 20");
+static_assert(WSP_GGML_UNARY_OP_COUNT == 22, "WSP_GGML_UNARY_OP_COUNT != 22");
 
 static const char * WSP_GGML_GLU_OP_NAME[WSP_GGML_GLU_OP_COUNT] = {
     "REGLU",
@@ -2266,6 +2276,30 @@ struct wsp_ggml_tensor * wsp_ggml_log_inplace(
     return wsp_ggml_log_impl(ctx, a, true);
 }
 
+struct wsp_ggml_tensor * wsp_ggml_expm1(
+        struct wsp_ggml_context * ctx,
+        struct wsp_ggml_tensor  * a) {
+    return wsp_ggml_unary(ctx, a, WSP_GGML_UNARY_OP_EXPM1);
+}
+
+struct wsp_ggml_tensor * wsp_ggml_expm1_inplace(
+        struct wsp_ggml_context * ctx,
+        struct wsp_ggml_tensor  * a) {
+    return wsp_ggml_unary_inplace(ctx, a, WSP_GGML_UNARY_OP_EXPM1);
+}
+
+struct wsp_ggml_tensor * wsp_ggml_softplus(
+        struct wsp_ggml_context * ctx,
+        struct wsp_ggml_tensor  * a) {
+    return wsp_ggml_unary(ctx, a, WSP_GGML_UNARY_OP_SOFTPLUS);
+}
+
+struct wsp_ggml_tensor * wsp_ggml_softplus_inplace(
+        struct wsp_ggml_context * ctx,
+        struct wsp_ggml_tensor  * a) {
+    return wsp_ggml_unary_inplace(ctx, a, WSP_GGML_UNARY_OP_SOFTPLUS);
+}
+
 // wsp_ggml_sin
 
 static struct wsp_ggml_tensor * wsp_ggml_sin_impl(
@@ -2349,6 +2383,21 @@ struct wsp_ggml_tensor * wsp_ggml_sum_rows(
     return result;
 }
 
+// wsp_ggml_cumsum
+
+struct wsp_ggml_tensor * wsp_ggml_cumsum(
+        struct wsp_ggml_context * ctx,
+        struct wsp_ggml_tensor  * a) {
+    WSP_GGML_ASSERT(a->type == WSP_GGML_TYPE_F32);
+
+    struct wsp_ggml_tensor * result = wsp_ggml_dup_tensor(ctx, a);
+
+    result->op     = WSP_GGML_OP_CUMSUM;
+    result->src[0] = a;
+
+    return result;
+}
+
 // wsp_ggml_mean
 
 struct wsp_ggml_tensor * wsp_ggml_mean(
@@ -2676,8 +2725,8 @@ struct wsp_ggml_tensor * wsp_ggml_xielu(
     struct wsp_ggml_tensor * result = wsp_ggml_dup_tensor(ctx, a);
 
     wsp_ggml_set_op_params_i32(result, 0, (int32_t) WSP_GGML_UNARY_OP_XIELU);
-    wsp_ggml_set_op_params_f32(result, 1, beta + wsp_ggml_softplus(alpha_n));
-    wsp_ggml_set_op_params_f32(result, 2, wsp_ggml_softplus(alpha_p));
+    wsp_ggml_set_op_params_f32(result, 1, beta + wsp_ggml_compute_softplus_f32(alpha_n));
+    wsp_ggml_set_op_params_f32(result, 2, wsp_ggml_compute_softplus_f32(alpha_p));
     wsp_ggml_set_op_params_f32(result, 3, beta);
     wsp_ggml_set_op_params_f32(result, 4, eps);
 
@@ -5036,6 +5085,61 @@ struct wsp_ggml_tensor * wsp_ggml_timestep_embedding(
     return result;
 }
 
+// wsp_ggml_tri
+
+struct wsp_ggml_tensor * wsp_ggml_tri(
+    struct wsp_ggml_context * ctx,
+    struct wsp_ggml_tensor  * a,
+    enum wsp_ggml_tri_type    type) {
+    WSP_GGML_ASSERT(a->type == WSP_GGML_TYPE_F32);
+
+    WSP_GGML_ASSERT(wsp_ggml_is_contiguous(a));
+    WSP_GGML_ASSERT(a->ne[0] == a->ne[1]);
+
+    struct wsp_ggml_tensor * result = wsp_ggml_dup_tensor(ctx, a);
+
+    wsp_ggml_set_op_params_i32(result, 0, type);
+
+    result->op = WSP_GGML_OP_TRI;
+    result->src[0] = a;
+
+    return result;
+}
+
+// wsp_ggml_fill
+
+static struct wsp_ggml_tensor * wsp_ggml_fill_impl(
+    struct wsp_ggml_context * ctx,
+    struct wsp_ggml_tensor  * a,
+    float                 c,
+    bool                  inplace) {
+    WSP_GGML_ASSERT(a->type == WSP_GGML_TYPE_F32);
+    WSP_GGML_ASSERT(wsp_ggml_is_contiguous(a));
+
+    struct wsp_ggml_tensor * result = inplace ? wsp_ggml_view_tensor(ctx, a) : wsp_ggml_dup_tensor(ctx, a);
+
+    wsp_ggml_set_op_params_f32(result, 0, c);
+
+    result->op = WSP_GGML_OP_FILL;
+    result->src[0] = a;
+
+    return result;
+}
+
+struct wsp_ggml_tensor * wsp_ggml_fill(
+    struct wsp_ggml_context * ctx,
+    struct wsp_ggml_tensor  * a,
+    float                 c) {
+    return wsp_ggml_fill_impl(ctx, a, c, false);
+}
+
+struct wsp_ggml_tensor * wsp_ggml_fill_inplace(
+    struct wsp_ggml_context * ctx,
+    struct wsp_ggml_tensor  * a,
+    float                 c) {
+    return wsp_ggml_fill_impl(ctx, a, c, true);
+}
+
 // wsp_ggml_argsort
 
 struct wsp_ggml_tensor * wsp_ggml_argsort(
@@ -5890,6 +5994,41 @@ struct wsp_ggml_tensor * wsp_ggml_opt_step_sgd(
     return result;
 }
 
+// solve_tri
+
+struct wsp_ggml_tensor * wsp_ggml_solve_tri(
+        struct wsp_ggml_context * ctx,
+        struct wsp_ggml_tensor  * a,
+        struct wsp_ggml_tensor  * b,
+        bool                  left,
+        bool                  lower,
+        bool                  uni) {
+    WSP_GGML_ASSERT(a->type == WSP_GGML_TYPE_F32);
+    WSP_GGML_ASSERT(b->type == WSP_GGML_TYPE_F32);
+
+    // A must be square and lower diagonal
+    WSP_GGML_ASSERT(a->ne[0] == a->ne[1]);
+    // B must have same outer dimension as A
+    WSP_GGML_ASSERT(a->ne[1] == b->ne[1]);
+
+    // batch dimensions must be equal
+    WSP_GGML_ASSERT(a->ne[2] == b->ne[2]);
+    WSP_GGML_ASSERT(a->ne[3] == b->ne[3]);
+
+    WSP_GGML_ASSERT(wsp_ggml_is_contiguous(a));
+    WSP_GGML_ASSERT(wsp_ggml_is_contiguous(b));
+
+    WSP_GGML_ASSERT(lower && left && !uni); // TODO: support other variants
+
+    struct wsp_ggml_tensor * result = wsp_ggml_new_tensor_4d(ctx, WSP_GGML_TYPE_F32, b->ne[0], b->ne[1], b->ne[2], b->ne[3]);
+
+    result->op     = WSP_GGML_OP_SOLVE_TRI;
+    result->src[0] = a;
+    result->src[1] = b;
+
+    return result;
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 
 struct wsp_ggml_hash_set wsp_ggml_hash_set_new(size_t size) {
@@ -6462,6 +6601,16 @@ static void wsp_ggml_compute_backward(
                         wsp_ggml_add_or_set(ctx, cgraph, isrc0, wsp_ggml_mul(ctx, tensor, grad));
                     }
                 } break;
+                case WSP_GGML_UNARY_OP_EXPM1: {
+                    if (src0_needs_grads) {
+                        wsp_ggml_add_or_set(ctx, cgraph, isrc0, wsp_ggml_mul(ctx, grad, wsp_ggml_exp(ctx, src0)));
+                    }
+                } break;
+                case WSP_GGML_UNARY_OP_SOFTPLUS: {
+                    if (src0_needs_grads) {
+                        wsp_ggml_add_or_set(ctx, cgraph, isrc0, wsp_ggml_mul(ctx, grad, wsp_ggml_sigmoid(ctx, src0)));
+                    }
+                } break;
                 default: {
                     fprintf(stderr, "%s: unsupported unary op for backward pass: %s\n",
                         __func__, wsp_ggml_unary_op_name(wsp_ggml_get_unary_op(tensor)));

package/cpp/ggml.h

@@ -242,6 +242,7 @@
 #define WSP_GGML_ROPE_TYPE_NEOX   2
 #define WSP_GGML_ROPE_TYPE_MROPE  8
 #define WSP_GGML_ROPE_TYPE_VISION 24
+#define WSP_GGML_ROPE_TYPE_IMROPE 40 // binary: 101000
 
 #define WSP_GGML_MROPE_SECTIONS   4
 
@@ -474,6 +475,7 @@ extern "C" {
         WSP_GGML_OP_COS,
         WSP_GGML_OP_SUM,
         WSP_GGML_OP_SUM_ROWS,
+        WSP_GGML_OP_CUMSUM,
         WSP_GGML_OP_MEAN,
         WSP_GGML_OP_ARGMAX,
         WSP_GGML_OP_COUNT_EQUAL,
@@ -529,6 +531,8 @@ extern "C" {
         WSP_GGML_OP_TIMESTEP_EMBEDDING,
         WSP_GGML_OP_ARGSORT,
         WSP_GGML_OP_LEAKY_RELU,
+        WSP_GGML_OP_TRI,
+        WSP_GGML_OP_FILL,
 
         WSP_GGML_OP_FLASH_ATTN_EXT,
         WSP_GGML_OP_FLASH_ATTN_BACK,
@@ -541,6 +545,7 @@ extern "C" {
         WSP_GGML_OP_RWKV_WKV6,
         WSP_GGML_OP_GATED_LINEAR_ATTN,
         WSP_GGML_OP_RWKV_WKV7,
+        WSP_GGML_OP_SOLVE_TRI,
 
         WSP_GGML_OP_UNARY,
 
@@ -575,6 +580,8 @@ extern "C" {
         WSP_GGML_UNARY_OP_HARDSWISH,
         WSP_GGML_UNARY_OP_HARDSIGMOID,
         WSP_GGML_UNARY_OP_EXP,
+        WSP_GGML_UNARY_OP_EXPM1,
+        WSP_GGML_UNARY_OP_SOFTPLUS,
         WSP_GGML_UNARY_OP_GELU_ERF,
         WSP_GGML_UNARY_OP_XIELU,
         WSP_GGML_UNARY_OP_FLOOR,
@@ -619,6 +626,13 @@ extern "C" {
         WSP_GGML_TENSOR_FLAG_LOSS   =  8, // ...defines loss for numerical optimization (multiple loss tensors add up)
     };
 
+    enum wsp_ggml_tri_type {
+        WSP_GGML_TRI_TYPE_UPPER_DIAG = 0,
+        WSP_GGML_TRI_TYPE_UPPER      = 1,
+        WSP_GGML_TRI_TYPE_LOWER_DIAG = 2,
+        WSP_GGML_TRI_TYPE_LOWER      = 3
+    };
+
     struct wsp_ggml_init_params {
         // memory pool
         size_t mem_size;   // bytes
@@ -956,6 +970,22 @@ extern "C" {
             struct wsp_ggml_context * ctx,
             struct wsp_ggml_tensor  * a);
 
+    WSP_GGML_API struct wsp_ggml_tensor * wsp_ggml_expm1(
+            struct wsp_ggml_context * ctx,
+            struct wsp_ggml_tensor  * a);
+
+    WSP_GGML_API struct wsp_ggml_tensor * wsp_ggml_expm1_inplace(
+            struct wsp_ggml_context * ctx,
+            struct wsp_ggml_tensor  * a);
+
+    WSP_GGML_API struct wsp_ggml_tensor * wsp_ggml_softplus(
+            struct wsp_ggml_context * ctx,
+            struct wsp_ggml_tensor  * a);
+
+    WSP_GGML_API struct wsp_ggml_tensor * wsp_ggml_softplus_inplace(
+            struct wsp_ggml_context * ctx,
+            struct wsp_ggml_tensor  * a);
+
     WSP_GGML_API struct wsp_ggml_tensor * wsp_ggml_sin(
             struct wsp_ggml_context * ctx,
             struct wsp_ggml_tensor  * a);
@@ -982,6 +1012,10 @@ extern "C" {
             struct wsp_ggml_context * ctx,
             struct wsp_ggml_tensor  * a);
 
+    WSP_GGML_API struct wsp_ggml_tensor * wsp_ggml_cumsum(
+        struct wsp_ggml_context * ctx,
+        struct wsp_ggml_tensor  * a);
+
     // mean along rows
     WSP_GGML_API struct wsp_ggml_tensor * wsp_ggml_mean(
             struct wsp_ggml_context * ctx,
@@ -2107,6 +2141,7 @@ extern "C" {
     enum wsp_ggml_scale_mode {
         WSP_GGML_SCALE_MODE_NEAREST  = 0,
         WSP_GGML_SCALE_MODE_BILINEAR = 1,
+        WSP_GGML_SCALE_MODE_BICUBIC  = 2,
 
         WSP_GGML_SCALE_MODE_COUNT
     };
@@ -2185,6 +2220,23 @@ extern "C" {
             int                   shift2,
             int                   shift3);
 
+    // Convert matrix into a triangular one (upper, strict upper, lower or strict lower) by writing
+    // zeroes everywhere outside the masked area
+    WSP_GGML_API struct wsp_ggml_tensor * wsp_ggml_tri(
+            struct wsp_ggml_context * ctx,
+            struct wsp_ggml_tensor  * a,
+            enum wsp_ggml_tri_type    type);
+
+    // Fill tensor a with constant c
+    WSP_GGML_API struct wsp_ggml_tensor * wsp_ggml_fill(
+            struct wsp_ggml_context * ctx,
+            struct wsp_ggml_tensor  * a,
+            float                 c);
+
+    WSP_GGML_API struct wsp_ggml_tensor * wsp_ggml_fill_inplace(
+            struct wsp_ggml_context * ctx,
+            struct wsp_ggml_tensor  * a,
+            float                 c);
 
     // Ref: https://github.com/CompVis/stable-diffusion/blob/main/ldm/modules/diffusionmodules/util.py#L151
     // timesteps: [N,]
@@ -2354,6 +2406,27 @@ extern "C" {
             struct wsp_ggml_tensor  * b,
             struct wsp_ggml_tensor  * state);
 
+    /* Solves a specific equation of the form Ax=B, where A is a triangular matrix
+    *  without zeroes on the diagonal (i.e. invertible).
+    *  B can have any number of columns, but must have the same number of rows as A
+    *  If A is [n, n] and B is [n, m], then the result will be [n, m] as well
+    *  Has O(n^3) complexity (unlike most matrix ops out there), so use on cases
+    *  where n > 100 sparingly, pre-chunk if necessary.
+    *
+    *  If left = false, solves xA=B instead
+    *  If lower = false, assumes upper triangular instead
+    *  If uni = true, assumes diagonal of A to be all ones (will override actual values)
+    *
+    *  TODO: currently only lower, right, non-unitriangular variant is implemented
+    */
+    WSP_GGML_API struct wsp_ggml_tensor * wsp_ggml_solve_tri(
+        struct wsp_ggml_context * ctx,
+        struct wsp_ggml_tensor  * a,
+        struct wsp_ggml_tensor  * b,
+        bool                  left,
+        bool                  lower,
+        bool                  uni);
+
     // custom operators
 
     typedef void (*wsp_ggml_custom1_op_t)(struct wsp_ggml_tensor * dst , const struct wsp_ggml_tensor * a, int ith, int nth, void * userdata);

package/cpp/whisper.cpp

@@ -1296,7 +1296,11 @@ static wsp_ggml_backend_t whisper_backend_init_gpu(const whisper_context_params
     if (params.use_gpu) {
         for (size_t i = 0; i < wsp_ggml_backend_dev_count(); ++i) {
             wsp_ggml_backend_dev_t dev_cur = wsp_ggml_backend_dev_get(i);
-            if (wsp_ggml_backend_dev_type(dev_cur) == WSP_GGML_BACKEND_DEVICE_TYPE_GPU || wsp_ggml_backend_dev_type(dev_cur) == WSP_GGML_BACKEND_DEVICE_TYPE_IGPU) {
+            enum wsp_ggml_backend_dev_type dev_type = wsp_ggml_backend_dev_type(dev_cur);
+            const char * dev_name = wsp_ggml_backend_dev_name(dev_cur);
+            WHISPER_LOG_INFO("%s: device %zu: %s (type: %d)\n", __func__, i, dev_name, dev_type);
+            if (dev_type == WSP_GGML_BACKEND_DEVICE_TYPE_GPU || dev_type == WSP_GGML_BACKEND_DEVICE_TYPE_IGPU) {
+                WHISPER_LOG_INFO("%s: found GPU device %zu: %s (type: %d, cnt: %d)\n", __func__, i, dev_name, dev_type, cnt);
                 if (cnt == params.gpu_device) {
                     dev = dev_cur;
                 }
@@ -6693,7 +6697,7 @@ static bool whisper_vad(
             }
 
             segment_start_samples = std::min(segment_start_samples, n_samples - 1);
-            segment_end_samples = std::min(segment_end_samples, n_samples);
+            segment_end_samples = std::min(segment_end_samples, n_samples - 1);
             int segment_length = segment_end_samples - segment_start_samples;
             if (segment_length > 0) {
                 whisper_state::vad_segment_info segment;

package/ios/rnwhisper.xcframework/ios-arm64/rnwhisper.framework/Headers/ggml-impl.h

@@ -102,7 +102,7 @@ static bool wsp_ggml_op_is_empty(enum wsp_ggml_op op) {
     }
 }
 
-static inline float wsp_ggml_softplus(float input) {
+static inline float wsp_ggml_compute_softplus_f32(float input) {
     return (input > 20.0f) ? input : logf(1 + expf(input));
 }
 //
@@ -682,6 +682,7 @@ static inline bool wsp_ggml_can_fuse_subgraph(const struct wsp_ggml_cgraph * cgr
 #endif
 
 #ifdef __cplusplus
+#include <array>
 #include <initializer_list>
 #include <vector>
 
@@ -697,6 +698,21 @@ inline bool wsp_ggml_can_fuse_subgraph(const struct wsp_ggml_cgraph *          c
     return wsp_ggml_can_fuse_subgraph(cgraph, start_idx, ops.size(), ops.begin(), outputs.begin(), outputs.size());
 }
 
+// Return true if the edges in the graph match expectations.
+inline bool wsp_ggml_check_edges(const struct wsp_ggml_cgraph *                cgraph,
+                             int                                       start_idx,
+                             std::initializer_list<std::array<int, 3>> edges) {
+    for (const auto & edge : edges) {
+        int dst_node = edge[0];
+        int src_idx  = edge[1];
+        int src_node = edge[2];
+        if (cgraph->nodes[start_idx + dst_node]->src[src_idx] != cgraph->nodes[start_idx + src_node]) {
+            return false;
+        }
+    }
+    return true;
+}
+
 // expose GGUF internals for test code
 WSP_GGML_API size_t wsp_gguf_type_size(enum wsp_gguf_type type);
 WSP_GGML_API struct wsp_gguf_context * wsp_gguf_init_from_file_impl(FILE * file, struct wsp_gguf_init_params params);

package/ios/rnwhisper.xcframework/ios-arm64/rnwhisper.framework/Headers/ggml.h

@@ -242,6 +242,7 @@
 #define WSP_GGML_ROPE_TYPE_NEOX   2
 #define WSP_GGML_ROPE_TYPE_MROPE  8
 #define WSP_GGML_ROPE_TYPE_VISION 24
+#define WSP_GGML_ROPE_TYPE_IMROPE 40 // binary: 101000
 
 #define WSP_GGML_MROPE_SECTIONS   4
 
@@ -474,6 +475,7 @@ extern "C" {
         WSP_GGML_OP_COS,
         WSP_GGML_OP_SUM,
         WSP_GGML_OP_SUM_ROWS,
+        WSP_GGML_OP_CUMSUM,
         WSP_GGML_OP_MEAN,
         WSP_GGML_OP_ARGMAX,
         WSP_GGML_OP_COUNT_EQUAL,
@@ -529,6 +531,8 @@ extern "C" {
         WSP_GGML_OP_TIMESTEP_EMBEDDING,
         WSP_GGML_OP_ARGSORT,
         WSP_GGML_OP_LEAKY_RELU,
+        WSP_GGML_OP_TRI,
+        WSP_GGML_OP_FILL,
 
         WSP_GGML_OP_FLASH_ATTN_EXT,
         WSP_GGML_OP_FLASH_ATTN_BACK,
@@ -541,6 +545,7 @@ extern "C" {
         WSP_GGML_OP_RWKV_WKV6,
         WSP_GGML_OP_GATED_LINEAR_ATTN,
         WSP_GGML_OP_RWKV_WKV7,
+        WSP_GGML_OP_SOLVE_TRI,
 
         WSP_GGML_OP_UNARY,
 
@@ -575,6 +580,8 @@ extern "C" {
         WSP_GGML_UNARY_OP_HARDSWISH,
         WSP_GGML_UNARY_OP_HARDSIGMOID,
         WSP_GGML_UNARY_OP_EXP,
+        WSP_GGML_UNARY_OP_EXPM1,
+        WSP_GGML_UNARY_OP_SOFTPLUS,
         WSP_GGML_UNARY_OP_GELU_ERF,
         WSP_GGML_UNARY_OP_XIELU,
         WSP_GGML_UNARY_OP_FLOOR,
@@ -619,6 +626,13 @@ extern "C" {
         WSP_GGML_TENSOR_FLAG_LOSS   =  8, // ...defines loss for numerical optimization (multiple loss tensors add up)
     };
 
+    enum wsp_ggml_tri_type {
+        WSP_GGML_TRI_TYPE_UPPER_DIAG = 0,
+        WSP_GGML_TRI_TYPE_UPPER      = 1,
+        WSP_GGML_TRI_TYPE_LOWER_DIAG = 2,
+        WSP_GGML_TRI_TYPE_LOWER      = 3
+    };
+
     struct wsp_ggml_init_params {
         // memory pool
         size_t mem_size;   // bytes
@@ -956,6 +970,22 @@ extern "C" {
             struct wsp_ggml_context * ctx,
             struct wsp_ggml_tensor  * a);
 
+    WSP_GGML_API struct wsp_ggml_tensor * wsp_ggml_expm1(
+            struct wsp_ggml_context * ctx,
+            struct wsp_ggml_tensor  * a);
+
+    WSP_GGML_API struct wsp_ggml_tensor * wsp_ggml_expm1_inplace(
+            struct wsp_ggml_context * ctx,
+            struct wsp_ggml_tensor  * a);
+
+    WSP_GGML_API struct wsp_ggml_tensor * wsp_ggml_softplus(
+            struct wsp_ggml_context * ctx,
+            struct wsp_ggml_tensor  * a);
+
+    WSP_GGML_API struct wsp_ggml_tensor * wsp_ggml_softplus_inplace(
+            struct wsp_ggml_context * ctx,
+            struct wsp_ggml_tensor  * a);
+
     WSP_GGML_API struct wsp_ggml_tensor * wsp_ggml_sin(
             struct wsp_ggml_context * ctx,
             struct wsp_ggml_tensor  * a);
@@ -982,6 +1012,10 @@ extern "C" {
             struct wsp_ggml_context * ctx,
             struct wsp_ggml_tensor  * a);
 
+    WSP_GGML_API struct wsp_ggml_tensor * wsp_ggml_cumsum(
+        struct wsp_ggml_context * ctx,
+        struct wsp_ggml_tensor  * a);
+
     // mean along rows
     WSP_GGML_API struct wsp_ggml_tensor * wsp_ggml_mean(
             struct wsp_ggml_context * ctx,
@@ -2107,6 +2141,7 @@ extern "C" {
     enum wsp_ggml_scale_mode {
         WSP_GGML_SCALE_MODE_NEAREST  = 0,
         WSP_GGML_SCALE_MODE_BILINEAR = 1,
+        WSP_GGML_SCALE_MODE_BICUBIC  = 2,
 
         WSP_GGML_SCALE_MODE_COUNT
     };
@@ -2185,6 +2220,23 @@ extern "C" {
             int                   shift2,
             int                   shift3);
 
+    // Convert matrix into a triangular one (upper, strict upper, lower or strict lower) by writing
+    // zeroes everywhere outside the masked area
+    WSP_GGML_API struct wsp_ggml_tensor * wsp_ggml_tri(
+            struct wsp_ggml_context * ctx,
+            struct wsp_ggml_tensor  * a,
+            enum wsp_ggml_tri_type    type);
+
+    // Fill tensor a with constant c
+    WSP_GGML_API struct wsp_ggml_tensor * wsp_ggml_fill(
+            struct wsp_ggml_context * ctx,
+            struct wsp_ggml_tensor  * a,
+            float                 c);
+
+    WSP_GGML_API struct wsp_ggml_tensor * wsp_ggml_fill_inplace(
+            struct wsp_ggml_context * ctx,
+            struct wsp_ggml_tensor  * a,
+            float                 c);
 
     // Ref: https://github.com/CompVis/stable-diffusion/blob/main/ldm/modules/diffusionmodules/util.py#L151
     // timesteps: [N,]
@@ -2354,6 +2406,27 @@ extern "C" {
             struct wsp_ggml_tensor  * b,
             struct wsp_ggml_tensor  * state);
 
+    /* Solves a specific equation of the form Ax=B, where A is a triangular matrix
+    *  without zeroes on the diagonal (i.e. invertible).
+    *  B can have any number of columns, but must have the same number of rows as A
+    *  If A is [n, n] and B is [n, m], then the result will be [n, m] as well
+    *  Has O(n^3) complexity (unlike most matrix ops out there), so use on cases
+    *  where n > 100 sparingly, pre-chunk if necessary.
+    *
+    *  If left = false, solves xA=B instead
+    *  If lower = false, assumes upper triangular instead
+    *  If uni = true, assumes diagonal of A to be all ones (will override actual values)
+    *
+    *  TODO: currently only lower, right, non-unitriangular variant is implemented
+    */
+    WSP_GGML_API struct wsp_ggml_tensor * wsp_ggml_solve_tri(
+        struct wsp_ggml_context * ctx,
+        struct wsp_ggml_tensor  * a,
+        struct wsp_ggml_tensor  * b,
+        bool                  left,
+        bool                  lower,
+        bool                  uni);
+
     // custom operators
 
     typedef void (*wsp_ggml_custom1_op_t)(struct wsp_ggml_tensor * dst , const struct wsp_ggml_tensor * a, int ith, int nth, void * userdata);