whisper.rn 0.5.2 → 0.5.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/README.md +1 -1
  2. package/cpp/ggml-alloc.c +11 -4
  3. package/cpp/ggml-backend-reg.cpp +8 -0
  4. package/cpp/ggml-backend.cpp +0 -2
  5. package/cpp/ggml-cpu/arch/arm/quants.c +428 -26
  6. package/cpp/ggml-cpu/ggml-cpu-impl.h +3 -1
  7. package/cpp/ggml-cpu/ggml-cpu.c +50 -21
  8. package/cpp/ggml-cpu/ops.cpp +458 -349
  9. package/cpp/ggml-cpu/ops.h +4 -4
  10. package/cpp/ggml-cpu/repack.cpp +143 -29
  11. package/cpp/ggml-cpu/simd-mappings.h +25 -25
  12. package/cpp/ggml-cpu/unary-ops.cpp +16 -0
  13. package/cpp/ggml-cpu/unary-ops.h +2 -0
  14. package/cpp/ggml-cpu/vec.cpp +17 -0
  15. package/cpp/ggml-cpu/vec.h +10 -0
  16. package/cpp/ggml-impl.h +17 -1
  17. package/cpp/ggml-metal/ggml-metal-context.m +5 -6
  18. package/cpp/ggml-metal/ggml-metal-device.cpp +101 -4
  19. package/cpp/ggml-metal/ggml-metal-device.h +8 -1
  20. package/cpp/ggml-metal/ggml-metal-device.m +216 -14
  21. package/cpp/ggml-metal/ggml-metal-impl.h +90 -2
  22. package/cpp/ggml-metal/ggml-metal-ops.cpp +346 -85
  23. package/cpp/ggml-metal/ggml-metal-ops.h +2 -0
  24. package/cpp/ggml-metal/ggml-metal.cpp +5 -0
  25. package/cpp/ggml-metal/ggml-metal.metal +12436 -0
  26. package/cpp/ggml.c +154 -5
  27. package/cpp/ggml.h +73 -0
  28. package/cpp/whisper.cpp +6 -2
  29. package/ios/rnwhisper.xcframework/ios-arm64/rnwhisper.framework/Headers/ggml-impl.h +17 -1
  30. package/ios/rnwhisper.xcframework/ios-arm64/rnwhisper.framework/Headers/ggml.h +73 -0
  31. package/ios/rnwhisper.xcframework/ios-arm64/rnwhisper.framework/Info.plist +0 -0
  32. package/ios/rnwhisper.xcframework/ios-arm64/rnwhisper.framework/ggml-metal.metal +12436 -0
  33. package/ios/rnwhisper.xcframework/ios-arm64/rnwhisper.framework/rnwhisper +0 -0
  34. package/ios/rnwhisper.xcframework/ios-arm64_x86_64-simulator/rnwhisper.framework/Headers/ggml-impl.h +17 -1
  35. package/ios/rnwhisper.xcframework/ios-arm64_x86_64-simulator/rnwhisper.framework/Headers/ggml.h +73 -0
  36. package/ios/rnwhisper.xcframework/ios-arm64_x86_64-simulator/rnwhisper.framework/Info.plist +0 -0
  37. package/ios/rnwhisper.xcframework/ios-arm64_x86_64-simulator/rnwhisper.framework/_CodeSignature/CodeResources +1 -1
  38. package/ios/rnwhisper.xcframework/ios-arm64_x86_64-simulator/rnwhisper.framework/ggml-metal.metal +12436 -0
  39. package/ios/rnwhisper.xcframework/ios-arm64_x86_64-simulator/rnwhisper.framework/rnwhisper +0 -0
  40. package/ios/rnwhisper.xcframework/tvos-arm64/rnwhisper.framework/Headers/ggml-impl.h +17 -1
  41. package/ios/rnwhisper.xcframework/tvos-arm64/rnwhisper.framework/Headers/ggml.h +73 -0
  42. package/ios/rnwhisper.xcframework/tvos-arm64/rnwhisper.framework/Info.plist +0 -0
  43. package/ios/rnwhisper.xcframework/tvos-arm64/rnwhisper.framework/ggml-metal.metal +12436 -0
  44. package/ios/rnwhisper.xcframework/tvos-arm64/rnwhisper.framework/rnwhisper +0 -0
  45. package/ios/rnwhisper.xcframework/tvos-arm64_x86_64-simulator/rnwhisper.framework/Headers/ggml-impl.h +17 -1
  46. package/ios/rnwhisper.xcframework/tvos-arm64_x86_64-simulator/rnwhisper.framework/Headers/ggml.h +73 -0
  47. package/ios/rnwhisper.xcframework/tvos-arm64_x86_64-simulator/rnwhisper.framework/Info.plist +0 -0
  48. package/ios/rnwhisper.xcframework/tvos-arm64_x86_64-simulator/rnwhisper.framework/_CodeSignature/CodeResources +1 -1
  49. package/ios/rnwhisper.xcframework/tvos-arm64_x86_64-simulator/rnwhisper.framework/ggml-metal.metal +12436 -0
  50. package/ios/rnwhisper.xcframework/tvos-arm64_x86_64-simulator/rnwhisper.framework/rnwhisper +0 -0
  51. package/lib/commonjs/realtime-transcription/RealtimeTranscriber.js +156 -12
  52. package/lib/commonjs/realtime-transcription/RealtimeTranscriber.js.map +1 -1
  53. package/lib/module/realtime-transcription/RealtimeTranscriber.js +155 -12
  54. package/lib/module/realtime-transcription/RealtimeTranscriber.js.map +1 -1
  55. package/lib/typescript/realtime-transcription/RealtimeTranscriber.d.ts +29 -0
  56. package/lib/typescript/realtime-transcription/RealtimeTranscriber.d.ts.map +1 -1
  57. package/lib/typescript/realtime-transcription/types.d.ts +7 -0
  58. package/lib/typescript/realtime-transcription/types.d.ts.map +1 -1
  59. package/package.json +1 -1
  60. package/src/realtime-transcription/RealtimeTranscriber.ts +179 -9
  61. package/src/realtime-transcription/types.ts +9 -0
  62. package/whisper-rn.podspec +1 -1
  63. package/cpp/ggml-metal/ggml-whisper-sim.metallib +0 -0
  64. package/cpp/ggml-metal/ggml-whisper.metallib +0 -0
  65. package/ios/rnwhisper.xcframework/ios-arm64/rnwhisper.framework/ggml-whisper.metallib +0 -0
  66. package/ios/rnwhisper.xcframework/ios-arm64_x86_64-simulator/rnwhisper.framework/ggml-whisper-sim.metallib +0 -0
  67. package/ios/rnwhisper.xcframework/tvos-arm64/rnwhisper.framework/ggml-whisper.metallib +0 -0
  68. package/ios/rnwhisper.xcframework/tvos-arm64_x86_64-simulator/rnwhisper.framework/ggml-whisper-sim.metallib +0 -0
@@ -102,7 +102,7 @@ static bool wsp_ggml_op_is_empty(enum wsp_ggml_op op) {
102
102
  }
103
103
  }
104
104
 
105
- static inline float wsp_ggml_softplus(float input) {
105
+ static inline float wsp_ggml_compute_softplus_f32(float input) {
106
106
  return (input > 20.0f) ? input : logf(1 + expf(input));
107
107
  }
108
108
  //
@@ -682,6 +682,7 @@ static inline bool wsp_ggml_can_fuse_subgraph(const struct wsp_ggml_cgraph * cgr
682
682
  #endif
683
683
 
684
684
  #ifdef __cplusplus
685
+ #include <array>
685
686
  #include <initializer_list>
686
687
  #include <vector>
687
688
 
@@ -697,6 +698,21 @@ inline bool wsp_ggml_can_fuse_subgraph(const struct wsp_ggml_cgraph * c
697
698
  return wsp_ggml_can_fuse_subgraph(cgraph, start_idx, ops.size(), ops.begin(), outputs.begin(), outputs.size());
698
699
  }
699
700
 
701
+ // Return true if the edges in the graph match expectations.
702
+ inline bool wsp_ggml_check_edges(const struct wsp_ggml_cgraph * cgraph,
703
+ int start_idx,
704
+ std::initializer_list<std::array<int, 3>> edges) {
705
+ for (const auto & edge : edges) {
706
+ int dst_node = edge[0];
707
+ int src_idx = edge[1];
708
+ int src_node = edge[2];
709
+ if (cgraph->nodes[start_idx + dst_node]->src[src_idx] != cgraph->nodes[start_idx + src_node]) {
710
+ return false;
711
+ }
712
+ }
713
+ return true;
714
+ }
715
+
700
716
  // expose GGUF internals for test code
701
717
  WSP_GGML_API size_t wsp_gguf_type_size(enum wsp_gguf_type type);
702
718
  WSP_GGML_API struct wsp_gguf_context * wsp_gguf_init_from_file_impl(FILE * file, struct wsp_gguf_init_params params);
@@ -242,6 +242,7 @@
242
242
  #define WSP_GGML_ROPE_TYPE_NEOX 2
243
243
  #define WSP_GGML_ROPE_TYPE_MROPE 8
244
244
  #define WSP_GGML_ROPE_TYPE_VISION 24
245
+ #define WSP_GGML_ROPE_TYPE_IMROPE 40 // binary: 101000
245
246
 
246
247
  #define WSP_GGML_MROPE_SECTIONS 4
247
248
 
@@ -474,6 +475,7 @@ extern "C" {
474
475
  WSP_GGML_OP_COS,
475
476
  WSP_GGML_OP_SUM,
476
477
  WSP_GGML_OP_SUM_ROWS,
478
+ WSP_GGML_OP_CUMSUM,
477
479
  WSP_GGML_OP_MEAN,
478
480
  WSP_GGML_OP_ARGMAX,
479
481
  WSP_GGML_OP_COUNT_EQUAL,
@@ -529,6 +531,8 @@ extern "C" {
529
531
  WSP_GGML_OP_TIMESTEP_EMBEDDING,
530
532
  WSP_GGML_OP_ARGSORT,
531
533
  WSP_GGML_OP_LEAKY_RELU,
534
+ WSP_GGML_OP_TRI,
535
+ WSP_GGML_OP_FILL,
532
536
 
533
537
  WSP_GGML_OP_FLASH_ATTN_EXT,
534
538
  WSP_GGML_OP_FLASH_ATTN_BACK,
@@ -541,6 +545,7 @@ extern "C" {
541
545
  WSP_GGML_OP_RWKV_WKV6,
542
546
  WSP_GGML_OP_GATED_LINEAR_ATTN,
543
547
  WSP_GGML_OP_RWKV_WKV7,
548
+ WSP_GGML_OP_SOLVE_TRI,
544
549
 
545
550
  WSP_GGML_OP_UNARY,
546
551
 
@@ -575,6 +580,8 @@ extern "C" {
575
580
  WSP_GGML_UNARY_OP_HARDSWISH,
576
581
  WSP_GGML_UNARY_OP_HARDSIGMOID,
577
582
  WSP_GGML_UNARY_OP_EXP,
583
+ WSP_GGML_UNARY_OP_EXPM1,
584
+ WSP_GGML_UNARY_OP_SOFTPLUS,
578
585
  WSP_GGML_UNARY_OP_GELU_ERF,
579
586
  WSP_GGML_UNARY_OP_XIELU,
580
587
  WSP_GGML_UNARY_OP_FLOOR,
@@ -619,6 +626,13 @@ extern "C" {
619
626
  WSP_GGML_TENSOR_FLAG_LOSS = 8, // ...defines loss for numerical optimization (multiple loss tensors add up)
620
627
  };
621
628
 
629
+ enum wsp_ggml_tri_type {
630
+ WSP_GGML_TRI_TYPE_UPPER_DIAG = 0,
631
+ WSP_GGML_TRI_TYPE_UPPER = 1,
632
+ WSP_GGML_TRI_TYPE_LOWER_DIAG = 2,
633
+ WSP_GGML_TRI_TYPE_LOWER = 3
634
+ };
635
+
622
636
  struct wsp_ggml_init_params {
623
637
  // memory pool
624
638
  size_t mem_size; // bytes
@@ -956,6 +970,22 @@ extern "C" {
956
970
  struct wsp_ggml_context * ctx,
957
971
  struct wsp_ggml_tensor * a);
958
972
 
973
+ WSP_GGML_API struct wsp_ggml_tensor * wsp_ggml_expm1(
974
+ struct wsp_ggml_context * ctx,
975
+ struct wsp_ggml_tensor * a);
976
+
977
+ WSP_GGML_API struct wsp_ggml_tensor * wsp_ggml_expm1_inplace(
978
+ struct wsp_ggml_context * ctx,
979
+ struct wsp_ggml_tensor * a);
980
+
981
+ WSP_GGML_API struct wsp_ggml_tensor * wsp_ggml_softplus(
982
+ struct wsp_ggml_context * ctx,
983
+ struct wsp_ggml_tensor * a);
984
+
985
+ WSP_GGML_API struct wsp_ggml_tensor * wsp_ggml_softplus_inplace(
986
+ struct wsp_ggml_context * ctx,
987
+ struct wsp_ggml_tensor * a);
988
+
959
989
  WSP_GGML_API struct wsp_ggml_tensor * wsp_ggml_sin(
960
990
  struct wsp_ggml_context * ctx,
961
991
  struct wsp_ggml_tensor * a);
@@ -982,6 +1012,10 @@ extern "C" {
982
1012
  struct wsp_ggml_context * ctx,
983
1013
  struct wsp_ggml_tensor * a);
984
1014
 
1015
+ WSP_GGML_API struct wsp_ggml_tensor * wsp_ggml_cumsum(
1016
+ struct wsp_ggml_context * ctx,
1017
+ struct wsp_ggml_tensor * a);
1018
+
985
1019
  // mean along rows
986
1020
  WSP_GGML_API struct wsp_ggml_tensor * wsp_ggml_mean(
987
1021
  struct wsp_ggml_context * ctx,
@@ -2107,6 +2141,7 @@ extern "C" {
2107
2141
  enum wsp_ggml_scale_mode {
2108
2142
  WSP_GGML_SCALE_MODE_NEAREST = 0,
2109
2143
  WSP_GGML_SCALE_MODE_BILINEAR = 1,
2144
+ WSP_GGML_SCALE_MODE_BICUBIC = 2,
2110
2145
 
2111
2146
  WSP_GGML_SCALE_MODE_COUNT
2112
2147
  };
@@ -2185,6 +2220,23 @@ extern "C" {
2185
2220
  int shift2,
2186
2221
  int shift3);
2187
2222
 
2223
+ // Convert matrix into a triangular one (upper, strict upper, lower or strict lower) by writing
2224
+ // zeroes everywhere outside the masked area
2225
+ WSP_GGML_API struct wsp_ggml_tensor * wsp_ggml_tri(
2226
+ struct wsp_ggml_context * ctx,
2227
+ struct wsp_ggml_tensor * a,
2228
+ enum wsp_ggml_tri_type type);
2229
+
2230
+ // Fill tensor a with constant c
2231
+ WSP_GGML_API struct wsp_ggml_tensor * wsp_ggml_fill(
2232
+ struct wsp_ggml_context * ctx,
2233
+ struct wsp_ggml_tensor * a,
2234
+ float c);
2235
+
2236
+ WSP_GGML_API struct wsp_ggml_tensor * wsp_ggml_fill_inplace(
2237
+ struct wsp_ggml_context * ctx,
2238
+ struct wsp_ggml_tensor * a,
2239
+ float c);
2188
2240
 
2189
2241
  // Ref: https://github.com/CompVis/stable-diffusion/blob/main/ldm/modules/diffusionmodules/util.py#L151
2190
2242
  // timesteps: [N,]
@@ -2354,6 +2406,27 @@ extern "C" {
2354
2406
  struct wsp_ggml_tensor * b,
2355
2407
  struct wsp_ggml_tensor * state);
2356
2408
 
2409
+ /* Solves a specific equation of the form Ax=B, where A is a triangular matrix
2410
+ * without zeroes on the diagonal (i.e. invertible).
2411
+ * B can have any number of columns, but must have the same number of rows as A
2412
+ * If A is [n, n] and B is [n, m], then the result will be [n, m] as well
2413
+ * Has O(n^3) complexity (unlike most matrix ops out there), so use on cases
2414
+ * where n > 100 sparingly, pre-chunk if necessary.
2415
+ *
2416
+ * If left = false, solves xA=B instead
2417
+ * If lower = false, assumes upper triangular instead
2418
+ * If uni = true, assumes diagonal of A to be all ones (will override actual values)
2419
+ *
2420
+ * TODO: currently only lower, right, non-unitriangular variant is implemented
2421
+ */
2422
+ WSP_GGML_API struct wsp_ggml_tensor * wsp_ggml_solve_tri(
2423
+ struct wsp_ggml_context * ctx,
2424
+ struct wsp_ggml_tensor * a,
2425
+ struct wsp_ggml_tensor * b,
2426
+ bool left,
2427
+ bool lower,
2428
+ bool uni);
2429
+
2357
2430
  // custom operators
2358
2431
 
2359
2432
  typedef void (*wsp_ggml_custom1_op_t)(struct wsp_ggml_tensor * dst , const struct wsp_ggml_tensor * a, int ith, int nth, void * userdata);
@@ -6,7 +6,7 @@
6
6
  <dict>
7
7
  <key>Info.plist</key>
8
8
  <data>
9
- Ql9T3+Qa6jsdDzLj+Tl4viO3bss=
9
+ 7DzBrnjIRZFh6pNXNMyPsSbc+O0=
10
10
  </data>
11
11
  </dict>
12
12
  <key>files2</key>