whisper.rn 0.5.4 → 0.5.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. package/android/src/main/java/com/rnwhisper/WhisperContext.java +5 -0
  2. package/android/src/main/jni.cpp +13 -0
  3. package/cpp/ggml-alloc.c +78 -26
  4. package/cpp/ggml-alloc.h +9 -0
  5. package/cpp/ggml-backend-impl.h +1 -1
  6. package/cpp/ggml-backend-reg.cpp +19 -3
  7. package/cpp/ggml-backend.cpp +72 -20
  8. package/cpp/ggml-backend.h +2 -1
  9. package/cpp/ggml-cpu/arch/arm/cpu-feats.cpp +4 -0
  10. package/cpp/ggml-cpu/arch/arm/repack.cpp +1004 -0
  11. package/cpp/ggml-cpu/arch/x86/repack.cpp +6 -6
  12. package/cpp/ggml-cpu/arch-fallback.h +50 -2
  13. package/cpp/ggml-cpu/ggml-cpu-impl.h +1 -1
  14. package/cpp/ggml-cpu/ggml-cpu.c +139 -58
  15. package/cpp/ggml-cpu/ggml-cpu.cpp +4 -0
  16. package/cpp/ggml-cpu/ops.cpp +170 -18
  17. package/cpp/ggml-cpu/ops.h +1 -0
  18. package/cpp/ggml-cpu/repack.cpp +531 -5
  19. package/cpp/ggml-cpu/repack.h +14 -0
  20. package/cpp/ggml-cpu/simd-mappings.h +16 -18
  21. package/cpp/ggml-cpu/vec.cpp +41 -1
  22. package/cpp/ggml-cpu/vec.h +241 -138
  23. package/cpp/ggml-cpu.h +1 -0
  24. package/cpp/ggml-impl.h +0 -4
  25. package/cpp/ggml-metal/ggml-metal-context.m +26 -16
  26. package/cpp/ggml-metal/ggml-metal-device.cpp +452 -371
  27. package/cpp/ggml-metal/ggml-metal-device.h +87 -65
  28. package/cpp/ggml-metal/ggml-metal-device.m +263 -104
  29. package/cpp/ggml-metal/ggml-metal-impl.h +58 -4
  30. package/cpp/ggml-metal/ggml-metal-ops.cpp +415 -98
  31. package/cpp/ggml-metal/ggml-metal-ops.h +4 -0
  32. package/cpp/ggml-metal/ggml-metal.cpp +6 -5
  33. package/cpp/ggml-metal/ggml-metal.metal +404 -34
  34. package/cpp/ggml.c +110 -31
  35. package/cpp/ggml.h +51 -12
  36. package/cpp/jsi/RNWhisperJSI.cpp +1 -0
  37. package/cpp/whisper.cpp +16 -3
  38. package/ios/CMakeLists.txt +21 -1
  39. package/ios/RNWhisperContext.mm +5 -0
  40. package/ios/rnwhisper.xcframework/ios-arm64/rnwhisper.framework/Headers/ggml-alloc.h +9 -0
  41. package/ios/rnwhisper.xcframework/ios-arm64/rnwhisper.framework/Headers/ggml-backend-impl.h +1 -1
  42. package/ios/rnwhisper.xcframework/ios-arm64/rnwhisper.framework/Headers/ggml-backend.h +2 -1
  43. package/ios/rnwhisper.xcframework/ios-arm64/rnwhisper.framework/Headers/ggml-cpu.h +1 -0
  44. package/ios/rnwhisper.xcframework/ios-arm64/rnwhisper.framework/Headers/ggml-impl.h +0 -4
  45. package/ios/rnwhisper.xcframework/ios-arm64/rnwhisper.framework/Headers/ggml.h +51 -12
  46. package/ios/rnwhisper.xcframework/ios-arm64/rnwhisper.framework/Info.plist +0 -0
  47. package/ios/rnwhisper.xcframework/ios-arm64/rnwhisper.framework/ggml-metal.metal +404 -34
  48. package/ios/rnwhisper.xcframework/ios-arm64/rnwhisper.framework/rnwhisper +0 -0
  49. package/ios/rnwhisper.xcframework/ios-arm64_x86_64-simulator/rnwhisper.framework/Headers/ggml-alloc.h +9 -0
  50. package/ios/rnwhisper.xcframework/ios-arm64_x86_64-simulator/rnwhisper.framework/Headers/ggml-backend-impl.h +1 -1
  51. package/ios/rnwhisper.xcframework/ios-arm64_x86_64-simulator/rnwhisper.framework/Headers/ggml-backend.h +2 -1
  52. package/ios/rnwhisper.xcframework/ios-arm64_x86_64-simulator/rnwhisper.framework/Headers/ggml-cpu.h +1 -0
  53. package/ios/rnwhisper.xcframework/ios-arm64_x86_64-simulator/rnwhisper.framework/Headers/ggml-impl.h +0 -4
  54. package/ios/rnwhisper.xcframework/ios-arm64_x86_64-simulator/rnwhisper.framework/Headers/ggml.h +51 -12
  55. package/ios/rnwhisper.xcframework/ios-arm64_x86_64-simulator/rnwhisper.framework/Info.plist +0 -0
  56. package/ios/rnwhisper.xcframework/ios-arm64_x86_64-simulator/rnwhisper.framework/_CodeSignature/CodeResources +1 -1
  57. package/ios/rnwhisper.xcframework/ios-arm64_x86_64-simulator/rnwhisper.framework/ggml-metal.metal +404 -34
  58. package/ios/rnwhisper.xcframework/ios-arm64_x86_64-simulator/rnwhisper.framework/rnwhisper +0 -0
  59. package/ios/rnwhisper.xcframework/tvos-arm64/rnwhisper.framework/Headers/ggml-alloc.h +9 -0
  60. package/ios/rnwhisper.xcframework/tvos-arm64/rnwhisper.framework/Headers/ggml-backend-impl.h +1 -1
  61. package/ios/rnwhisper.xcframework/tvos-arm64/rnwhisper.framework/Headers/ggml-backend.h +2 -1
  62. package/ios/rnwhisper.xcframework/tvos-arm64/rnwhisper.framework/Headers/ggml-cpu.h +1 -0
  63. package/ios/rnwhisper.xcframework/tvos-arm64/rnwhisper.framework/Headers/ggml-impl.h +0 -4
  64. package/ios/rnwhisper.xcframework/tvos-arm64/rnwhisper.framework/Headers/ggml.h +51 -12
  65. package/ios/rnwhisper.xcframework/tvos-arm64/rnwhisper.framework/Info.plist +0 -0
  66. package/ios/rnwhisper.xcframework/tvos-arm64/rnwhisper.framework/ggml-metal.metal +404 -34
  67. package/ios/rnwhisper.xcframework/tvos-arm64/rnwhisper.framework/rnwhisper +0 -0
  68. package/ios/rnwhisper.xcframework/tvos-arm64_x86_64-simulator/rnwhisper.framework/Headers/ggml-alloc.h +9 -0
  69. package/ios/rnwhisper.xcframework/tvos-arm64_x86_64-simulator/rnwhisper.framework/Headers/ggml-backend-impl.h +1 -1
  70. package/ios/rnwhisper.xcframework/tvos-arm64_x86_64-simulator/rnwhisper.framework/Headers/ggml-backend.h +2 -1
  71. package/ios/rnwhisper.xcframework/tvos-arm64_x86_64-simulator/rnwhisper.framework/Headers/ggml-cpu.h +1 -0
  72. package/ios/rnwhisper.xcframework/tvos-arm64_x86_64-simulator/rnwhisper.framework/Headers/ggml-impl.h +0 -4
  73. package/ios/rnwhisper.xcframework/tvos-arm64_x86_64-simulator/rnwhisper.framework/Headers/ggml.h +51 -12
  74. package/ios/rnwhisper.xcframework/tvos-arm64_x86_64-simulator/rnwhisper.framework/Info.plist +0 -0
  75. package/ios/rnwhisper.xcframework/tvos-arm64_x86_64-simulator/rnwhisper.framework/_CodeSignature/CodeResources +1 -1
  76. package/ios/rnwhisper.xcframework/tvos-arm64_x86_64-simulator/rnwhisper.framework/ggml-metal.metal +404 -34
  77. package/ios/rnwhisper.xcframework/tvos-arm64_x86_64-simulator/rnwhisper.framework/rnwhisper +0 -0
  78. package/lib/commonjs/NativeRNWhisper.js.map +1 -1
  79. package/lib/commonjs/jest-mock.js +2 -0
  80. package/lib/commonjs/jest-mock.js.map +1 -1
  81. package/lib/commonjs/version.json +1 -1
  82. package/lib/module/NativeRNWhisper.js.map +1 -1
  83. package/lib/module/jest-mock.js +2 -0
  84. package/lib/module/jest-mock.js.map +1 -1
  85. package/lib/module/version.json +1 -1
  86. package/lib/typescript/NativeRNWhisper.d.ts +1 -0
  87. package/lib/typescript/NativeRNWhisper.d.ts.map +1 -1
  88. package/package.json +1 -1
  89. package/src/NativeRNWhisper.ts +1 -0
  90. package/src/jest-mock.ts +2 -0
  91. package/src/version.json +1 -1
package/cpp/ggml-metal/ggml-metal-device.h CHANGED
@@ -35,20 +35,6 @@ typedef struct wsp_ggml_metal_pipeline * wsp_ggml_metal_pipeline_t;
35
35
  wsp_ggml_metal_pipeline_t wsp_ggml_metal_pipeline_init(void);
36
36
  void wsp_ggml_metal_pipeline_free(wsp_ggml_metal_pipeline_t pipeline);
37
37
 
38
- void wsp_ggml_metal_pipeline_set_nsg(wsp_ggml_metal_pipeline_t pipeline, int nsg);
39
- int wsp_ggml_metal_pipeline_get_nsg(wsp_ggml_metal_pipeline_t pipeline);
40
-
41
- void wsp_ggml_metal_pipeline_set_nr0(wsp_ggml_metal_pipeline_t pipeline, int nr0);
42
- int wsp_ggml_metal_pipeline_get_nr0(wsp_ggml_metal_pipeline_t pipeline);
43
-
44
- void wsp_ggml_metal_pipeline_set_nr1(wsp_ggml_metal_pipeline_t pipeline, int nr1);
45
- int wsp_ggml_metal_pipeline_get_nr1(wsp_ggml_metal_pipeline_t pipeline);
46
-
47
- void wsp_ggml_metal_pipeline_set_smem(wsp_ggml_metal_pipeline_t pipeline, size_t smem);
48
- size_t wsp_ggml_metal_pipeline_get_smem(wsp_ggml_metal_pipeline_t pipeline);
49
-
50
- int wsp_ggml_metal_pipeline_max_theads_per_threadgroup(wsp_ggml_metal_pipeline_t pipeline);
51
-
52
38
  // a collection of pipelines
53
39
  typedef struct wsp_ggml_metal_pipelines * wsp_ggml_metal_pipelines_t;
54
40
 
@@ -58,6 +44,19 @@ void wsp_ggml_metal_pipelines_free(wsp_ggml_metal_pipelines_t ppls);
58
44
  void wsp_ggml_metal_pipelines_add(wsp_ggml_metal_pipelines_t ppls, const char * name, wsp_ggml_metal_pipeline_t pipeline);
59
45
  wsp_ggml_metal_pipeline_t wsp_ggml_metal_pipelines_get(wsp_ggml_metal_pipelines_t ppls, const char * name);
60
46
 
47
+ struct wsp_ggml_metal_pipeline_with_params {
48
+ wsp_ggml_metal_pipeline_t pipeline;
49
+
50
+ int nsg;
51
+
52
+ int nr0;
53
+ int nr1;
54
+
55
+ size_t smem;
56
+ };
57
+
58
+ int wsp_ggml_metal_pipeline_max_theads_per_threadgroup(struct wsp_ggml_metal_pipeline_with_params pipeline);
59
+
61
60
  //
62
61
  // MTLCommandBuffer wrapper
63
62
  //
@@ -76,7 +75,7 @@ void wsp_ggml_metal_encoder_free(wsp_ggml_metal_encoder_t encoder);
76
75
  void wsp_ggml_metal_encoder_debug_group_push(wsp_ggml_metal_encoder_t encoder, const char * name);
77
76
  void wsp_ggml_metal_encoder_debug_group_pop (wsp_ggml_metal_encoder_t encoder);
78
77
 
79
- void wsp_ggml_metal_encoder_set_pipeline(wsp_ggml_metal_encoder_t encoder, wsp_ggml_metal_pipeline_t pipeline);
78
+ void wsp_ggml_metal_encoder_set_pipeline(wsp_ggml_metal_encoder_t encoder, struct wsp_ggml_metal_pipeline_with_params pipeline);
80
79
 
81
80
  void wsp_ggml_metal_encoder_set_bytes (wsp_ggml_metal_encoder_t encoder, void * data, size_t size, int idx);
82
81
  void wsp_ggml_metal_encoder_set_buffer(wsp_ggml_metal_encoder_t encoder, struct wsp_ggml_metal_buffer_id buffer, int idx);
@@ -100,64 +99,70 @@ wsp_ggml_metal_library_t wsp_ggml_metal_library_init_from_source(wsp_ggml_metal_
100
99
 
101
100
  void wsp_ggml_metal_library_free(wsp_ggml_metal_library_t lib);
102
101
 
103
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline (wsp_ggml_metal_library_t lib, const char * name);
104
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_compile_pipeline(wsp_ggml_metal_library_t lib, const char * base, const char * name, wsp_ggml_metal_cv_t cv);
105
-
106
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_base (wsp_ggml_metal_library_t lib, enum wsp_ggml_op op);
107
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_cpy (wsp_ggml_metal_library_t lib, enum wsp_ggml_type tsrc, enum wsp_ggml_type tdst);
108
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_pool_2d (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op, enum wsp_ggml_op_pool op_pool);
109
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_get_rows (wsp_ggml_metal_library_t lib, enum wsp_ggml_type tsrc);
110
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_set_rows (wsp_ggml_metal_library_t lib, enum wsp_ggml_type tidx, enum wsp_ggml_type tdst);
111
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_repeat (wsp_ggml_metal_library_t lib, enum wsp_ggml_type tsrc);
112
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_unary (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
113
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_glu (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
114
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_sum (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
115
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_sum_rows (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
116
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_cumsum_blk (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
117
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_cumsum_add (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
118
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_soft_max (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
119
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_ssm_conv (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
120
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_ssm_scan (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
121
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_rwkv (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
122
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_mul_mv_ext (wsp_ggml_metal_library_t lib, enum wsp_ggml_type tsrc0, enum wsp_ggml_type tsrc1, int nsg, int nxpsg, int r1ptg);
123
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_mul_mm (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
124
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_mul_mv (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
125
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_mul_mm_id_map0 (wsp_ggml_metal_library_t lib, int ne02, int ne20);
126
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_mul_mm_id (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
127
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_mul_mv_id (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
128
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_argmax (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
129
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_argsort (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
130
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_argsort_merge (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
131
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_bin (wsp_ggml_metal_library_t lib, enum wsp_ggml_op op, int32_t n_fuse, bool row);
132
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_l2_norm (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
133
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_group_norm (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
134
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_norm (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op, int32_t n_fuse);
135
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_rope (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
136
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_im2col (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
137
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_conv_transpose_1d (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
138
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_conv_transpose_2d (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
139
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_conv_2d (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
140
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_upscale (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
141
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_pad (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
142
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_pad_reflect_1d (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
143
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_arange (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
144
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_timestep_embedding(wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
145
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_opt_step_adamw (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
146
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_opt_step_sgd (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
147
-
148
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_flash_attn_ext_pad(
102
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline (wsp_ggml_metal_library_t lib, const char * name);
103
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_compile_pipeline(wsp_ggml_metal_library_t lib, const char * base, const char * name, wsp_ggml_metal_cv_t cv);
104
+
105
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_base (wsp_ggml_metal_library_t lib, enum wsp_ggml_op op);
106
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_cpy (wsp_ggml_metal_library_t lib, enum wsp_ggml_type tsrc, enum wsp_ggml_type tdst);
107
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_pool_2d (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op, enum wsp_ggml_op_pool op_pool);
108
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_get_rows (wsp_ggml_metal_library_t lib, enum wsp_ggml_type tsrc);
109
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_set_rows (wsp_ggml_metal_library_t lib, enum wsp_ggml_type tidx, enum wsp_ggml_type tdst);
110
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_repeat (wsp_ggml_metal_library_t lib, enum wsp_ggml_type tsrc);
111
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_unary (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
112
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_glu (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
113
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_sum (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
114
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_sum_rows (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
115
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_cumsum_blk (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
116
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_cumsum_add (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
117
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_tri (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
118
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_soft_max (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
119
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_ssm_conv (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
120
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_ssm_conv_batched (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op, int ssm_conv_bs);
121
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_ssm_scan (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
122
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_rwkv (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
123
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_mul_mv_ext (wsp_ggml_metal_library_t lib, enum wsp_ggml_type tsrc0, enum wsp_ggml_type tsrc1, int nsg, int nxpsg, int r1ptg);
124
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_mul_mm (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
125
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_mul_mv (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
126
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_mul_mm_id_map0 (wsp_ggml_metal_library_t lib, int ne02, int ne20);
127
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_mul_mm_id (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
128
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_mul_mv_id (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
129
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_argmax (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
130
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_argsort (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
131
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_argsort_merge (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
132
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_top_k (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
133
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_top_k_merge (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
134
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_bin (wsp_ggml_metal_library_t lib, enum wsp_ggml_op op, int32_t n_fuse, bool row);
135
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_l2_norm (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
136
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_group_norm (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
137
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_norm (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op, int32_t n_fuse);
138
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_rope (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
139
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_im2col (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
140
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_conv_transpose_1d (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
141
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_conv_transpose_2d (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
142
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_conv_2d (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
143
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_upscale (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
144
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_pad (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
145
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_pad_reflect_1d (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
146
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_arange (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
147
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_timestep_embedding(wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
148
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_opt_step_adamw (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
149
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_opt_step_sgd (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
150
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_memset (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
151
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_count_equal (wsp_ggml_metal_library_t lib, const struct wsp_ggml_tensor * op);
152
+
153
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_flash_attn_ext_pad(
149
154
  wsp_ggml_metal_library_t lib,
150
155
  const struct wsp_ggml_tensor * op,
151
156
  bool has_mask,
152
157
  int32_t ncpsg);
153
158
 
154
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_flash_attn_ext_blk(
159
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_flash_attn_ext_blk(
155
160
  wsp_ggml_metal_library_t lib,
156
161
  const struct wsp_ggml_tensor * op,
157
162
  int32_t nqptg,
158
163
  int32_t ncpsg);
159
164
 
160
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_flash_attn_ext(
165
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_flash_attn_ext(
161
166
  wsp_ggml_metal_library_t lib,
162
167
  const struct wsp_ggml_tensor * op,
163
168
  bool has_mask,
@@ -167,7 +172,7 @@ wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_flash_attn_ext(
167
172
  bool has_kvpad,
168
173
  int32_t nsg);
169
174
 
170
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_flash_attn_ext_vec(
175
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_flash_attn_ext_vec(
171
176
  wsp_ggml_metal_library_t lib,
172
177
  const struct wsp_ggml_tensor * op,
173
178
  bool has_mask,
@@ -178,12 +183,22 @@ wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_flash_attn_ext_vec
178
183
  int32_t nsg,
179
184
  int32_t nwg);
180
185
 
181
- wsp_ggml_metal_pipeline_t wsp_ggml_metal_library_get_pipeline_flash_attn_ext_vec_reduce(
186
+ struct wsp_ggml_metal_pipeline_with_params wsp_ggml_metal_library_get_pipeline_flash_attn_ext_vec_reduce(
182
187
  wsp_ggml_metal_library_t lib,
183
188
  const struct wsp_ggml_tensor * op,
184
189
  int32_t dv,
185
190
  int32_t nwg);
186
191
 
192
+ // MTLResidencySet wrapper
193
+
194
+ typedef void * wsp_ggml_metal_rset_t;
195
+
196
+ // a collection of residency sets (non-owning)
197
+ typedef struct wsp_ggml_metal_rsets * wsp_ggml_metal_rsets_t;
198
+
199
+ wsp_ggml_metal_rsets_t wsp_ggml_metal_rsets_init(void);
200
+ void wsp_ggml_metal_rsets_free(wsp_ggml_metal_rsets_t rsets);
201
+
187
202
  //
188
203
  // device
189
204
  //
@@ -204,6 +219,8 @@ struct wsp_ggml_metal_device_props {
204
219
  bool use_shared_buffers;
205
220
 
206
221
  bool supports_gpu_family_apple7;
222
+
223
+ int op_offload_min_batch_size;
207
224
  };
208
225
 
209
226
  wsp_ggml_metal_device_t wsp_ggml_metal_device_init(void);
@@ -217,6 +234,11 @@ void * wsp_ggml_metal_device_get_queue(wsp_ggml_metal_device_t dev); // id<MTLCo
217
234
 
218
235
  wsp_ggml_metal_library_t wsp_ggml_metal_device_get_library(wsp_ggml_metal_device_t dev);
219
236
 
237
+ void wsp_ggml_metal_device_rsets_add(wsp_ggml_metal_device_t dev, wsp_ggml_metal_rset_t rset);
238
+ void wsp_ggml_metal_device_rsets_rm (wsp_ggml_metal_device_t dev, wsp_ggml_metal_rset_t rset);
239
+
240
+ void wsp_ggml_metal_device_rsets_keep_alive(wsp_ggml_metal_device_t dev);
241
+
220
242
  void wsp_ggml_metal_device_get_memory(wsp_ggml_metal_device_t dev, size_t * free, size_t * total);
221
243
  bool wsp_ggml_metal_device_supports_op(wsp_ggml_metal_device_t dev, const struct wsp_ggml_tensor * op);
222
244