whisper.rn 0.4.2 → 0.4.3

package/android/src/main/java/com/rnwhisper/RNWhisper.java

@@ -326,7 +326,7 @@ public class RNWhisper implements LifecycleEventListener {
       @Override
       protected Void doInBackground(Void... voids) {
         try {
-          onHostDestroy();
+          releaseAllContexts();
         } catch (Exception e) {
           exception = e;
         }
@@ -529,10 +529,7 @@ public class RNWhisper implements LifecycleEventListener {
       @Override
       protected Void doInBackground(Void... voids) {
         try {
-          for (WhisperVadContext vadContext : vadContexts.values()) {
-            vadContext.release();
-          }
-          vadContexts.clear();
+          releaseAllVadContexts();
         } catch (Exception e) {
           exception = e;
         }
@@ -560,27 +557,35 @@ public class RNWhisper implements LifecycleEventListener {
   public void onHostPause() {
   }
 
-  @Override
-  public void onHostDestroy() {
+  private void releaseAllContexts() {
     for (WhisperContext context : contexts.values()) {
       context.stopCurrentTranscribe();
     }
-    for (AsyncTask task : tasks.keySet()) {
-      try {
-        task.get();
-      } catch (Exception e) {
-        Log.e(NAME, "Failed to wait for task", e);
-      }
-    }
+    WhisperContext.abortAllTranscribe(); // graceful abort
     for (WhisperContext context : contexts.values()) {
       context.release();
     }
+    contexts.clear();
+  }
+
+  private void releaseAllVadContexts() {
     for (WhisperVadContext vadContext : vadContexts.values()) {
       vadContext.release();
     }
-    WhisperContext.abortAllTranscribe(); // graceful abort
-    contexts.clear();
     vadContexts.clear();
+  }
+
+  @Override
+  public void onHostDestroy() {
+    for (AsyncTask task : tasks.keySet()) {
+      try {
+        task.get();
+      } catch (Exception e) {
+        Log.e(NAME, "Failed to wait for task", e);
+      }
+    }
     downloader.clearCache();
+    releaseAllContexts();
+    releaseAllVadContexts();
   }
 }

package/cpp/ggml-backend.cpp

@@ -817,8 +817,9 @@ static void wsp_ggml_backend_sched_print_assignments(wsp_ggml_backend_sched_t sc
         }
         if (sched->debug > 1) {
             wsp_ggml_backend_t tensor_backend = wsp_ggml_backend_sched_get_tensor_backend(sched, node);
-            WSP_GGML_LOG_DEBUG("node #%3d (%10.10s): %20.20s (%5.5s) [%5.5s %8.8s]:", i, wsp_ggml_op_name(node->op), node->name,
-                fmt_size(wsp_ggml_nbytes(node)), tensor_backend ? wsp_ggml_backend_name(tensor_backend) : "NULL", GET_CAUSE(node));
+            WSP_GGML_LOG_DEBUG("node #%3d (%10.10s): %20.20s (%5.5s) [%5.5s %8.8s] use=%d:", i, wsp_ggml_op_name(node->op), node->name,
+                fmt_size(wsp_ggml_nbytes(node)), tensor_backend ? wsp_ggml_backend_name(tensor_backend) : "NULL", GET_CAUSE(node),
+                graph->use_counts[wsp_ggml_hash_find(&graph->visited_hash_set, node)]);
             for (int j = 0; j < WSP_GGML_MAX_SRC; j++) {
                 struct wsp_ggml_tensor * src = node->src[j];
                 if (src == NULL) {
@@ -1826,7 +1827,7 @@ void wsp_ggml_backend_graph_copy_free(struct wsp_ggml_backend_graph_copy copy) {
     wsp_ggml_free(copy.ctx_unallocated);
 }
 
-bool wsp_ggml_backend_compare_graph_backend(wsp_ggml_backend_t backend1, wsp_ggml_backend_t backend2, struct wsp_ggml_cgraph * graph, wsp_ggml_backend_eval_callback callback, void * user_data) {
+bool wsp_ggml_backend_compare_graph_backend(wsp_ggml_backend_t backend1, wsp_ggml_backend_t backend2, struct wsp_ggml_cgraph * graph, wsp_ggml_backend_eval_callback callback, void * user_data, struct wsp_ggml_tensor * test_node) {
     struct wsp_ggml_backend_graph_copy copy = wsp_ggml_backend_graph_copy(backend2, graph);
     if (copy.buffer == NULL) {
         return false;
@@ -1837,28 +1838,45 @@ bool wsp_ggml_backend_compare_graph_backend(wsp_ggml_backend_t backend1, wsp_ggm
 
     assert(g1->n_nodes == g2->n_nodes);
 
-    for (int i = 0; i < g1->n_nodes; i++) {
-        struct wsp_ggml_tensor * t1 = g1->nodes[i];
-        struct wsp_ggml_tensor * t2 = g2->nodes[i];
+    if (test_node != nullptr) {
+        // Compute the whole graph and only test the output for a specific tensor
+        wsp_ggml_backend_graph_compute(backend1, g1);
+        wsp_ggml_backend_graph_compute(backend2, g2);
 
-        assert(t1->op == t2->op && wsp_ggml_are_same_layout(t1, t2));
+        int test_node_idx = -1;
+        for (int i = 0; i < g1->n_nodes; i++) {
+            struct wsp_ggml_tensor * t1 = g1->nodes[i];
+            if (t1 == test_node) {
+                test_node_idx = i;
+                break;
+            }
+        }
+        WSP_GGML_ASSERT(test_node_idx != -1);
 
-        struct wsp_ggml_cgraph g1v = wsp_ggml_graph_view(g1, i, i + 1);
-        struct wsp_ggml_cgraph g2v = wsp_ggml_graph_view(g2, i, i + 1);
+        callback(test_node_idx, g1->nodes[test_node_idx], g2->nodes[test_node_idx], user_data);
+    } else {
+        for (int i = 0; i < g1->n_nodes; i++) {
+            struct wsp_ggml_tensor * t1 = g1->nodes[i];
+            struct wsp_ggml_tensor * t2 = g2->nodes[i];
 
-        wsp_ggml_backend_graph_compute(backend1, &g1v);
-        wsp_ggml_backend_graph_compute(backend2, &g2v);
+            assert(t1->op == t2->op && wsp_ggml_are_same_layout(t1, t2));
 
-        if (wsp_ggml_is_view_op(t1->op)) {
-            continue;
-        }
+            struct wsp_ggml_cgraph g1v = wsp_ggml_graph_view(g1, i, i + 1);
+            struct wsp_ggml_cgraph g2v = wsp_ggml_graph_view(g2, i, i + 1);
 
-        // compare results, calculate rms etc
-        if (!callback(i, t1, t2, user_data)) {
-            break;
+            wsp_ggml_backend_graph_compute(backend1, &g1v);
+            wsp_ggml_backend_graph_compute(backend2, &g2v);
+
+            if (wsp_ggml_is_view_op(t1->op)) {
+                continue;
+            }
+
+            // compare results, calculate rms etc
+            if (!callback(i, t1, t2, user_data)) {
+                break;
+            }
         }
     }
-
     wsp_ggml_backend_graph_copy_free(copy);
 
     return true;

package/cpp/ggml-backend.h

@@ -339,7 +339,7 @@ extern "C" {
     typedef bool (*wsp_ggml_backend_eval_callback)(int node_index, struct wsp_ggml_tensor * t1, struct wsp_ggml_tensor * t2, void * user_data);
 
     // Compare the output of two backends
-    WSP_GGML_API bool wsp_ggml_backend_compare_graph_backend(wsp_ggml_backend_t backend1, wsp_ggml_backend_t backend2, struct wsp_ggml_cgraph * graph, wsp_ggml_backend_eval_callback callback, void * user_data);
+    WSP_GGML_API bool wsp_ggml_backend_compare_graph_backend(wsp_ggml_backend_t backend1, wsp_ggml_backend_t backend2, struct wsp_ggml_cgraph * graph, wsp_ggml_backend_eval_callback callback, void * user_data, struct wsp_ggml_tensor * test_node);
 
     // Tensor initialization
     WSP_GGML_API enum wsp_ggml_status wsp_ggml_backend_tensor_alloc(wsp_ggml_backend_buffer_t buffer, struct wsp_ggml_tensor * tensor, void * addr);

package/cpp/ggml-cpu/amx/mmq.cpp

@@ -8,6 +8,7 @@
 #include "mmq.h"
 #include "ggml-impl.h"
 #include "ggml-cpu-impl.h"
+#include "simd-mappings.h"
 #include "quants.h"
 #include "ggml-quants.h"
 #include <algorithm>
@@ -453,7 +454,7 @@ void wsp_quantize_row_q8_K_vnni(const float * RESTRICT x, void * RESTRICT vy, in
 
         // Quantize these floats
         const float iscale = 127.f / amax;
-        y[i].d = WSP_GGML_FP32_TO_FP16(1 / iscale);
+        y[i].d = WSP_GGML_CPU_FP32_TO_FP16(1 / iscale);
         const float id = ( amax != 0.0f ) ? iscale : 0.f;
         const __m512 vscale = _mm512_set1_ps(id);
 
@@ -1090,7 +1091,7 @@ struct acc_C<block_q8_0, block_q4_0, is_acc> {
         const __m512 vd0 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)((const char *)packed_B + offset)));
 
         for (int m = 0; m < nr; ++m) {
-            const __m512 vd1 = _mm512_set1_ps(WSP_GGML_FP16_TO_FP32(A[m * lda].d));
+            const __m512 vd1 = _mm512_set1_ps(WSP_GGML_CPU_FP16_TO_FP32(A[m * lda].d));
             const __m512 vtile = _mm512_cvtepi32_ps(_mm512_loadu_si512(tile + m * TILE_N));
 
             __m512 vsum;
@@ -1113,8 +1114,8 @@ struct acc_C<block_q8_1, block_q4_1, is_acc> {
         const __m512 vm0 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)((const char *)packed_B + offset + TILE_N * sizeof(wsp_ggml_half))));
 
         for (int m = 0; m < nr; ++m) {
-            const __m512 vd1 = _mm512_set1_ps(WSP_GGML_FP16_TO_FP32(A[m * lda].d));
-            const __m512 vs1 = _mm512_set1_ps(WSP_GGML_FP16_TO_FP32(A[m * lda].s));
+            const __m512 vd1 = _mm512_set1_ps(WSP_GGML_CPU_FP16_TO_FP32(A[m * lda].d));
+            const __m512 vs1 = _mm512_set1_ps(WSP_GGML_CPU_FP16_TO_FP32(A[m * lda].s));
             const __m512 vtile = _mm512_cvtepi32_ps(_mm512_loadu_si512(tile + m * TILE_N));
 
             __m512 vsum;
@@ -1137,7 +1138,7 @@ struct acc_C<block_q8_0, block_q8_0, is_acc> {
         const __m512 vd0 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)((const char *)packed_B + offset)));
 
         for (int m = 0; m < nr; ++m) {
-            const __m512 vd1 = _mm512_set1_ps(WSP_GGML_FP16_TO_FP32(A[m * lda].d));
+            const __m512 vd1 = _mm512_set1_ps(WSP_GGML_CPU_FP16_TO_FP32(A[m * lda].d));
             const __m512 vtile = _mm512_cvtepi32_ps(_mm512_loadu_si512(tile + m * TILE_N));
 
             __m512 vsum;
@@ -1437,7 +1438,7 @@ struct tinygemm_kernel_vnni<block_q8_0, block_q4_0, float, BLOCK_M, BLOCK_N, BLO
                     va[k] = _mm512_set1_epi32(a_ptr[k]);
                     vcomp = _mm512_dpbusd_epi32(vcomp, off, va[k]);
                 }
-                vd1 = _mm512_set1_ps(WSP_GGML_FP16_TO_FP32(A[0 * KB + i].d));
+                vd1 = _mm512_set1_ps(WSP_GGML_CPU_FP16_TO_FP32(A[0 * KB + i].d));
             }
 
             // load b
@@ -1498,8 +1499,8 @@ struct tinygemm_kernel_vnni<block_q8_1, block_q4_1, float, 1, BLOCK_N, BLOCK_K>
                 for (int k = 0; k < 8; ++k) {
                     va[k] = _mm512_set1_epi32(a_ptr[k]);
                 }
-                vd1 = _mm512_set1_ps(WSP_GGML_FP16_TO_FP32(A[0 * KB + i].d));
-                vs1 = _mm512_set1_ps(WSP_GGML_FP16_TO_FP32(A[0 * KB + i].s));
+                vd1 = _mm512_set1_ps(WSP_GGML_CPU_FP16_TO_FP32(A[0 * KB + i].d));
+                vs1 = _mm512_set1_ps(WSP_GGML_CPU_FP16_TO_FP32(A[0 * KB + i].s));
             }
 
             // load b
@@ -1571,7 +1572,7 @@ struct tinygemm_kernel_vnni<block_q8_0, block_q8_0, float, BLOCK_M, BLOCK_N, BLO
                     va[k] = _mm512_set1_epi32(a_ptr[k]);
                     va[k] = _mm512_add_epi8(va[k], off);
                 }
-                vd1 = _mm512_set1_ps(WSP_GGML_FP16_TO_FP32(A[0 * KB + i].d));
+                vd1 = _mm512_set1_ps(WSP_GGML_CPU_FP16_TO_FP32(A[0 * KB + i].d));
             }
 
             // load b