whisper.rn 0.4.0-rc.9 → 0.4.0

package/cpp/whisper.h

@@ -2,6 +2,7 @@
 #define WHISPER_H
 
 #include "ggml.h"
+#include "ggml-cpu.h"
 
 #include <stddef.h>
 #include <stdint.h>
@@ -189,6 +190,15 @@ extern "C" {
         uint32_t             value; // Unicode code point or rule ID
     } whisper_grammar_element;
 
+    typedef struct whisper_vad_params {
+        float threshold;               // Probability threshold to consider as speech.
+        int   min_speech_duration_ms;  // Min duration for a valid speech segment.
+        int   min_silence_duration_ms; // Min silence duration to consider speech as ended.
+        float max_speech_duration_s;   // Max duration of a speech segment before forcing a new segment.
+        int   speech_pad_ms;           // Padding added before and after speech segments.
+        float samples_overlap;         // Overlap in seconds when copying audio samples from speech segment.
+    } whisper_vad_params;
+
     // Various functions for loading a ggml whisper model.
     // Allocate (almost) all memory needed for the model.
     // Return NULL on failure
@@ -425,21 +435,11 @@ extern "C" {
 
     // Performance information from the default state.
     struct whisper_timings {
-        int64_t load_us;
-        int64_t t_start_us;
-        int32_t fail_p;
-        int32_t fail_h;
-        int64_t t_mel_us;
-        int32_t n_sample;
-        int32_t n_encode;
-        int32_t n_decode;
-        int32_t n_batchd;
-        int32_t n_prompt;
-        int64_t t_sample_us;
-        int64_t t_encode_us;
-        int64_t t_decode_us;
-        int64_t t_batchd_us;
-        int64_t t_prompt_us;
+        float sample_ms;
+        float encode_ms;
+        float decode_ms;
+        float batchd_ms;
+        float prompt_ms;
     };
     WHISPER_API struct whisper_timings * whisper_get_timings(struct whisper_context * ctx);
     WHISPER_API void whisper_print_timings(struct whisper_context * ctx);
@@ -532,8 +532,8 @@ extern "C" {
         bool detect_language;
 
         // common decoding parameters:
-        bool suppress_blank;    // ref: https://github.com/openai/whisper/blob/f82bc59f5ea234d4b97fb2860842ed38519f7e65/whisper/decoding.py#L89
-        bool suppress_non_speech_tokens; // ref: https://github.com/openai/whisper/blob/7858aa9c08d98f75575035ecd6481f462d66ca27/whisper/tokenizer.py#L224-L253
+        bool suppress_blank; // ref: https://github.com/openai/whisper/blob/f82bc59f5ea234d4b97fb2860842ed38519f7e65/whisper/decoding.py#L89
+        bool suppress_nst;   // non-speech tokens, ref: https://github.com/openai/whisper/blob/7858aa9c08d98f75575035ecd6481f462d66ca27/whisper/tokenizer.py#L224-L253
 
         float temperature;      // initial decoding temperature, ref: https://ai.stackexchange.com/a/32478
         float max_initial_ts;   // ref: https://github.com/openai/whisper/blob/f82bc59f5ea234d4b97fb2860842ed38519f7e65/whisper/decoding.py#L97
@@ -544,7 +544,7 @@ extern "C" {
         float temperature_inc;
         float entropy_thold;    // similar to OpenAI's "compression_ratio_threshold"
         float logprob_thold;
-        float no_speech_thold;  // TODO: not implemented
+        float no_speech_thold;
 
         struct {
             int best_of;    // ref: https://github.com/openai/whisper/blob/f82bc59f5ea234d4b97fb2860842ed38519f7e65/whisper/transcribe.py#L264
@@ -580,11 +580,18 @@ extern "C" {
         size_t                           n_grammar_rules;
         size_t                           i_start_rule;
         float                            grammar_penalty;
+
+        // Voice Activity Detection (VAD) params
+        bool         vad;                         // Enable VAD
+        const char * vad_model_path;              // Path to VAD model
+
+        whisper_vad_params vad_params;
     };
 
     // NOTE: this function allocates memory, and it is the responsibility of the caller to free the pointer - see whisper_free_context_params & whisper_free_params()
     WHISPER_API struct whisper_context_params * whisper_context_default_params_by_ref(void);
     WHISPER_API struct whisper_context_params   whisper_context_default_params       (void);
+
     WHISPER_API struct whisper_full_params * whisper_full_default_params_by_ref(enum whisper_sampling_strategy strategy);
     WHISPER_API struct whisper_full_params   whisper_full_default_params       (enum whisper_sampling_strategy strategy);
 
@@ -662,6 +669,53 @@ extern "C" {
     WHISPER_API float whisper_full_get_token_p           (struct whisper_context * ctx, int i_segment, int i_token);
     WHISPER_API float whisper_full_get_token_p_from_state(struct whisper_state * state, int i_segment, int i_token);
 
+    //
+    // Voice Activity Detection (VAD)
+    //
+
+    struct whisper_vad_context;
+
+    WHISPER_API struct whisper_vad_params whisper_vad_default_params(void);
+
+    struct whisper_vad_context_params {
+        int   n_threads;  // The number of threads to use for processing.
+        bool  use_gpu;
+        int   gpu_device; // CUDA device
+    };
+
+    WHISPER_API struct whisper_vad_context_params whisper_vad_default_context_params(void);
+
+    WHISPER_API struct whisper_vad_context * whisper_vad_init_from_file_with_params(const char * path_model,              struct whisper_vad_context_params params);
+    WHISPER_API struct whisper_vad_context * whisper_vad_init_with_params          (struct whisper_model_loader * loader, struct whisper_vad_context_params params);
+
+    WHISPER_API bool whisper_vad_detect_speech(
+            struct whisper_vad_context * vctx,
+                           const float * samples,
+                                   int   n_samples);
+
+    WHISPER_API int     whisper_vad_n_probs(struct whisper_vad_context * vctx);
+    WHISPER_API float * whisper_vad_probs  (struct whisper_vad_context * vctx);
+
+    struct whisper_vad_segments;
+
+    WHISPER_API struct whisper_vad_segments * whisper_vad_segments_from_probs(
+            struct whisper_vad_context * vctx,
+            struct whisper_vad_params    params);
+
+    WHISPER_API struct whisper_vad_segments * whisper_vad_segments_from_samples(
+            struct whisper_vad_context * vctx,
+            struct whisper_vad_params    params,
+                           const float * samples,
+                                   int   n_samples);
+
+    WHISPER_API int whisper_vad_segments_n_segments(struct whisper_vad_segments * segments);
+
+    WHISPER_API float whisper_vad_segments_get_segment_t0(struct whisper_vad_segments * segments, int i_segment);
+    WHISPER_API float whisper_vad_segments_get_segment_t1(struct whisper_vad_segments * segments, int i_segment);
+
+    WHISPER_API void whisper_vad_free_segments(struct whisper_vad_segments * segments);
+    WHISPER_API void whisper_vad_free         (struct whisper_vad_context  * ctx);
+
     ////////////////////////////////////////////////////////////////////////////
 
     // Temporary helpers needed for exposing ggml interface
@@ -675,6 +729,9 @@ extern "C" {
 
     WHISPER_API void whisper_log_set(wsp_ggml_log_callback log_callback, void * user_data);
 
+    // Get the no_speech probability for the specified segment
+    WHISPER_API float whisper_full_get_segment_no_speech_prob           (struct whisper_context * ctx, int i_segment);
+    WHISPER_API float whisper_full_get_segment_no_speech_prob_from_state(struct whisper_state * state, int i_segment);
 #ifdef __cplusplus
 }
 #endif

package/ios/CMakeLists.txt

@@ -0,0 +1,95 @@
+cmake_minimum_required(VERSION 3.16)
+project(rnwhisper VERSION 1.0.0 LANGUAGES CXX C)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+# iOS specific settings
+set(CMAKE_OSX_DEPLOYMENT_TARGET 13.0)
+set(CMAKE_XCODE_ATTRIBUTE_ENABLE_BITCODE NO)
+
+# Dependencies and compile options
+add_definitions(
+    -DNDEBUG
+    -DO3
+    -DWSP_GGML_USE_CPU
+    -DWSP_GGML_USE_ACCELERATE
+    -DWSP_GGML_USE_METAL
+    -DWSP_GGML_METAL_USE_BF16
+)
+
+if (CMAKE_OSX_ARCHITECTURES STREQUAL "arm64;x86_64")
+    add_definitions(-DWSP_GGML_CPU_GENERIC)
+endif ()
+
+set(SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../cpp)
+
+if (CMAKE_OSX_ARCHITECTURES STREQUAL "arm64")
+    set(SOURCE_FILES_ARCH
+        ${SOURCE_DIR}/ggml-cpu/arch/arm/quants.c
+        ${SOURCE_DIR}/ggml-cpu/arch/arm/repack.cpp
+    )
+endif ()
+
+# Define public headers
+set(PUBLIC_HEADERS
+    ${SOURCE_DIR}/rn-whisper.h
+    ${SOURCE_DIR}/whisper.h
+    ${SOURCE_DIR}/ggml.h
+)
+
+# Create library target
+add_library(rnwhisper SHARED
+    ${SOURCE_DIR}/ggml.c
+    ${SOURCE_DIR}/ggml-alloc.c
+    ${SOURCE_DIR}/ggml-backend.cpp
+    ${SOURCE_DIR}/ggml-backend-reg.cpp
+    ${SOURCE_DIR}/ggml-cpu/amx/amx.cpp
+    ${SOURCE_DIR}/ggml-cpu/amx/mmq.cpp
+    ${SOURCE_DIR}/ggml-cpu/ggml-cpu.c
+    ${SOURCE_DIR}/ggml-cpu/ggml-cpu.cpp
+    ${SOURCE_DIR}/ggml-cpu/quants.c
+    ${SOURCE_DIR}/ggml-cpu/traits.cpp
+    ${SOURCE_DIR}/ggml-cpu/repack.cpp
+    ${SOURCE_DIR}/ggml-cpu/unary-ops.cpp
+    ${SOURCE_DIR}/ggml-cpu/binary-ops.cpp
+    ${SOURCE_DIR}/ggml-cpu/vec.cpp
+    ${SOURCE_DIR}/ggml-cpu/ops.cpp
+    ${SOURCE_DIR}/ggml-metal.m
+    ${SOURCE_DIR}/ggml-opt.cpp
+    ${SOURCE_DIR}/ggml-threading.cpp
+    ${SOURCE_DIR}/ggml-quants.c
+    ${SOURCE_DIR}/gguf.cpp
+    ${SOURCE_DIR}/whisper.cpp
+    ${SOURCE_DIR}/rn-whisper.cpp
+    ${SOURCE_DIR}/rn-audioutils.cpp
+    ${SOURCE_FILES_ARCH}
+)
+
+# Setup include directories
+target_include_directories(rnwhisper
+    PUBLIC
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../cpp>
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../cpp/ggml-cpu>
+        $<INSTALL_INTERFACE:include>
+)
+
+# Link required frameworks
+target_link_libraries(rnwhisper PRIVATE
+    "-framework Accelerate"
+    "-framework Foundation"
+    "-framework Metal"
+    "-framework MetalKit"
+)
+
+# Set properties for framework
+set_target_properties(rnwhisper PROPERTIES
+    MACOSX_FRAMEWORK_IDENTIFIER "com.rnwhisper"
+    MACOSX_FRAMEWORK_BUNDLE_VERSION 1.0.0
+    MACOSX_FRAMEWORK_SHORT_VERSION_STRING 1.0.0
+    FRAMEWORK TRUE
+    FRAMEWORK_VERSION 1.0.0
+    VERSION 1.0.0
+    PUBLIC_HEADER "${PUBLIC_HEADERS}"
+    XCODE_ATTRIBUTE_CLANG_ENABLE_OBJC_ARC NO
+)

package/ios/RNWhisper.h

@@ -1,6 +1,11 @@
 #ifdef __cplusplus
+#if RNWHISPER_BUILD_FROM_SOURCE
 #import "whisper.h"
 #import "rn-whisper.h"
+#else
+#import <rnwhisper/whisper.h>
+#import <rnwhisper/rn-whisper.h>
+#endif
 #endif
 
 #import <React/RCTBridgeModule.h>

package/ios/RNWhisperAudioUtils.m

@@ -1,5 +1,9 @@
 #import "RNWhisperAudioUtils.h"
+#if RNWHISPER_BUILD_FROM_SOURCE
 #import "whisper.h"
+#else
+#import <rnwhisper/whisper.h>
+#endif
 
 @implementation RNWhisperAudioUtils
 

package/ios/RNWhisperContext.h

@@ -1,6 +1,11 @@
 #ifdef __cplusplus
+#if RNWHISPER_BUILD_FROM_SOURCE
 #import "whisper.h"
 #import "rn-whisper.h"
+#else
+#import <rnwhisper/whisper.h>
+#import <rnwhisper/rn-whisper.h>
+#endif
 #endif
 
 #import <AVFoundation/AVFoundation.h>

package/ios/RNWhisperContext.mm

@@ -19,8 +19,9 @@
     cparams.use_gpu = !noMetal;
     cparams.flash_attn = useFlashAttn;
 
-    // TODO: Figure out why it leads to re-init crash
+    // TODO: Expose dtw_token_timestamps and dtw_aheads_preset
     cparams.dtw_token_timestamps = false;
+    // cparams.dtw_aheads_preset = WHISPER_AHEADS_BASE;
 
     cparams.use_coreml = !noCoreML;
 #ifndef WHISPER_USE_COREML
@@ -431,6 +432,7 @@ struct rnwhisper_segments_callback_data {
         self->recordState.job = job;
         int code = [self fullTranscribe:job audioData:audioData audioDataCount:audioDataCount];
         rnwhisper::job_remove(jobId);
+        self->recordState.job = nullptr;
         self->recordState.isTranscribing = false;
         onEnd(code);
     });
@@ -445,7 +447,7 @@ struct rnwhisper_segments_callback_data {
 }
 
 - (void)stopTranscribe:(int)jobId {
-    if (self->recordState.job) self->recordState.job->abort();
+    if (self->recordState.job != nullptr) self->recordState.job->abort();
     if (self->recordState.isRealtime && self->recordState.isCapturing) {
         [self stopAudio];
         if (!self->recordState.isTranscribing) {

package/ios/rnwhisper.xcframework/Info.plist

@@ -0,0 +1,74 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>AvailableLibraries</key>
+	<array>
+		<dict>
+			<key>LibraryIdentifier</key>
+			<string>ios-arm64</string>
+			<key>LibraryPath</key>
+			<string>rnwhisper.framework</string>
+			<key>SupportedArchitectures</key>
+			<array>
+				<string>arm64</string>
+			</array>
+			<key>SupportedPlatform</key>
+			<string>ios</string>
+		</dict>
+		<dict>
+			<key>LibraryIdentifier</key>
+			<string>ios-arm64_x86_64-simulator</string>
+			<key>LibraryPath</key>
+			<string>rnwhisper.framework</string>
+			<key>SupportedArchitectures</key>
+			<array>
+				<string>arm64</string>
+				<string>x86_64</string>
+			</array>
+			<key>SupportedPlatform</key>
+			<string>ios</string>
+			<key>SupportedPlatformVariant</key>
+			<string>simulator</string>
+		</dict>
+		<dict>
+			<key>LibraryIdentifier</key>
+			<string>tvos-arm64</string>
+			<key>LibraryPath</key>
+			<string>rnwhisper.framework</string>
+			<key>SupportedArchitectures</key>
+			<array>
+				<string>arm64</string>
+			</array>
+			<key>SupportedPlatform</key>
+			<string>tvos</string>
+		</dict>
+		<dict>
+			<key>LibraryIdentifier</key>
+			<string>tvos-arm64_x86_64-simulator</string>
+			<key>LibraryPath</key>
+			<string>rnwhisper.framework</string>
+			<key>SupportedArchitectures</key>
+			<array>
+				<string>arm64</string>
+				<string>x86_64</string>
+			</array>
+			<key>SupportedPlatform</key>
+			<string>tvos</string>
+			<key>SupportedPlatformVariant</key>
+			<string>simulator</string>
+		</dict>
+
+	</array>
+	<key>CFBundlePackageType</key>
+	<string>XFWK</string>
+	<key>XCFrameworkFormatVersion</key>
+	<string>1.0</string>
+	<key>CFBundleVersion</key>
+	<string>1.0.0</string>
+	<key>CFBundleShortVersionString</key>
+	<string>1.0.0</string>
+	<key>CFBundleIdentifier</key>
+	<string>com.rnwhisper</string>
+</dict>
+</plist>

package/ios/rnwhisper.xcframework/ios-arm64/rnwhisper.framework/Headers/ggml-alloc.h

@@ -0,0 +1,76 @@
+#pragma once
+
+#include "ggml.h"
+
+#ifdef  __cplusplus
+extern "C" {
+#endif
+
+typedef struct wsp_ggml_backend_buffer_type * wsp_ggml_backend_buffer_type_t;
+typedef struct      wsp_ggml_backend_buffer * wsp_ggml_backend_buffer_t;
+typedef struct             wsp_ggml_backend * wsp_ggml_backend_t;
+
+// Tensor allocator
+struct wsp_ggml_tallocr {
+    wsp_ggml_backend_buffer_t buffer;
+    void * base;
+    size_t alignment;
+    size_t offset;
+};
+
+WSP_GGML_API struct wsp_ggml_tallocr wsp_ggml_tallocr_new(wsp_ggml_backend_buffer_t buffer);
+WSP_GGML_API enum wsp_ggml_status    wsp_ggml_tallocr_alloc(struct wsp_ggml_tallocr * talloc, struct wsp_ggml_tensor * tensor);
+
+// Graph allocator
+/*
+  Example usage:
+    wsp_ggml_gallocr_t galloc = wsp_ggml_gallocr_new(wsp_ggml_backend_cpu_buffer_type());
+
+    // optional: create a worst-case graph and reserve the buffers to avoid reallocations
+    wsp_ggml_gallocr_reserve(galloc, build_graph(max_batch));
+
+    // allocate the graph
+    struct wsp_ggml_cgraph * graph = build_graph(batch);
+    wsp_ggml_gallocr_alloc_graph(galloc, graph);
+
+    printf("compute buffer size: %zu bytes\n", wsp_ggml_gallocr_get_buffer_size(galloc, 0));
+
+    // evaluate the graph
+    wsp_ggml_backend_graph_compute(backend, graph);
+*/
+
+// special tensor flags for use with the graph allocator:
+//   wsp_ggml_set_input(): all input tensors are allocated at the beginning of the graph in non-overlapping addresses
+//   wsp_ggml_set_output(): output tensors are never freed and never overwritten
+
+typedef struct wsp_ggml_gallocr * wsp_ggml_gallocr_t;
+
+WSP_GGML_API wsp_ggml_gallocr_t wsp_ggml_gallocr_new(wsp_ggml_backend_buffer_type_t buft);
+WSP_GGML_API wsp_ggml_gallocr_t wsp_ggml_gallocr_new_n(wsp_ggml_backend_buffer_type_t * bufts, int n_bufs);
+WSP_GGML_API void           wsp_ggml_gallocr_free(wsp_ggml_gallocr_t galloc);
+
+// pre-allocate buffers from a measure graph - does not allocate or modify the graph
+// call with a worst-case graph to avoid buffer reallocations
+// not strictly required for single buffer usage: wsp_ggml_gallocr_alloc_graph will reallocate the buffers automatically if needed
+// returns false if the buffer allocation failed
+WSP_GGML_API bool wsp_ggml_gallocr_reserve(wsp_ggml_gallocr_t galloc, struct wsp_ggml_cgraph * graph);
+WSP_GGML_API bool wsp_ggml_gallocr_reserve_n(
+    wsp_ggml_gallocr_t galloc,
+    struct wsp_ggml_cgraph * graph,
+    const int * node_buffer_ids,
+    const int * leaf_buffer_ids);
+
+// automatic reallocation if the topology changes when using a single buffer
+// returns false if using multiple buffers and a re-allocation is needed (call wsp_ggml_gallocr_reserve_n first to set the node buffers)
+WSP_GGML_API bool wsp_ggml_gallocr_alloc_graph(wsp_ggml_gallocr_t galloc, struct wsp_ggml_cgraph * graph);
+
+WSP_GGML_API size_t wsp_ggml_gallocr_get_buffer_size(wsp_ggml_gallocr_t galloc, int buffer_id);
+
+// Utils
+// Create a buffer and allocate all the tensors in a wsp_ggml_context
+WSP_GGML_API struct wsp_ggml_backend_buffer * wsp_ggml_backend_alloc_ctx_tensors_from_buft(struct wsp_ggml_context * ctx, wsp_ggml_backend_buffer_type_t buft);
+WSP_GGML_API struct wsp_ggml_backend_buffer * wsp_ggml_backend_alloc_ctx_tensors(struct wsp_ggml_context * ctx, wsp_ggml_backend_t backend);
+
+#ifdef  __cplusplus
+}
+#endif