whisper.rn 0.4.0-rc.3 → 0.4.0-rc.5

package/cpp/whisper.h

@@ -1,10 +1,20 @@
 #ifndef WHISPER_H
 #define WHISPER_H
 
+#include "ggml.h"
+
 #include <stddef.h>
 #include <stdint.h>
 #include <stdbool.h>
 
+#ifdef __GNUC__
+#    define WHISPER_DEPRECATED(func, hint) func __attribute__((deprecated(hint)))
+#elif defined(_MSC_VER)
+#    define WHISPER_DEPRECATED(func, hint) __declspec(deprecated(hint)) func
+#else
+#    define WHISPER_DEPRECATED(func, hint) func
+#endif
+
 #ifdef WHISPER_SHARED
 #    ifdef _WIN32
 #        ifdef WHISPER_BUILD
@@ -21,7 +31,6 @@
 
 #define WHISPER_SAMPLE_RATE 16000
 #define WHISPER_N_FFT       400
-#define WHISPER_N_MEL       80
 #define WHISPER_HOP_LENGTH  160
 #define WHISPER_CHUNK_SIZE  30
 
@@ -41,7 +50,9 @@ extern "C" {
     //
     //     ...
     //
-    //     struct whisper_context * ctx = whisper_init_from_file("/path/to/ggml-base.en.bin");
+    //     whisper_context_params cparams = whisper_context_default_params();
+    //
+    //     struct whisper_context * ctx = whisper_init_from_file_with_params("/path/to/ggml-base.en.bin", cparams);
     //
     //     if (whisper_full(ctx, wparams, pcmf32.data(), pcmf32.size()) != 0) {
     //         fprintf(stderr, "failed to process audio\n");
@@ -69,7 +80,14 @@ extern "C" {
     struct whisper_state;
     struct whisper_full_params;
 
-    typedef int whisper_token;
+    typedef int32_t whisper_pos;
+    typedef int32_t whisper_token;
+    typedef int32_t whisper_seq_id;
+
+    struct whisper_context_params {
+        bool  use_gpu;
+        bool  use_coreml;
+    };
 
     typedef struct whisper_token_data {
         whisper_token id;  // token id
@@ -96,21 +114,74 @@ extern "C" {
         void  (*close)(void * ctx);
     } whisper_model_loader;
 
+    // grammar element type
+    enum whisper_gretype {
+        // end of rule definition
+        WHISPER_GRETYPE_END            = 0,
+
+        // start of alternate definition for rule
+        WHISPER_GRETYPE_ALT            = 1,
+
+        // non-terminal element: reference to rule
+        WHISPER_GRETYPE_RULE_REF       = 2,
+
+        // terminal element: character (code point)
+        WHISPER_GRETYPE_CHAR           = 3,
+
+        // inverse char(s) ([^a], [^a-b] [^abc])
+        WHISPER_GRETYPE_CHAR_NOT       = 4,
+
+        // modifies a preceding WHISPER_GRETYPE_CHAR or LLAMA_GRETYPE_CHAR_ALT to
+        // be an inclusive range ([a-z])
+        WHISPER_GRETYPE_CHAR_RNG_UPPER = 5,
+
+        // modifies a preceding WHISPER_GRETYPE_CHAR or
+        // WHISPER_GRETYPE_CHAR_RNG_UPPER to add an alternate char to match ([ab], [a-zA])
+        WHISPER_GRETYPE_CHAR_ALT       = 6,
+    };
+
+    typedef struct whisper_grammar_element {
+        enum whisper_gretype type;
+        uint32_t             value; // Unicode code point or rule ID
+    } whisper_grammar_element;
+
     // Various functions for loading a ggml whisper model.
     // Allocate (almost) all memory needed for the model.
     // Return NULL on failure
-#ifdef WHISPER_USE_COREML
-    WHISPER_API struct whisper_context * whisper_init_from_file_no_coreml(const char * path_model);
-#endif
-    WHISPER_API struct whisper_context * whisper_init_from_file(const char * path_model);
-    WHISPER_API struct whisper_context * whisper_init_from_buffer(void * buffer, size_t buffer_size);
-    WHISPER_API struct whisper_context * whisper_init(struct whisper_model_loader * loader);
+    WHISPER_API struct whisper_context * whisper_init_from_file_with_params  (const char * path_model,              struct whisper_context_params params);
+    WHISPER_API struct whisper_context * whisper_init_from_buffer_with_params(void * buffer, size_t buffer_size,    struct whisper_context_params params);
+    WHISPER_API struct whisper_context * whisper_init_with_params            (struct whisper_model_loader * loader, struct whisper_context_params params);
 
     // These are the same as the above, but the internal state of the context is not allocated automatically
     // It is the responsibility of the caller to allocate the state using whisper_init_state() (#523)
-    WHISPER_API struct whisper_context * whisper_init_from_file_no_state(const char * path_model);
-    WHISPER_API struct whisper_context * whisper_init_from_buffer_no_state(void * buffer, size_t buffer_size);
-    WHISPER_API struct whisper_context * whisper_init_no_state(struct whisper_model_loader * loader);
+    WHISPER_API struct whisper_context * whisper_init_from_file_with_params_no_state  (const char * path_model,              struct whisper_context_params params);
+    WHISPER_API struct whisper_context * whisper_init_from_buffer_with_params_no_state(void * buffer, size_t buffer_size,    struct whisper_context_params params);
+    WHISPER_API struct whisper_context * whisper_init_with_params_no_state            (struct whisper_model_loader * loader, struct whisper_context_params params);
+
+    WHISPER_DEPRECATED(
+        WHISPER_API struct whisper_context * whisper_init_from_file(const char * path_model),
+        "use whisper_init_from_file_with_params instead"
+    );
+    WHISPER_DEPRECATED(
+        WHISPER_API struct whisper_context * whisper_init_from_buffer(void * buffer, size_t buffer_size),
+        "use whisper_init_from_buffer_with_params instead"
+    );
+    WHISPER_DEPRECATED(
+        WHISPER_API struct whisper_context * whisper_init(struct whisper_model_loader * loader),
+        "use whisper_init_with_params instead"
+    );
+    WHISPER_DEPRECATED(
+        WHISPER_API struct whisper_context * whisper_init_from_file_no_state(const char * path_model),
+        "use whisper_init_from_file_with_params_no_state instead"
+    );
+    WHISPER_DEPRECATED(
+        WHISPER_API struct whisper_context * whisper_init_from_buffer_no_state(void * buffer, size_t buffer_size),
+        "use whisper_init_from_buffer_with_params_no_state instead"
+    );
+    WHISPER_DEPRECATED(
+        WHISPER_API struct whisper_context * whisper_init_no_state(struct whisper_model_loader * loader),
+        "use whisper_init_with_params_no_state instead"
+    );
 
     WHISPER_API struct whisper_state * whisper_init_state(struct whisper_context * ctx);
 
@@ -135,6 +206,7 @@ extern "C" {
     WHISPER_API void whisper_free      (struct whisper_context * ctx);
     WHISPER_API void whisper_free_state(struct whisper_state * state);
     WHISPER_API void whisper_free_params(struct whisper_full_params * params);
+    WHISPER_API void whisper_free_context_params(struct whisper_context_params * params);
 
     // Convert RAW PCM audio to log mel spectrogram.
     // The resulting spectrogram is stored inside the default state of the provided whisper context.
@@ -244,6 +316,9 @@ extern "C" {
     // Return the short string of the specified language id (e.g. 2 -> "de"), returns nullptr if not found
     WHISPER_API const char * whisper_lang_str(int id);
 
+    // Return the short string of the specified language name (e.g. 2 -> "german"), returns nullptr if not found
+    WHISPER_API const char * whisper_lang_str_full(int id);
+
     // Use mel data at offset_ms to try and auto-detect the spoken language
     // Make sure to call whisper_pcm_to_mel() or whisper_set_mel() first
     // Returns the top language id or negative on failure
@@ -366,6 +441,7 @@ extern "C" {
 
         bool translate;
         bool no_context;        // do not use past transcription (if any) as initial prompt for the decoder
+        bool no_timestamps;     // do not generate timestamps
         bool single_segment;    // force single segment output (useful for streaming)
         bool print_special;     // print special tokens (e.g. <SOT>, <EOT>, <BEG>, etc.)
         bool print_progress;    // print progress information
@@ -443,9 +519,16 @@ extern "C" {
         // called by each decoder to filter obtained logits
         whisper_logits_filter_callback logits_filter_callback;
         void * logits_filter_callback_user_data;
+
+        const whisper_grammar_element ** grammar_rules;
+        size_t                           n_grammar_rules;
+        size_t                           i_start_rule;
+        float                            grammar_penalty;
     };
 
-    // NOTE: this function allocates memory, and it is the responsibility of the caller to free the pointer - see whisper_free_params()
+    // NOTE: this function allocates memory, and it is the responsibility of the caller to free the pointer - see whisper_free_context_params & whisper_free_params()
+    WHISPER_API struct whisper_context_params * whisper_context_default_params_by_ref();
+    WHISPER_API struct whisper_context_params whisper_context_default_params(void);
     WHISPER_API struct whisper_full_params * whisper_full_default_params_by_ref(enum whisper_sampling_strategy strategy);
     WHISPER_API struct whisper_full_params whisper_full_default_params(enum whisper_sampling_strategy strategy);
 
@@ -534,8 +617,7 @@ extern "C" {
 
     // Control logging output; default behavior is to print to stderr
 
-    typedef void (*whisper_log_callback)(const char * line);
-    WHISPER_API void whisper_set_log_callback(whisper_log_callback callback);
+    WHISPER_API void whisper_log_set(wsp_ggml_log_callback log_callback, void * user_data);
 
 #ifdef __cplusplus
 }

package/ios/RNWhisper.mm

@@ -48,6 +48,7 @@ RCT_REMAP_METHOD(initContext,
 
     NSString *modelPath = [modelOptions objectForKey:@"filePath"];
     BOOL isBundleAsset = [[modelOptions objectForKey:@"isBundleAsset"] boolValue];
+    BOOL useGpu = [[modelOptions objectForKey:@"useGpu"] boolValue];
     BOOL useCoreMLIos = [[modelOptions objectForKey:@"useCoreMLIos"] boolValue];
 
     // For support debug assets in development mode
@@ -77,6 +78,7 @@ RCT_REMAP_METHOD(initContext,
         initWithModelPath:path
         contextId:contextId
         noCoreML:!useCoreMLIos
+        noMetal:!useGpu
     ];
     if ([context getContext] == NULL) {
         reject(@"whisper_cpp_error", @"Failed to load the model", nil);
@@ -85,7 +87,11 @@ RCT_REMAP_METHOD(initContext,
 
     [contexts setObject:context forKey:[NSNumber numberWithInt:contextId]];
 
-    resolve([NSNumber numberWithInt:contextId]);
+    resolve(@{
+        @"contextId": @(contextId),
+        @"gpu": @([context isMetalEnabled]),
+        @"reasonNoGPU": [context reasonNoMetal],
+    });
 }
 
 - (NSArray *)supportedEvents {
@@ -136,9 +142,9 @@ RCT_REMAP_METHOD(transcribeFile,
         audioDataCount:count
         options:options
         onProgress: ^(int progress) {
-            if (rn_whisper_transcribe_is_aborted(jobId)) {
-                return;
-            }
+            rnwhisper::job* job = rnwhisper::job_get(jobId);
+            if (job && job->is_aborted()) return;
+
             dispatch_async(dispatch_get_main_queue(), ^{
                 [self sendEventWithName:@"@RNWhisper_onTranscribeProgress"
                     body:@{
@@ -150,9 +156,9 @@ RCT_REMAP_METHOD(transcribeFile,
             });
         }
         onNewSegments: ^(NSDictionary *result) {
-            if (rn_whisper_transcribe_is_aborted(jobId)) {
-                return;
-            }
+            rnwhisper::job* job = rnwhisper::job_get(jobId);
+            if (job && job->is_aborted()) return;
+
             dispatch_async(dispatch_get_main_queue(), ^{
                 [self sendEventWithName:@"@RNWhisper_onTranscribeNewSegments"
                     body:@{
@@ -164,7 +170,7 @@ RCT_REMAP_METHOD(transcribeFile,
             });
         }
         onEnd: ^(int code) {
-            if (code != 0) {
+            if (code != 0 && code != 999) {
                 free(waveFile);
                 reject(@"whisper_cpp_error", [NSString stringWithFormat:@"Failed to transcribe the file. Code: %d", code], nil);
                 return;
@@ -273,7 +279,7 @@ RCT_REMAP_METHOD(releaseAllContexts,
         [context invalidate];
     }
 
-    rn_whisper_abort_all_transcribe(); // graceful abort
+    rnwhisper::job_abort_all(); // graceful abort
 
     [contexts removeAllObjects];
     contexts = nil;

package/ios/RNWhisper.xcodeproj/project.xcworkspace/contents.xcworkspacedata

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Workspace
+   version = "1.0">
+</Workspace>

package/ios/RNWhisper.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>IDEDidComputeMac32BitWarning</key>
+	<true/>
+</dict>
+</plist>

package/ios/RNWhisper.xcodeproj/xcuserdata/jhen.xcuserdatad/xcschemes/xcschememanagement.plist

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>SchemeUserState</key>
+	<dict>
+		<key>RNWhisper.xcscheme_^#shared#^_</key>
+		<dict>
+			<key>orderHint</key>
+			<integer>0</integer>
+		</dict>
+		<key>WhisperCpp.xcscheme_^#shared#^_</key>
+		<dict>
+			<key>orderHint</key>
+			<integer>0</integer>
+		</dict>
+	</dict>
+</dict>
+</plist>

package/ios/RNWhisperAudioUtils.h

@@ -2,8 +2,6 @@
 
 @interface RNWhisperAudioUtils : NSObject
 
-+ (NSData *)concatShortBuffers:(NSMutableArray<NSValue *> *)buffers sliceNSamples:(NSMutableArray<NSNumber *> *)sliceNSamples;
-+ (void)saveWavFile:(NSData *)rawData audioOutputFile:(NSString *)audioOutputFile;
 + (float *)decodeWaveFile:(NSString*)filePath count:(int *)count;
 
 @end

package/ios/RNWhisperAudioUtils.m

@@ -3,62 +3,6 @@
 
 @implementation RNWhisperAudioUtils
 
-+ (NSData *)concatShortBuffers:(NSMutableArray<NSValue *> *)buffers sliceNSamples:(NSMutableArray<NSNumber *> *)sliceNSamples {
-    NSMutableData *outputData = [NSMutableData data];
-    for (int i = 0; i < buffers.count; i++) {
-        int size = [sliceNSamples objectAtIndex:i].intValue;
-        NSValue *buffer = [buffers objectAtIndex:i];
-        short *bufferPtr = buffer.pointerValue;
-        [outputData appendBytes:bufferPtr length:size * sizeof(short)];
-    }
-    return outputData;
-}
-
-+ (void)saveWavFile:(NSData *)rawData audioOutputFile:(NSString *)audioOutputFile {
-    NSMutableData *outputData = [NSMutableData data];
-    
-    // WAVE header
-    [outputData appendData:[@"RIFF" dataUsingEncoding:NSUTF8StringEncoding]]; // chunk id
-    int chunkSize = CFSwapInt32HostToLittle(36 + rawData.length);
-    [outputData appendBytes:&chunkSize length:sizeof(chunkSize)];
-    [outputData appendData:[@"WAVE" dataUsingEncoding:NSUTF8StringEncoding]]; // format
-    [outputData appendData:[@"fmt " dataUsingEncoding:NSUTF8StringEncoding]]; // subchunk 1 id
-    
-    int subchunk1Size = CFSwapInt32HostToLittle(16);
-    [outputData appendBytes:&subchunk1Size length:sizeof(subchunk1Size)];
-
-    short audioFormat = CFSwapInt16HostToLittle(1); // PCM
-    [outputData appendBytes:&audioFormat length:sizeof(audioFormat)];
-
-    short numChannels = CFSwapInt16HostToLittle(1); // mono
-    [outputData appendBytes:&numChannels length:sizeof(numChannels)];
-
-    int sampleRate = CFSwapInt32HostToLittle(WHISPER_SAMPLE_RATE);
-    [outputData appendBytes:&sampleRate length:sizeof(sampleRate)];
-
-    // (bitDepth * sampleRate * channels) >> 3
-    int byteRate = CFSwapInt32HostToLittle(WHISPER_SAMPLE_RATE * 1 * 16 / 8);
-    [outputData appendBytes:&byteRate length:sizeof(byteRate)];
-
-    // (bitDepth * channels) >> 3
-    short blockAlign = CFSwapInt16HostToLittle(16 / 8);
-    [outputData appendBytes:&blockAlign length:sizeof(blockAlign)];
-
-    // bitDepth
-    short bitsPerSample = CFSwapInt16HostToLittle(16);
-    [outputData appendBytes:&bitsPerSample length:sizeof(bitsPerSample)];
-
-    [outputData appendData:[@"data" dataUsingEncoding:NSUTF8StringEncoding]]; // subchunk 2 id
-    int subchunk2Size = CFSwapInt32HostToLittle((int)rawData.length);
-    [outputData appendBytes:&subchunk2Size length:sizeof(subchunk2Size)];
-
-    // Audio data
-    [outputData appendData:rawData];
-    
-    // Save to file
-    [outputData writeToFile:audioOutputFile atomically:YES];
-}
-
 + (float *)decodeWaveFile:(NSString*)filePath count:(int *)count {
     NSURL *url = [NSURL fileURLWithPath:filePath];
     NSData *fileData = [NSData dataWithContentsOfURL:url];

package/ios/RNWhisperContext.h

@@ -11,29 +11,21 @@
 
 typedef struct {
     __unsafe_unretained id mSelf;
-
-    int jobId;
     NSDictionary* options;
 
+    struct rnwhisper::job * job;
+
     bool isTranscribing;
     bool isRealtime;
     bool isCapturing;
     bool isStoppedByAction;
-    int maxAudioSec;
     int nSamplesTranscribing;
-    NSMutableArray<NSValue *> *shortBufferSlices;
-    NSMutableArray<NSNumber *> *sliceNSamples;
+    std::vector<int> sliceNSamples;
     bool isUseSlices;
     int sliceIndex;
     int transcribeSliceIndex;
-    int audioSliceSec;
     NSString* audioOutputPath;
 
-    bool useVad;
-    int vadMs;
-    float vadThold;
-    float vadFreqThold;
-
     AudioQueueRef queue;
     AudioStreamBasicDescription dataFormat;
     AudioQueueBufferRef buffers[NUM_BUFFERS];
@@ -46,9 +38,13 @@ typedef struct {
     dispatch_queue_t dQueue;
     struct whisper_context * ctx;
     RNWhisperContextRecordState recordState;
+    NSString * reasonNoMetal;
+    bool isMetalEnabled;
 }
 
-+ (instancetype)initWithModelPath:(NSString *)modelPath contextId:(int)contextId noCoreML:(BOOL)noCoreML;
++ (instancetype)initWithModelPath:(NSString *)modelPath contextId:(int)contextId noCoreML:(BOOL)noCoreML noMetal:(BOOL)noMetal;
+- (bool)isMetalEnabled;
+- (NSString *)reasonNoMetal;
 - (struct whisper_context *)getContext;
 - (dispatch_queue_t)getDispatchQueue;
 - (OSStatus)transcribeRealtime:(int)jobId