whisper.rn 0.4.0-rc.8 → 0.4.0

package/ios/RNWhisperContext.h

@@ -1,6 +1,11 @@
 #ifdef __cplusplus
+#if RNWHISPER_BUILD_FROM_SOURCE
 #import "whisper.h"
 #import "rn-whisper.h"
+#else
+#import <rnwhisper/whisper.h>
+#import <rnwhisper/rn-whisper.h>
+#endif
 #endif
 
 #import <AVFoundation/AVFoundation.h>
@@ -42,7 +47,7 @@ typedef struct {
     bool isMetalEnabled;
 }
 
-+ (instancetype)initWithModelPath:(NSString *)modelPath contextId:(int)contextId noCoreML:(BOOL)noCoreML noMetal:(BOOL)noMetal;
++ (instancetype)initWithModelPath:(NSString *)modelPath contextId:(int)contextId noCoreML:(BOOL)noCoreML noMetal:(BOOL)noMetal useFlashAttn:(BOOL)useFlashAttn;
 - (bool)isMetalEnabled;
 - (NSString *)reasonNoMetal;
 - (struct whisper_context *)getContext;
@@ -50,7 +55,7 @@ typedef struct {
 - (OSStatus)transcribeRealtime:(int)jobId
     options:(NSDictionary *)options
     onTranscribe:(void (^)(int, NSString *, NSDictionary *))onTranscribe;
-- (void)transcribeFile:(int)jobId
+- (void)transcribeData:(int)jobId
     audioData:(float *)audioData
     audioDataCount:(int)audioDataCount
     options:(NSDictionary *)options
@@ -63,6 +68,7 @@ typedef struct {
 - (bool)isTranscribing;
 - (bool)isStoppedByAction;
 - (NSMutableDictionary *)getTextSegments;
+- (NSString *)bench:(int)maxThreads;
 - (void)invalidate;
 
 @end

package/ios/RNWhisperContext.mm

@@ -10,12 +10,18 @@
     contextId:(int)contextId
     noCoreML:(BOOL)noCoreML
     noMetal:(BOOL)noMetal
+    useFlashAttn:(BOOL)useFlashAttn
 {
     RNWhisperContext *context = [[RNWhisperContext alloc] init];
     context->contextId = contextId;
     struct whisper_context_params cparams;
     NSString *reasonNoMetal = @"";
     cparams.use_gpu = !noMetal;
+    cparams.flash_attn = useFlashAttn;
+
+    // TODO: Expose dtw_token_timestamps and dtw_aheads_preset
+    cparams.dtw_token_timestamps = false;
+    // cparams.dtw_aheads_preset = WHISPER_AHEADS_BASE;
 
     cparams.use_coreml = !noCoreML;
 #ifndef WHISPER_USE_COREML
@@ -353,9 +359,10 @@ void AudioInputCallback(void * inUserData,
 struct rnwhisper_segments_callback_data {
     void (^onNewSegments)(NSDictionary *);
     int total_n_new;
+    bool tdrzEnable;
 };
 
-- (void)transcribeFile:(int)jobId
+- (void)transcribeData:(int)jobId
     audioData:(float *)audioData
     audioDataCount:(int)audioDataCount
     options:(NSDictionary *)options
@@ -386,12 +393,18 @@ struct rnwhisper_segments_callback_data {
                 NSMutableArray *segments = [[NSMutableArray alloc] init];
                 for (int i = data->total_n_new - n_new; i < data->total_n_new; i++) {
                     const char * text_cur = whisper_full_get_segment_text(ctx, i);
-                    text = [text stringByAppendingString:[NSString stringWithUTF8String:text_cur]];
+                    NSMutableString *mutable_ns_text = [NSMutableString stringWithUTF8String:text_cur];
+
+                    if (data->tdrzEnable && whisper_full_get_segment_speaker_turn_next(ctx, i)) {
+                        [mutable_ns_text appendString:@" [SPEAKER_TURN]"];
+                    }
+
+                    text = [text stringByAppendingString:mutable_ns_text];
 
                     const int64_t t0 = whisper_full_get_segment_t0(ctx, i);
                     const int64_t t1 = whisper_full_get_segment_t1(ctx, i);
                     NSDictionary *segment = @{
-                        @"text": [NSString stringWithUTF8String:text_cur],
+                        @"text": [NSString stringWithString:mutable_ns_text],
                         @"t0": [NSNumber numberWithLongLong:t0],
                         @"t1": [NSNumber numberWithLongLong:t1]
                     };
@@ -409,7 +422,8 @@ struct rnwhisper_segments_callback_data {
             };
             struct rnwhisper_segments_callback_data user_data = {
                 .onNewSegments = onNewSegments,
-                .total_n_new = 0
+                .tdrzEnable = options[@"tdrzEnable"] && [options[@"tdrzEnable"] boolValue],
+                .total_n_new = 0,
             };
             params.new_segment_callback_user_data = &user_data;
         }
@@ -418,6 +432,7 @@ struct rnwhisper_segments_callback_data {
         self->recordState.job = job;
         int code = [self fullTranscribe:job audioData:audioData audioDataCount:audioDataCount];
         rnwhisper::job_remove(jobId);
+        self->recordState.job = nullptr;
         self->recordState.isTranscribing = false;
         onEnd(code);
     });
@@ -432,7 +447,7 @@ struct rnwhisper_segments_callback_data {
 }
 
 - (void)stopTranscribe:(int)jobId {
-    if (self->recordState.job) self->recordState.job->abort();
+    if (self->recordState.job != nullptr) self->recordState.job->abort();
     if (self->recordState.isRealtime && self->recordState.isCapturing) {
         [self stopAudio];
         if (!self->recordState.isTranscribing) {
@@ -469,7 +484,6 @@ struct rnwhisper_segments_callback_data {
     params.print_progress   = false;
     params.print_timestamps = false;
     params.print_special    = false;
-    params.speed_up         = options[@"speedUp"] != nil ? [options[@"speedUp"] boolValue] : false;
     params.translate        = options[@"translate"] != nil ? [options[@"translate"] boolValue] : false;
     params.language         = options[@"language"] != nil ? strdup([options[@"language"] UTF8String]) : "auto";
     params.n_threads        = n_threads > 0 ? n_threads : default_n_threads;
@@ -481,6 +495,7 @@ struct rnwhisper_segments_callback_data {
         params.max_len = [options[@"maxLen"] intValue];
     }
     params.token_timestamps = options[@"tokenTimestamps"] != nil ? [options[@"tokenTimestamps"] boolValue] : false;
+    params.tdrz_enable = options[@"tdrzEnable"] != nil ? [options[@"tdrzEnable"] boolValue] : false;
 
     if (options[@"bestOf"] != nil) {
         params.greedy.best_of = [options[@"bestOf"] intValue];
@@ -530,12 +545,21 @@ struct rnwhisper_segments_callback_data {
     NSMutableArray *segments = [[NSMutableArray alloc] init];
     for (int i = 0; i < n_segments; i++) {
         const char * text_cur = whisper_full_get_segment_text(self->ctx, i);
-        text = [text stringByAppendingString:[NSString stringWithUTF8String:text_cur]];
+        NSMutableString *mutable_ns_text = [NSMutableString stringWithUTF8String:text_cur];
+
+        // Simplified condition
+        if (self->recordState.options[@"tdrzEnable"] &&
+            [self->recordState.options[@"tdrzEnable"] boolValue] &&
+            whisper_full_get_segment_speaker_turn_next(self->ctx, i)) {
+            [mutable_ns_text appendString:@" [SPEAKER_TURN]"];
+        }
+
+        text = [text stringByAppendingString:mutable_ns_text];
 
         const int64_t t0 = whisper_full_get_segment_t0(self->ctx, i);
         const int64_t t1 = whisper_full_get_segment_t1(self->ctx, i);
         NSDictionary *segment = @{
-            @"text": [NSString stringWithUTF8String:text_cur],
+            @"text": [NSString stringWithString:mutable_ns_text],
             @"t0": [NSNumber numberWithLongLong:t0],
             @"t1": [NSNumber numberWithLongLong:t1]
         };
@@ -547,6 +571,16 @@ struct rnwhisper_segments_callback_data {
     return result;
 }
 
+- (NSString *)bench:(int)maxThreads {
+    const int n_threads = maxThreads > 0 ? maxThreads : 0;
+
+    const int max_threads = (int) [[NSProcessInfo processInfo] processorCount];
+    // Use 2 threads by default on 4-core devices, 4 threads on more cores
+    const int default_n_threads = max_threads == 4 ? 2 : MIN(4, max_threads);
+    NSString *result = [NSString stringWithUTF8String:rnwhisper::bench(self->ctx, n_threads).c_str()];
+    return result;
+}
+
 - (void)invalidate {
     [self stopCurrentTranscribe];
     whisper_free(self->ctx);

package/ios/rnwhisper.xcframework/Info.plist

@@ -0,0 +1,74 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>AvailableLibraries</key>
+	<array>
+		<dict>
+			<key>LibraryIdentifier</key>
+			<string>ios-arm64</string>
+			<key>LibraryPath</key>
+			<string>rnwhisper.framework</string>
+			<key>SupportedArchitectures</key>
+			<array>
+				<string>arm64</string>
+			</array>
+			<key>SupportedPlatform</key>
+			<string>ios</string>
+		</dict>
+		<dict>
+			<key>LibraryIdentifier</key>
+			<string>ios-arm64_x86_64-simulator</string>
+			<key>LibraryPath</key>
+			<string>rnwhisper.framework</string>
+			<key>SupportedArchitectures</key>
+			<array>
+				<string>arm64</string>
+				<string>x86_64</string>
+			</array>
+			<key>SupportedPlatform</key>
+			<string>ios</string>
+			<key>SupportedPlatformVariant</key>
+			<string>simulator</string>
+		</dict>
+		<dict>
+			<key>LibraryIdentifier</key>
+			<string>tvos-arm64</string>
+			<key>LibraryPath</key>
+			<string>rnwhisper.framework</string>
+			<key>SupportedArchitectures</key>
+			<array>
+				<string>arm64</string>
+			</array>
+			<key>SupportedPlatform</key>
+			<string>tvos</string>
+		</dict>
+		<dict>
+			<key>LibraryIdentifier</key>
+			<string>tvos-arm64_x86_64-simulator</string>
+			<key>LibraryPath</key>
+			<string>rnwhisper.framework</string>
+			<key>SupportedArchitectures</key>
+			<array>
+				<string>arm64</string>
+				<string>x86_64</string>
+			</array>
+			<key>SupportedPlatform</key>
+			<string>tvos</string>
+			<key>SupportedPlatformVariant</key>
+			<string>simulator</string>
+		</dict>
+
+	</array>
+	<key>CFBundlePackageType</key>
+	<string>XFWK</string>
+	<key>XCFrameworkFormatVersion</key>
+	<string>1.0</string>
+	<key>CFBundleVersion</key>
+	<string>1.0.0</string>
+	<key>CFBundleShortVersionString</key>
+	<string>1.0.0</string>
+	<key>CFBundleIdentifier</key>
+	<string>com.rnwhisper</string>
+</dict>
+</plist>

package/ios/rnwhisper.xcframework/ios-arm64/rnwhisper.framework/Headers/ggml-alloc.h

@@ -0,0 +1,76 @@
+#pragma once
+
+#include "ggml.h"
+
+#ifdef  __cplusplus
+extern "C" {
+#endif
+
+typedef struct wsp_ggml_backend_buffer_type * wsp_ggml_backend_buffer_type_t;
+typedef struct      wsp_ggml_backend_buffer * wsp_ggml_backend_buffer_t;
+typedef struct             wsp_ggml_backend * wsp_ggml_backend_t;
+
+// Tensor allocator
+struct wsp_ggml_tallocr {
+    wsp_ggml_backend_buffer_t buffer;
+    void * base;
+    size_t alignment;
+    size_t offset;
+};
+
+WSP_GGML_API struct wsp_ggml_tallocr wsp_ggml_tallocr_new(wsp_ggml_backend_buffer_t buffer);
+WSP_GGML_API enum wsp_ggml_status    wsp_ggml_tallocr_alloc(struct wsp_ggml_tallocr * talloc, struct wsp_ggml_tensor * tensor);
+
+// Graph allocator
+/*
+  Example usage:
+    wsp_ggml_gallocr_t galloc = wsp_ggml_gallocr_new(wsp_ggml_backend_cpu_buffer_type());
+
+    // optional: create a worst-case graph and reserve the buffers to avoid reallocations
+    wsp_ggml_gallocr_reserve(galloc, build_graph(max_batch));
+
+    // allocate the graph
+    struct wsp_ggml_cgraph * graph = build_graph(batch);
+    wsp_ggml_gallocr_alloc_graph(galloc, graph);
+
+    printf("compute buffer size: %zu bytes\n", wsp_ggml_gallocr_get_buffer_size(galloc, 0));
+
+    // evaluate the graph
+    wsp_ggml_backend_graph_compute(backend, graph);
+*/
+
+// special tensor flags for use with the graph allocator:
+//   wsp_ggml_set_input(): all input tensors are allocated at the beginning of the graph in non-overlapping addresses
+//   wsp_ggml_set_output(): output tensors are never freed and never overwritten
+
+typedef struct wsp_ggml_gallocr * wsp_ggml_gallocr_t;
+
+WSP_GGML_API wsp_ggml_gallocr_t wsp_ggml_gallocr_new(wsp_ggml_backend_buffer_type_t buft);
+WSP_GGML_API wsp_ggml_gallocr_t wsp_ggml_gallocr_new_n(wsp_ggml_backend_buffer_type_t * bufts, int n_bufs);
+WSP_GGML_API void           wsp_ggml_gallocr_free(wsp_ggml_gallocr_t galloc);
+
+// pre-allocate buffers from a measure graph - does not allocate or modify the graph
+// call with a worst-case graph to avoid buffer reallocations
+// not strictly required for single buffer usage: wsp_ggml_gallocr_alloc_graph will reallocate the buffers automatically if needed
+// returns false if the buffer allocation failed
+WSP_GGML_API bool wsp_ggml_gallocr_reserve(wsp_ggml_gallocr_t galloc, struct wsp_ggml_cgraph * graph);
+WSP_GGML_API bool wsp_ggml_gallocr_reserve_n(
+    wsp_ggml_gallocr_t galloc,
+    struct wsp_ggml_cgraph * graph,
+    const int * node_buffer_ids,
+    const int * leaf_buffer_ids);
+
+// automatic reallocation if the topology changes when using a single buffer
+// returns false if using multiple buffers and a re-allocation is needed (call wsp_ggml_gallocr_reserve_n first to set the node buffers)
+WSP_GGML_API bool wsp_ggml_gallocr_alloc_graph(wsp_ggml_gallocr_t galloc, struct wsp_ggml_cgraph * graph);
+
+WSP_GGML_API size_t wsp_ggml_gallocr_get_buffer_size(wsp_ggml_gallocr_t galloc, int buffer_id);
+
+// Utils
+// Create a buffer and allocate all the tensors in a wsp_ggml_context
+WSP_GGML_API struct wsp_ggml_backend_buffer * wsp_ggml_backend_alloc_ctx_tensors_from_buft(struct wsp_ggml_context * ctx, wsp_ggml_backend_buffer_type_t buft);
+WSP_GGML_API struct wsp_ggml_backend_buffer * wsp_ggml_backend_alloc_ctx_tensors(struct wsp_ggml_context * ctx, wsp_ggml_backend_t backend);
+
+#ifdef  __cplusplus
+}
+#endif

package/ios/rnwhisper.xcframework/ios-arm64/rnwhisper.framework/Headers/ggml-backend-impl.h

@@ -0,0 +1,255 @@
+#pragma once
+
+// ggml-backend internal header
+
+#include "ggml-backend.h"
+
+#ifdef  __cplusplus
+extern "C" {
+#endif
+
+    #define WSP_GGML_BACKEND_API_VERSION 1
+
+    //
+    // Backend buffer type
+    //
+
+    struct wsp_ggml_backend_buffer_type_i {
+        const char *          (*get_name)      (wsp_ggml_backend_buffer_type_t buft);
+        // allocate a buffer of this type
+        wsp_ggml_backend_buffer_t (*alloc_buffer)  (wsp_ggml_backend_buffer_type_t buft, size_t size);
+        // tensor alignment
+        size_t                (*get_alignment) (wsp_ggml_backend_buffer_type_t buft);
+        // (optional) max buffer size that can be allocated (defaults to SIZE_MAX)
+        size_t                (*get_max_size)  (wsp_ggml_backend_buffer_type_t buft);
+        // (optional) data size needed to allocate the tensor, including padding (defaults to wsp_ggml_nbytes)
+        size_t                (*get_alloc_size)(wsp_ggml_backend_buffer_type_t buft, const struct wsp_ggml_tensor * tensor);
+        // (optional) check if tensor data is in host memory and uses standard ggml tensor layout (defaults to false)
+        bool                  (*is_host)       (wsp_ggml_backend_buffer_type_t buft);
+    };
+
+    struct wsp_ggml_backend_buffer_type {
+        struct wsp_ggml_backend_buffer_type_i  iface;
+        wsp_ggml_backend_dev_t device;
+        void * context;
+    };
+
+    //
+    // Backend buffer
+    //
+
+    struct wsp_ggml_backend_buffer_i {
+        // (optional) free the buffer
+        void         (*free_buffer)  (wsp_ggml_backend_buffer_t buffer);
+        // base address of the buffer
+        void *       (*get_base)     (wsp_ggml_backend_buffer_t buffer);
+        // (optional) initialize a tensor in the buffer (eg. add tensor extras)
+        enum wsp_ggml_status (*init_tensor)(wsp_ggml_backend_buffer_t buffer, struct wsp_ggml_tensor * tensor);
+        // tensor data access
+        void         (*memset_tensor)(wsp_ggml_backend_buffer_t buffer,       struct wsp_ggml_tensor * tensor,     uint8_t value, size_t offset, size_t size);
+        void         (*set_tensor)   (wsp_ggml_backend_buffer_t buffer,       struct wsp_ggml_tensor * tensor, const void * data, size_t offset, size_t size);
+        void         (*get_tensor)   (wsp_ggml_backend_buffer_t buffer, const struct wsp_ggml_tensor * tensor,       void * data, size_t offset, size_t size);
+        // (optional) tensor copy: dst is in the buffer, src may be in any buffer, including buffers from a different backend (return false if not supported)
+        bool         (*cpy_tensor)   (wsp_ggml_backend_buffer_t buffer, const struct wsp_ggml_tensor * src, struct wsp_ggml_tensor * dst);
+        // clear the entire buffer
+        void         (*clear)        (wsp_ggml_backend_buffer_t buffer, uint8_t value);
+        // (optional) reset any internal state due to tensor initialization, such as tensor extras
+        void         (*reset)        (wsp_ggml_backend_buffer_t buffer);
+    };
+
+    struct wsp_ggml_backend_buffer {
+        struct wsp_ggml_backend_buffer_i  iface;
+        wsp_ggml_backend_buffer_type_t    buft;
+        void * context;
+        size_t size;
+        enum wsp_ggml_backend_buffer_usage usage;
+    };
+
+    WSP_GGML_API wsp_ggml_backend_buffer_t wsp_ggml_backend_buffer_init(
+                   wsp_ggml_backend_buffer_type_t buft,
+            struct wsp_ggml_backend_buffer_i      iface,
+                   void *                     context,
+                   size_t                     size);
+
+    // do not use directly, use wsp_ggml_backend_tensor_copy instead
+    WSP_GGML_API bool wsp_ggml_backend_buffer_copy_tensor(const struct wsp_ggml_tensor * src, struct wsp_ggml_tensor * dst);
+
+    // multi-buffer
+    // buffer that contains a collection of buffers
+    WSP_GGML_API wsp_ggml_backend_buffer_t wsp_ggml_backend_multi_buffer_alloc_buffer(wsp_ggml_backend_buffer_t * buffers, size_t n_buffers);
+    WSP_GGML_API bool                  wsp_ggml_backend_buffer_is_multi_buffer(wsp_ggml_backend_buffer_t buffer);
+    WSP_GGML_API void                  wsp_ggml_backend_multi_buffer_set_usage(wsp_ggml_backend_buffer_t buffer, enum wsp_ggml_backend_buffer_usage usage);
+
+    //
+    // Backend (stream)
+    //
+
+    struct wsp_ggml_backend_i {
+        const char * (*get_name)(wsp_ggml_backend_t backend);
+
+        void (*free)(wsp_ggml_backend_t backend);
+
+        // (optional) asynchronous tensor data access
+        void (*set_tensor_async)(wsp_ggml_backend_t backend,       struct wsp_ggml_tensor * tensor, const void * data, size_t offset, size_t size);
+        void (*get_tensor_async)(wsp_ggml_backend_t backend, const struct wsp_ggml_tensor * tensor,       void * data, size_t offset, size_t size);
+        bool (*cpy_tensor_async)(wsp_ggml_backend_t backend_src, wsp_ggml_backend_t backend_dst, const struct wsp_ggml_tensor * src, struct wsp_ggml_tensor * dst);
+
+        // (optional) complete all pending operations (required if the backend supports async operations)
+        void (*synchronize)(wsp_ggml_backend_t backend);
+
+        // (optional) graph plans (not used currently)
+        // compute graph with a plan
+        wsp_ggml_backend_graph_plan_t (*graph_plan_create) (wsp_ggml_backend_t backend, const struct wsp_ggml_cgraph * cgraph);
+        void                      (*graph_plan_free)   (wsp_ggml_backend_t backend, wsp_ggml_backend_graph_plan_t plan);
+        // update the plan with a new graph - this should be faster than creating a new plan when the graph has the same topology
+        void                      (*graph_plan_update) (wsp_ggml_backend_t backend, wsp_ggml_backend_graph_plan_t plan, const struct wsp_ggml_cgraph * cgraph);
+        // compute the graph with the plan
+        enum wsp_ggml_status          (*graph_plan_compute)(wsp_ggml_backend_t backend, wsp_ggml_backend_graph_plan_t plan);
+
+        // compute graph (always async if supported by the backend)
+        enum wsp_ggml_status          (*graph_compute)     (wsp_ggml_backend_t backend, struct wsp_ggml_cgraph * cgraph);
+
+        // (optional) event synchronization
+        // record an event on this stream
+        void (*event_record)(wsp_ggml_backend_t backend, wsp_ggml_backend_event_t event);
+        // wait for an event on on a different stream
+        void (*event_wait)  (wsp_ggml_backend_t backend, wsp_ggml_backend_event_t event);
+    };
+
+    struct wsp_ggml_backend {
+        wsp_ggml_guid_t guid;
+        struct wsp_ggml_backend_i iface;
+        wsp_ggml_backend_dev_t device;
+        void * context;
+    };
+
+    struct wsp_ggml_backend_event {
+        struct wsp_ggml_backend_device * device;
+        void * context;
+    };
+
+    //
+    // Backend device
+    //
+
+    // Note: if additional properties are needed, we should add a struct with all of them
+    //       the current functions to obtain the properties can remain, since they are more convenient for often used properties
+    struct wsp_ggml_backend_device_i {
+        // device name: short identifier for this device, such as "CPU" or "CUDA0"
+        const char * (*get_name)(wsp_ggml_backend_dev_t dev);
+
+        // device description: short informative description of the device, could be the model name
+        const char * (*get_description)(wsp_ggml_backend_dev_t dev);
+
+        // device memory in bytes
+        void         (*get_memory)(wsp_ggml_backend_dev_t dev, size_t * free, size_t * total);
+
+        // device type
+        enum wsp_ggml_backend_dev_type (*get_type)(wsp_ggml_backend_dev_t dev);
+
+        // device properties
+        void (*get_props)(wsp_ggml_backend_dev_t dev, struct wsp_ggml_backend_dev_props * props);
+
+        // backend (stream) initialization
+        wsp_ggml_backend_t (*init_backend)(wsp_ggml_backend_dev_t dev, const char * params);
+
+        // preferred buffer type
+        wsp_ggml_backend_buffer_type_t (*get_buffer_type)(wsp_ggml_backend_dev_t dev);
+
+        // (optional) host buffer type (in system memory, typically this is a pinned memory buffer for faster transfers between host and device)
+        wsp_ggml_backend_buffer_type_t (*get_host_buffer_type)(wsp_ggml_backend_dev_t dev);
+
+        // (optional) buffer from pointer: create a buffer from a host pointer (useful for memory mapped models and importing data from other libraries)
+        wsp_ggml_backend_buffer_t (*buffer_from_host_ptr)(wsp_ggml_backend_dev_t dev, void * ptr, size_t size, size_t max_tensor_size);
+
+        // check if the backend can compute an operation
+        bool (*supports_op)(wsp_ggml_backend_dev_t dev, const struct wsp_ggml_tensor * op);
+
+        // check if the backend can use tensors allocated in a buffer type
+        bool (*supports_buft)(wsp_ggml_backend_dev_t dev, wsp_ggml_backend_buffer_type_t buft);
+
+        // (optional) check if the backend wants to run an operation, even if the weights are allocated in an incompatible buffer
+        // these should be expensive operations that may benefit from running on this backend instead of the CPU backend
+        bool (*offload_op)(wsp_ggml_backend_dev_t dev, const struct wsp_ggml_tensor * op);
+
+        // (optional) event synchronization
+        wsp_ggml_backend_event_t (*event_new)         (wsp_ggml_backend_dev_t dev);
+        void                 (*event_free)        (wsp_ggml_backend_dev_t dev, wsp_ggml_backend_event_t event);
+        void                 (*event_synchronize) (wsp_ggml_backend_dev_t dev, wsp_ggml_backend_event_t event);
+    };
+
+    struct wsp_ggml_backend_device {
+        struct wsp_ggml_backend_device_i iface;
+        wsp_ggml_backend_reg_t reg;
+        void * context;
+    };
+
+    //
+    // Backend (reg)
+    //
+
+    struct wsp_ggml_backend_reg_i {
+        const char * (*get_name)(wsp_ggml_backend_reg_t reg);
+
+        // enumerate available devices
+        size_t             (*get_device_count)(wsp_ggml_backend_reg_t reg);
+        wsp_ggml_backend_dev_t (*get_device)(wsp_ggml_backend_reg_t reg, size_t index);
+
+        // (optional) get a pointer to a function in the backend
+        // backends can add custom functions that are not part of the standard ggml-backend interface
+        void * (*get_proc_address)(wsp_ggml_backend_reg_t reg, const char * name);
+    };
+
+    struct wsp_ggml_backend_reg {
+        int api_version; // initialize to WSP_GGML_BACKEND_API_VERSION
+        struct wsp_ggml_backend_reg_i iface;
+        void * context;
+    };
+
+    // Internal backend registry API
+    WSP_GGML_API void wsp_ggml_backend_register(wsp_ggml_backend_reg_t reg);
+
+    // Add backend dynamic loading support to the backend
+
+    // Initialize the backend
+    typedef wsp_ggml_backend_reg_t (*wsp_ggml_backend_init_t)(void);
+    // Optional: obtain a score for the backend based on the system configuration
+    // Higher scores are preferred, 0 means the backend is not supported in the current system
+    typedef int                (*wsp_ggml_backend_score_t)(void);
+
+#ifdef WSP_GGML_BACKEND_DL
+#    ifdef __cplusplus
+#        define WSP_GGML_BACKEND_DL_IMPL(reg_fn)                             \
+            extern "C" {                                                 \
+            WSP_GGML_BACKEND_API wsp_ggml_backend_reg_t wsp_ggml_backend_init(void); \
+            }                                                            \
+            wsp_ggml_backend_reg_t wsp_ggml_backend_init(void) {                 \
+                return reg_fn();                                         \
+            }
+#        define WSP_GGML_BACKEND_DL_SCORE_IMPL(score_fn)       \
+            extern "C" {                                   \
+            WSP_GGML_BACKEND_API int wsp_ggml_backend_score(void); \
+            }                                              \
+            int wsp_ggml_backend_score(void) {                 \
+                return score_fn();                         \
+            }
+#    else
+#        define WSP_GGML_BACKEND_DL_IMPL(reg_fn)                              \
+            WSP_GGML_BACKEND_API wsp_ggml_backend_reg_t wsp_ggml_backend_init(void);  \
+            wsp_ggml_backend_reg_t                  wsp_ggml_backend_init(void) { \
+                return reg_fn();                                          \
+            }
+#        define WSP_GGML_BACKEND_DL_SCORE_IMPL(score_fn)        \
+            WSP_GGML_BACKEND_API int wsp_ggml_backend_score(void);  \
+            int                  wsp_ggml_backend_score(void) { \
+                return score_fn();                          \
+            }
+#    endif
+#else
+#    define WSP_GGML_BACKEND_DL_IMPL(reg_fn)
+#    define WSP_GGML_BACKEND_DL_SCORE_IMPL(score_fn)
+#endif
+
+#ifdef  __cplusplus
+}
+#endif