whisper.rn 0.4.0-rc.8 → 0.4.0-rc.9

package/src/index.ts

@@ -174,6 +174,15 @@ export type TranscribeRealtimeNativeEvent = {
   payload: TranscribeRealtimeNativePayload
 }
 
+export type BenchResult = {
+  config: string
+  nThreads: number
+  encodeMs: number
+  decodeMs: number
+  batchMs: number
+  promptMs: number
+}
+
 const updateAudioSession = async (setting: AudioSessionSettingIos) => {
   await AudioSessionIos.setCategory(
     setting.category,
@@ -202,32 +211,10 @@ export class WhisperContext {
     this.reasonNoGPU = reasonNoGPU
   }
 
-  /** Transcribe audio file */
-  transcribe(
-    filePath: string | number,
-    options: TranscribeFileOptions = {},
-  ): {
-    /** Stop the transcribe */
+  private transcribeWithNativeMethod(method: 'transcribeFile' | 'transcribeData', data: string, options: TranscribeFileOptions = {}): {
     stop: () => Promise<void>
-    /** Transcribe result promise */
     promise: Promise<TranscribeResult>
   } {
-    let path = ''
-    if (typeof filePath === 'number') {
-      try {
-        const source = Image.resolveAssetSource(filePath)
-        if (source) path = source.uri
-      } catch (e) {
-        throw new Error(`Invalid asset: ${filePath}`)
-      }
-    } else {
-      if (filePath.startsWith('http'))
-        throw new Error(
-          'Transcribe remote file is not supported, please download it first',
-        )
-      path = filePath
-    }
-    if (path.startsWith('file://')) path = path.slice(7)
     const jobId: number = Math.floor(Math.random() * 10000)
 
     const { onProgress, onNewSegments, ...rest } = options
@@ -276,7 +263,7 @@ export class WhisperContext {
         removeProgressListener()
         removeNewSegmenetsListener()
       },
-      promise: RNWhisper.transcribeFile(this.id, jobId, path, {
+      promise: RNWhisper[method](this.id, jobId, data, {
         ...rest,
         onProgress: !!onProgress,
         onNewSegments: !!onNewSegments,
@@ -298,6 +285,48 @@ export class WhisperContext {
     }
   }
 
+  /**
+   * Transcribe audio file (path or base64 encoded wav file)
+   * base64: need add `data:audio/wav;base64,` prefix
+   */
+  transcribe(
+    filePathOrBase64: string | number,
+    options: TranscribeFileOptions = {},
+  ): {
+    /** Stop the transcribe */
+    stop: () => Promise<void>
+    /** Transcribe result promise */
+    promise: Promise<TranscribeResult>
+  } {
+    let path = ''
+    if (typeof filePathOrBase64 === 'number') {
+      try {
+        const source = Image.resolveAssetSource(filePathOrBase64)
+        if (source) path = source.uri
+      } catch (e) {
+        throw new Error(`Invalid asset: ${filePathOrBase64}`)
+      }
+    } else {
+      if (filePathOrBase64.startsWith('http'))
+        throw new Error(
+          'Transcribe remote file is not supported, please download it first',
+        )
+      path = filePathOrBase64
+    }
+    if (path.startsWith('file://')) path = path.slice(7)
+    return this.transcribeWithNativeMethod('transcribeFile', path, options)
+  }
+
+  /**
+   * Transcribe audio data (base64 encoded float32 PCM data)
+   */
+  transcribeData(data: string, options: TranscribeFileOptions = {}): {
+    stop: () => Promise<void>
+    promise: Promise<TranscribeResult>
+  } {
+    return this.transcribeWithNativeMethod('transcribeData', data, options)
+  }
+
   /** Transcribe the microphone audio stream, the microphone user permission is required */
   async transcribeRealtime(options: TranscribeRealtimeOptions = {}): Promise<{
     /** Stop the realtime transcribe */
@@ -433,6 +462,12 @@ export class WhisperContext {
     }
   }
 
+  async bench(maxThreads: number): Promise<BenchResult> {
+    const result = await RNWhisper.bench(this.id, maxThreads)
+    const [config, nThreads, encodeMs, decodeMs, batchMs, promptMs] = JSON.parse(result)
+    return { config, nThreads, encodeMs, decodeMs, batchMs, promptMs } as BenchResult
+  }
+
   async release(): Promise<void> {
     return RNWhisper.releaseContext(this.id)
   }
@@ -455,6 +490,8 @@ export type ContextOptions = {
   useCoreMLIos?: boolean
   /** Use GPU if available. Currently iOS only, if it's enabled, Core ML option will be ignored. */
   useGpu?: boolean
+  /** Use Flash Attention, only recommended if GPU available */
+  useFlashAttn?: boolean,
 }
 
 const coreMLModelAssetPaths = [
@@ -470,6 +507,7 @@ export async function initWhisper({
   isBundleAsset,
   useGpu = true,
   useCoreMLIos = true,
+  useFlashAttn = false,
 }: ContextOptions): Promise<WhisperContext> {
   let path = ''
   let coreMLAssets: CoreMLAsset[] | undefined
@@ -518,6 +556,7 @@ export async function initWhisper({
   const { contextId, gpu, reasonNoGPU } = await RNWhisper.initContext({
     filePath: path,
     isBundleAsset: !!isBundleAsset,
+    useFlashAttn,
     useGpu,
     useCoreMLIos,
     // Only development mode need download Core ML model assets (from packager server)

package/src/version.json

@@ -1 +1 @@
-{"version":"1.5.4"}
+{"version":"1.7.1"}

package/whisper-rn.podspec

@@ -31,8 +31,10 @@ Pod::Spec.new do |s|
   s.platforms    = { :ios => "11.0", :tvos => "11.0" }
   s.source       = { :git => "https://github.com/mybigday/whisper.rn.git", :tag => "#{s.version}" }
 
-  s.source_files = "ios/**/*.{h,m,mm}", "cpp/**/*.{h,cpp,c,m,mm}"
-  s.resources = "cpp/**/*.{metal}"
+  s.source_files = "ios/**/*.{h,m,mm}", "cpp/*.{h,cpp,c}", "cpp/coreml/*.{h,m,mm}"
+  s.resources = "cpp/*.{metallib}"
+
+  s.requires_arc = true
 
   s.dependency "React-Core"
 
@@ -60,4 +62,9 @@ Pod::Spec.new do |s|
     s.dependency "RCTTypeSafety"
     s.dependency "ReactCommon/turbomodule/core"
   end
+
+  s.subspec "no-require-arc" do |ss|
+    ss.requires_arc = false
+    ss.source_files = "cpp/*.m"
+  end
 end