whisper.rn 0.3.0-rc.5 → 0.3.0-rc.7

package/src/index.ts

@@ -3,9 +3,14 @@ import {
   DeviceEventEmitter,
   Platform,
   DeviceEventEmitterStatic,
+  Image,
 } from 'react-native'
 import RNWhisper from './NativeRNWhisper'
-import type { TranscribeOptions, TranscribeResult } from './NativeRNWhisper'
+import type {
+  TranscribeOptions,
+  TranscribeResult,
+  CoreMLAsset,
+} from './NativeRNWhisper'
 import { version } from './version.json'
 
 let EventEmitter: NativeEventEmitter | DeviceEventEmitterStatic
@@ -24,11 +29,11 @@ const EVENT_ON_REALTIME_TRANSCRIBE_END = '@RNWhisper_onRealtimeTranscribeEnd'
 
 export type TranscribeRealtimeOptions = TranscribeOptions & {
   /**
-   * Realtime record max duration in seconds. 
+   * Realtime record max duration in seconds.
    * Due to the whisper.cpp hard constraint - processes the audio in chunks of 30 seconds,
    * the recommended value will be <= 30 seconds. (Default: 30)
    */
-  realtimeAudioSec?: number,
+  realtimeAudioSec?: number
   /**
    * Optimize audio transcription performance by slicing audio samples when `realtimeAudioSec` > 30.
    * Set `realtimeAudioSliceSec` < 30 so performance improvements can be achieved in the Whisper hard constraint (processes the audio in chunks of 30 seconds).
@@ -38,42 +43,42 @@ export type TranscribeRealtimeOptions = TranscribeOptions & {
 }
 
 export type TranscribeRealtimeEvent = {
-  contextId: number,
-  jobId: number,
+  contextId: number
+  jobId: number
   /** Is capturing audio, when false, the event is the final result */
-  isCapturing: boolean,
-  isStoppedByAction?: boolean,
-  code: number,
-  data?: TranscribeResult,
-  error?: string,
-  processTime: number,
-  recordingTime: number,
+  isCapturing: boolean
+  isStoppedByAction?: boolean
+  code: number
+  data?: TranscribeResult
+  error?: string
+  processTime: number
+  recordingTime: number
   slices?: Array<{
-    code: number,
-    error?: string,
-    data?: TranscribeResult,
-    processTime: number,
-    recordingTime: number,
-  }>,
+    code: number
+    error?: string
+    data?: TranscribeResult
+    processTime: number
+    recordingTime: number
+  }>
 }
 
 export type TranscribeRealtimeNativePayload = {
   /** Is capturing audio, when false, the event is the final result */
-  isCapturing: boolean,
-  isStoppedByAction?: boolean,
-  code: number,
-  processTime: number,
-  recordingTime: number,
-  isUseSlices: boolean,
-  sliceIndex: number,
-  data?: TranscribeResult,
-  error?: string,
+  isCapturing: boolean
+  isStoppedByAction?: boolean
+  code: number
+  processTime: number
+  recordingTime: number
+  isUseSlices: boolean
+  sliceIndex: number
+  data?: TranscribeResult
+  error?: string
 }
 
 export type TranscribeRealtimeNativeEvent = {
-  contextId: number,
-  jobId: number,
-  payload: TranscribeRealtimeNativePayload,
+  contextId: number
+  jobId: number
+  payload: TranscribeRealtimeNativePayload
 }
 
 export class WhisperContext {
@@ -84,12 +89,29 @@ export class WhisperContext {
   }
 
   /** Transcribe audio file */
-  transcribe(path: string, options: TranscribeOptions = {}): {
+  transcribe(
+    filePath: string | number,
+    options: TranscribeOptions = {},
+  ): {
     /** Stop the transcribe */
-    stop: () => void,
+    stop: () => void
     /** Transcribe result promise */
-    promise: Promise<TranscribeResult>,
+    promise: Promise<TranscribeResult>
   } {
+    let path = ''
+    if (typeof filePath === 'number') {
+      try {
+        const source = Image.resolveAssetSource(filePath)
+        if (source) path = source.uri
+      } catch (e) {
+        throw new Error(`Invalid asset: ${filePath}`)
+      }
+    } else {
+      if (filePath.startsWith('http'))
+        throw new Error('Transcribe remote file is not supported, please download it first')
+      path = filePath
+    }
+    if (path.startsWith('file://')) path = path.slice(7)
     const jobId: number = Math.floor(Math.random() * 10000)
     return {
       stop: () => RNWhisper.abortTranscribe(this.id, jobId),
@@ -100,9 +122,9 @@ export class WhisperContext {
   /** Transcribe the microphone audio stream, the microphone user permission is required */
   async transcribeRealtime(options: TranscribeRealtimeOptions = {}): Promise<{
     /** Stop the realtime transcribe */
-    stop: () => void,
+    stop: () => void
     /** Subscribe to realtime transcribe events */
-    subscribe: (callback: (event: TranscribeRealtimeEvent) => void) => void,
+    subscribe: (callback: (event: TranscribeRealtimeEvent) => void) => void
   }> {
     const jobId: number = Math.floor(Math.random() * 10000)
     await RNWhisper.startRealtimeTranscribe(this.id, jobId, options)
@@ -118,37 +140,42 @@ export class WhisperContext {
         const { segments = [] } = slices[sliceIndex]?.data || {}
         tOffset = segments[segments.length - 1]?.t1 || 0
       }
-      ({ sliceIndex } = payload)
+      ;({ sliceIndex } = payload)
       slices[sliceIndex] = {
         ...payload,
-        data: payload.data ? {
-          ...payload.data,
-          segments: payload.data.segments.map((segment) => ({
-            ...segment,
-            t0: segment.t0 + tOffset,
-            t1: segment.t1 + tOffset,
-          })) || [],
-        } : undefined,
+        data: payload.data
+          ? {
+              ...payload.data,
+              segments:
+                payload.data.segments.map((segment) => ({
+                  ...segment,
+                  t0: segment.t0 + tOffset,
+                  t1: segment.t1 + tOffset,
+                })) || [],
+            }
+          : undefined,
       }
     }
 
-    const mergeSlicesIfNeeded = (payload: TranscribeRealtimeNativePayload): TranscribeRealtimeNativePayload => {
+    const mergeSlicesIfNeeded = (
+      payload: TranscribeRealtimeNativePayload,
+    ): TranscribeRealtimeNativePayload => {
       if (!payload.isUseSlices) return payload
 
       const mergedPayload: any = {}
-      slices.forEach(
-        (slice) => {
-          mergedPayload.data = {
-            result: (mergedPayload.data?.result || '') + (slice.data?.result || ''),
-            segments: [
-              ...(mergedPayload?.data?.segments || []),
-              ...(slice.data?.segments || []),
-            ],
-          }
-          mergedPayload.processTime = slice.processTime
-          mergedPayload.recordingTime = (mergedPayload?.recordingTime || 0) + slice.recordingTime
+      slices.forEach((slice) => {
+        mergedPayload.data = {
+          result:
+            (mergedPayload.data?.result || '') + (slice.data?.result || ''),
+          segments: [
+            ...(mergedPayload?.data?.segments || []),
+            ...(slice.data?.segments || []),
+          ],
         }
-      )
+        mergedPayload.processTime = slice.processTime
+        mergedPayload.recordingTime =
+          (mergedPayload?.recordingTime || 0) + slice.recordingTime
+      })
       return { ...payload, ...mergedPayload, slices }
     }
 
@@ -167,7 +194,7 @@ export class WhisperContext {
               jobId: evt.jobId,
               ...mergeSlicesIfNeeded(payload),
             })
-          }
+          },
         )
         let endListener: any = EventEmitter.addListener(
           EVENT_ON_REALTIME_TRANSCRIBE_END,
@@ -183,7 +210,7 @@ export class WhisperContext {
               contextId,
               jobId: evt.jobId,
               ...mergeSlicesIfNeeded(lastPayload),
-              isCapturing: false
+              isCapturing: false,
             })
             if (transcribeListener) {
               transcribeListener.remove()
@@ -193,7 +220,7 @@ export class WhisperContext {
               endListener.remove()
               endListener = null
             }
-          }
+          },
         )
       },
     }
@@ -204,10 +231,75 @@ export class WhisperContext {
   }
 }
 
-export async function initWhisper(
-  { filePath, isBundleAsset }: { filePath: string; isBundleAsset?: boolean }
-): Promise<WhisperContext> {
-  const id = await RNWhisper.initContext(filePath, !!isBundleAsset)
+export type ContextOptions = {
+  filePath: string | number
+  /**
+   * CoreML model assets, if you're using `require` on filePath,
+   * use this option is required if you want to enable Core ML,
+   * you will need bundle weights/weight.bin, model.mil, coremldata.bin into app by `require`
+   */
+  coreMLModelAsset?: {
+    filename: string
+    assets: number[]
+  }
+  /** Is the file path a bundle asset for pure string filePath */
+  isBundleAsset?: boolean
+}
+
+const coreMLModelAssetPaths = [
+  'analytics/coremldata.bin',
+  'weights/weight.bin',
+  'model.mil',
+  'coremldata.bin',
+]
+
+export async function initWhisper({
+  filePath,
+  coreMLModelAsset,
+  isBundleAsset,
+}: ContextOptions): Promise<WhisperContext> {
+  let path = ''
+  let coreMLAssets: CoreMLAsset[] | undefined
+  if (coreMLModelAsset) {
+    const { filename, assets } = coreMLModelAsset
+    if (filename && assets) {
+      coreMLAssets = assets
+        ?.map((asset) => {
+          const { uri } = Image.resolveAssetSource(asset)
+          const filepath = coreMLModelAssetPaths.find((p) => uri.includes(p))
+          if (filepath) {
+            return {
+              uri,
+              filepath: `${filename}/${filepath}`,
+            }
+          }
+          return undefined
+        })
+        .filter((asset): asset is CoreMLAsset => asset !== undefined)
+    }
+  }
+  if (typeof filePath === 'number') {
+    try {
+      const source = Image.resolveAssetSource(filePath)
+      if (source) {
+        path = source.uri
+      }
+    } catch (e) {
+      throw new Error(`Invalid asset: ${filePath}`)
+    }
+  } else {
+    if (!isBundleAsset && filePath.startsWith('http'))
+      throw new Error('Transcribe remote file is not supported, please download it first')
+    path = filePath
+  }
+  if (path.startsWith('file://')) path = path.slice(7)
+  const id = await RNWhisper.initContext({
+    filePath: path,
+    isBundleAsset: !!isBundleAsset,
+    // Only development mode need download Core ML model assets (from packager server)
+    downloadCoreMLAssets: __DEV__ && !!coreMLAssets,
+    coreMLAssets,
+  })
   return new WhisperContext(id)
 }