whisper.rn 0.4.0-rc.8 → 0.4.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "whisper.rn",
-  "version": "0.4.0-rc.8",
+  "version": "0.4.0",
   "description": "React Native binding of whisper.cpp",
   "main": "lib/commonjs/index",
   "module": "lib/module/index",
@@ -12,9 +12,8 @@
     "lib",
     "jest",
     "android",
+    "cpp",
     "ios",
-    "cpp/*.*",
-    "cpp/coreml/*.*",
     "*.podspec",
     "!lib/typescript/example",
     "!ios/build",
@@ -30,13 +29,18 @@
   ],
   "scripts": {
     "bootstrap": "./scripts/bootstrap.sh",
-    "docgen": "typedoc src/index.ts --plugin typedoc-plugin-markdown --readme none --out docs/API",
+    "docgen": "typedoc src/index.ts --plugin typedoc-plugin-markdown --excludePrivate --readme none --out docs/API",
     "lint": "eslint \"**/*.{js,ts,tsx}\"",
     "typecheck": "tsc --noEmit",
-    "prepack": "yarn docgen && bob build",
+    "build": "bob build",
+    "prepack": "yarn docgen && ./scripts/build-ios.sh && ./scripts/build-android.sh && yarn build",
     "test": "jest",
     "example": "yarn --cwd example",
-    "clean": "del-cli android/build example/android/build example/android/app/build example/ios/build"
+    "clean": "del-cli android/build example/android/build example/android/app/build example/ios/build",
+    "build:ios-frameworks": "./scripts/build-ios.sh",
+    "build:ios": "cd example/ios && xcodebuild -workspace RNWhisperExample.xcworkspace -scheme RNWhisperExample -configuration Debug -sdk iphonesimulator CC=clang CPLUSPLUS=clang++ LD=clang LDPLUSPLUS=clang++ GCC_OPTIMIZATION_LEVEL=0 GCC_PRECOMPILE_PREFIX_HEADER=YES ASSETCATALOG_COMPILER_OPTIMIZATION=time DEBUG_INFORMATION_FORMAT=dwarf COMPILER_INDEX_STORE_ENABLE=NO",
+    "build:android-libs": "./scripts/build-android.sh",
+    "build:android": "cd example/android && ./gradlew assembleDebug"
   },
   "keywords": [
     "react-native",
@@ -59,21 +63,21 @@
     "@commitlint/config-conventional": "^17.0.2",
     "@evilmartians/lefthook": "^1.2.2",
     "@fugood/eslint-config-react": "^0.5.0",
+    "@react-native/babel-preset": "0.74.88",
+    "@react-native/metro-config": "^0.73.2",
     "@release-it/conventional-changelog": "^5.0.0",
     "@types/jest": "^29.4.4",
     "@types/react": "^18.2.6",
     "@types/react-native": "0.70.0",
     "@typescript-eslint/eslint-plugin": "^5.55.0",
     "@typescript-eslint/parser": "^5.55.0",
-    "@react-native/babel-preset": "^0.73.18",
-    "@react-native/metro-config": "^0.73.2",
     "commitlint": "^17.0.2",
     "del-cli": "^5.0.0",
     "eslint": "^8.36.0",
     "jest": "^29.5.0",
     "pod-install": "^0.1.38",
     "react": "18.2.0",
-    "react-native": "0.73.0",
+    "react-native": "0.74.6",
     "react-native-builder-bob": "^0.20.4",
     "release-it": "^15.8.0",
     "typedoc": "^0.24.7",
@@ -142,5 +146,6 @@
     "name": "RNWhisperSpec",
     "type": "all",
     "jsSrcsDir": "./src/"
-  }
+  },
+  "packageManager": "yarn@1.22.22"
 }

package/src/NativeRNWhisper.ts

@@ -15,6 +15,8 @@ export type TranscribeOptions = {
   maxLen?: number,
   /** Enable token-level timestamps */
   tokenTimestamps?: boolean,
+  /** Enable tinydiarize (requires a tdrz model) */
+  tdrzEnable?: boolean,
   /** Word timestamp probability threshold */
   wordThold?: number,
   /** Time offset in milliseconds */
@@ -28,8 +30,6 @@ export type TranscribeOptions = {
   beamSize?: number,
   /** Number of best candidates to keep */
   bestOf?: number,
-  /** Speed up audio by x2 (reduced accuracy) */
-  speedUp?: boolean,
   /** Initial Prompt */
   prompt?: string,
 }
@@ -52,6 +52,7 @@ export type CoreMLAsset = {
 type NativeContextOptions = {
   filePath: string,
   isBundleAsset: boolean,
+  useFlashAttn?: boolean,
   useGpu?: boolean,
   useCoreMLIos?: boolean,
   downloadCoreMLAssets?: boolean,
@@ -75,7 +76,13 @@ export interface Spec extends TurboModule {
   transcribeFile(
     contextId: number,
     jobId: number,
-    path: string,
+    pathOrBase64: string,
+    options: {}, // TranscribeOptions & { onProgress?: boolean, onNewSegments?: boolean }
+  ): Promise<TranscribeResult>;
+  transcribeData(
+    contextId: number,
+    jobId: number,
+    dataBase64: string,
     options: {}, // TranscribeOptions & { onProgress?: boolean, onNewSegments?: boolean }
   ): Promise<TranscribeResult>;
   startRealtimeTranscribe(
@@ -85,6 +92,8 @@ export interface Spec extends TurboModule {
   ): Promise<void>;
   abortTranscribe(contextId: number, jobId: number): Promise<void>;
 
+  bench(contextId: number, maxThreads: number): Promise<string>;
+
   // iOS specific
   getAudioSessionCurrentCategory: () => Promise<{
     category: string,

package/src/index.ts

@@ -174,6 +174,15 @@ export type TranscribeRealtimeNativeEvent = {
   payload: TranscribeRealtimeNativePayload
 }
 
+export type BenchResult = {
+  config: string
+  nThreads: number
+  encodeMs: number
+  decodeMs: number
+  batchMs: number
+  promptMs: number
+}
+
 const updateAudioSession = async (setting: AudioSessionSettingIos) => {
   await AudioSessionIos.setCategory(
     setting.category,
@@ -202,32 +211,10 @@ export class WhisperContext {
     this.reasonNoGPU = reasonNoGPU
   }
 
-  /** Transcribe audio file */
-  transcribe(
-    filePath: string | number,
-    options: TranscribeFileOptions = {},
-  ): {
-    /** Stop the transcribe */
+  private transcribeWithNativeMethod(method: 'transcribeFile' | 'transcribeData', data: string, options: TranscribeFileOptions = {}): {
     stop: () => Promise<void>
-    /** Transcribe result promise */
     promise: Promise<TranscribeResult>
   } {
-    let path = ''
-    if (typeof filePath === 'number') {
-      try {
-        const source = Image.resolveAssetSource(filePath)
-        if (source) path = source.uri
-      } catch (e) {
-        throw new Error(`Invalid asset: ${filePath}`)
-      }
-    } else {
-      if (filePath.startsWith('http'))
-        throw new Error(
-          'Transcribe remote file is not supported, please download it first',
-        )
-      path = filePath
-    }
-    if (path.startsWith('file://')) path = path.slice(7)
     const jobId: number = Math.floor(Math.random() * 10000)
 
     const { onProgress, onNewSegments, ...rest } = options
@@ -276,7 +263,7 @@ export class WhisperContext {
         removeProgressListener()
         removeNewSegmenetsListener()
       },
-      promise: RNWhisper.transcribeFile(this.id, jobId, path, {
+      promise: RNWhisper[method](this.id, jobId, data, {
         ...rest,
         onProgress: !!onProgress,
         onNewSegments: !!onNewSegments,
@@ -298,6 +285,48 @@ export class WhisperContext {
     }
   }
 
+  /**
+   * Transcribe audio file (path or base64 encoded wav file)
+   * base64: need add `data:audio/wav;base64,` prefix
+   */
+  transcribe(
+    filePathOrBase64: string | number,
+    options: TranscribeFileOptions = {},
+  ): {
+    /** Stop the transcribe */
+    stop: () => Promise<void>
+    /** Transcribe result promise */
+    promise: Promise<TranscribeResult>
+  } {
+    let path = ''
+    if (typeof filePathOrBase64 === 'number') {
+      try {
+        const source = Image.resolveAssetSource(filePathOrBase64)
+        if (source) path = source.uri
+      } catch (e) {
+        throw new Error(`Invalid asset: ${filePathOrBase64}`)
+      }
+    } else {
+      if (filePathOrBase64.startsWith('http'))
+        throw new Error(
+          'Transcribe remote file is not supported, please download it first',
+        )
+      path = filePathOrBase64
+    }
+    if (path.startsWith('file://')) path = path.slice(7)
+    return this.transcribeWithNativeMethod('transcribeFile', path, options)
+  }
+
+  /**
+   * Transcribe audio data (base64 encoded float32 PCM data)
+   */
+  transcribeData(data: string, options: TranscribeFileOptions = {}): {
+    stop: () => Promise<void>
+    promise: Promise<TranscribeResult>
+  } {
+    return this.transcribeWithNativeMethod('transcribeData', data, options)
+  }
+
   /** Transcribe the microphone audio stream, the microphone user permission is required */
   async transcribeRealtime(options: TranscribeRealtimeOptions = {}): Promise<{
     /** Stop the realtime transcribe */
@@ -433,6 +462,12 @@ export class WhisperContext {
     }
   }
 
+  async bench(maxThreads: number): Promise<BenchResult> {
+    const result = await RNWhisper.bench(this.id, maxThreads)
+    const [config, nThreads, encodeMs, decodeMs, batchMs, promptMs] = JSON.parse(result)
+    return { config, nThreads, encodeMs, decodeMs, batchMs, promptMs } as BenchResult
+  }
+
   async release(): Promise<void> {
     return RNWhisper.releaseContext(this.id)
   }
@@ -455,6 +490,8 @@ export type ContextOptions = {
   useCoreMLIos?: boolean
   /** Use GPU if available. Currently iOS only, if it's enabled, Core ML option will be ignored. */
   useGpu?: boolean
+  /** Use Flash Attention, only recommended if GPU available */
+  useFlashAttn?: boolean,
 }
 
 const coreMLModelAssetPaths = [
@@ -470,6 +507,7 @@ export async function initWhisper({
   isBundleAsset,
   useGpu = true,
   useCoreMLIos = true,
+  useFlashAttn = false,
 }: ContextOptions): Promise<WhisperContext> {
   let path = ''
   let coreMLAssets: CoreMLAsset[] | undefined
@@ -518,6 +556,7 @@ export async function initWhisper({
   const { contextId, gpu, reasonNoGPU } = await RNWhisper.initContext({
     filePath: path,
     isBundleAsset: !!isBundleAsset,
+    useFlashAttn,
     useGpu,
     useCoreMLIos,
     // Only development mode need download Core ML model assets (from packager server)

package/src/version.json

@@ -1 +1 @@
-{"version":"1.5.4"}
+{"version":"1.7.6"}

package/whisper-rn.podspec

@@ -2,7 +2,7 @@ require "json"
 
 package = JSON.parse(File.read(File.join(__dir__, "package.json")))
 base_ld_flags = "-framework Accelerate -framework Foundation -framework Metal -framework MetalKit"
-base_compiler_flags = "-DWSP_GGML_USE_ACCELERATE -Wno-shorten-64-to-32"
+base_compiler_flags = "-DWSP_GGML_USE_CPU -DWSP_GGML_USE_ACCELERATE -Wno-shorten-64-to-32"
 folly_compiler_flags = "-DFOLLY_NO_CONFIG -DFOLLY_MOBILE=1 -DFOLLY_USE_LIBCPP=1 -Wno-comma"
 
 # Use base_optimizer_flags = "" for debug builds
@@ -31,8 +31,16 @@ Pod::Spec.new do |s|
   s.platforms    = { :ios => "11.0", :tvos => "11.0" }
   s.source       = { :git => "https://github.com/mybigday/whisper.rn.git", :tag => "#{s.version}" }
 
-  s.source_files = "ios/**/*.{h,m,mm}", "cpp/**/*.{h,cpp,c,m,mm}"
-  s.resources = "cpp/**/*.{metal}"
+  if ENV["RNWHISPER_BUILD_FROM_SOURCE"] == "1"
+    s.source_files = "ios/**/*.{h,m,mm}", "cpp/**/*.{h,cpp,hpp,c,m,mm}"
+    s.resources = "cpp/**/*.{metallib}"
+    base_compiler_flags += " -DRNWHISPER_BUILD_FROM_SOURCE"
+  else
+    s.source_files = "ios/**/*.{h,m,mm}"
+    s.vendored_frameworks = "ios/rnwhisper.xcframework"
+  end
+
+  s.requires_arc = true
 
   s.dependency "React-Core"
 
@@ -40,24 +48,16 @@ Pod::Spec.new do |s|
   s.pod_target_xcconfig = {
     "OTHER_LDFLAGS" => base_ld_flags,
     "OTHER_CFLAGS" => base_optimizer_flags,
-    "OTHER_CPLUSPLUSFLAGS" => base_optimizer_flags
+    "OTHER_CPLUSPLUSFLAGS" => base_optimizer_flags + " -std=c++17"
   }
 
   # Don't install the dependencies when we run `pod install` in the old architecture.
   if ENV['RCT_NEW_ARCH_ENABLED'] == '1' then
-    s.compiler_flags = base_compiler_flags + " " + folly_compiler_flags + " -DRCT_NEW_ARCH_ENABLED=1"
-    new_arch_cpp_flags = "-DFOLLY_NO_CONFIG -DFOLLY_MOBILE=1 -DFOLLY_USE_LIBCPP=1"
-    s.pod_target_xcconfig = {
-      "CLANG_CXX_LANGUAGE_STANDARD" => "c++17",
-      "HEADER_SEARCH_PATHS" => "\"$(PODS_ROOT)/boost\"",
-      "OTHER_LDFLAGS" => base_ld_flags,
-      "OTHER_CFLAGS" => base_optimizer_flags,
-      "OTHER_CPLUSPLUSFLAGS" => new_arch_cpp_flags + " " + base_optimizer_flags
-    }
-    s.dependency "React-Codegen"
-    s.dependency "RCT-Folly"
-    s.dependency "RCTRequired"
-    s.dependency "RCTTypeSafety"
-    s.dependency "ReactCommon/turbomodule/core"
+    install_modules_dependencies(s)
+  end
+
+  s.subspec "no-require-arc" do |ss|
+    ss.requires_arc = false
+    ss.source_files = "cpp/*.m"
   end
 end

package/cpp/README.md

@@ -1,4 +0,0 @@
-# Note
-
-- Only `rn-*` are the specific files for this project, others are sync from [whisper.cpp](https://github.com/ggerganov/whisper.cpp).
-- We can update the native source by using the [bootstrap](../scripts/bootstrap.sh) script.