whisper.rn 0.5.0-rc.0 → 0.5.0-rc.10

package/src/realtime-transcription/SliceManager.ts

@@ -0,0 +1,252 @@
+import type { AudioSlice, MemoryUsage } from './types'
+
+export class SliceManager {
+  private slices: AudioSlice[] = []
+
+  private currentSliceIndex = 0
+
+  private transcribeSliceIndex = 0
+
+  private maxSlicesInMemory: number
+
+  private sliceDurationSec: number
+
+  private sampleRate: number
+
+  constructor(
+    sliceDurationSec = 30,
+    maxSlicesInMemory = 1,
+    sampleRate = 16000,
+  ) {
+    this.sliceDurationSec = sliceDurationSec
+    this.maxSlicesInMemory = maxSlicesInMemory
+    this.sampleRate = sampleRate
+  }
+
+  /**
+   * Add audio data to the current slice
+   */
+  addAudioData(audioData: Uint8Array): {
+    slice?: AudioSlice
+  } {
+    // Get or create current slice
+    const currentSlice = this.getCurrentSlice()
+
+    // Calculate bytes per slice (2 bytes per sample for 16-bit PCM)
+    const bytesPerSlice = this.sliceDurationSec * this.sampleRate * 2
+
+    // Check if adding this data would exceed slice capacity
+    if (currentSlice.sampleCount + audioData.length > bytesPerSlice) {
+      // Finalize current slice and create new one
+      this.finalizeCurrentSlice()
+      this.currentSliceIndex += 1
+      return this.addAudioData(audioData) // Recursively add to new slice
+    }
+
+    // Append data to current slice
+    const newData = new Uint8Array(currentSlice.sampleCount + audioData.length)
+    newData.set(currentSlice.data.subarray(0, currentSlice.sampleCount))
+    newData.set(audioData, currentSlice.sampleCount)
+
+    currentSlice.data = newData
+    currentSlice.sampleCount += audioData.length
+    currentSlice.endTime = Date.now()
+
+    // Check if slice is complete
+    const isSliceComplete = currentSlice.sampleCount >= bytesPerSlice * 0.8 // 80% full
+
+    if (isSliceComplete) {
+      this.finalizeCurrentSlice()
+    }
+
+    return { slice: currentSlice }
+  }
+
+  /**
+   * Get the current slice being built
+   */
+  private getCurrentSlice(): AudioSlice {
+    let slice = this.slices.find((s) => s.index === this.currentSliceIndex)
+
+    if (!slice) {
+      const bytesPerSlice = this.sliceDurationSec * this.sampleRate * 2 // 2 bytes per sample
+      slice = {
+        index: this.currentSliceIndex,
+        data: new Uint8Array(bytesPerSlice),
+        sampleCount: 0,
+        startTime: Date.now(),
+        endTime: Date.now(),
+        isProcessed: false,
+        isReleased: false,
+      }
+      this.slices.push(slice)
+
+      // Clean up old slices if we have too many
+      this.cleanupOldSlices()
+    }
+
+    return slice
+  }
+
+  /**
+   * Finalize the current slice
+   */
+  private finalizeCurrentSlice(): void {
+    const slice = this.slices.find((s) => s.index === this.currentSliceIndex)
+    if (slice && slice.sampleCount > 0) {
+      // Trim the data array to actual size
+      slice.data = slice.data.subarray(0, slice.sampleCount)
+      slice.endTime = Date.now()
+    }
+  }
+
+  /**
+   * Get a slice for transcription
+   */
+  getSliceForTranscription(): AudioSlice | null {
+    const slice = this.slices.find(
+      (s) => s.index === this.transcribeSliceIndex && !s.isProcessed,
+    )
+
+    if (slice && slice.sampleCount > 0) {
+      return slice
+    }
+
+    return null
+  }
+
+  /**
+   * Mark a slice as processed
+   */
+  markSliceAsProcessed(sliceIndex: number): void {
+    const slice = this.slices.find((s) => s.index === sliceIndex)
+    if (slice) {
+      slice.isProcessed = true
+    }
+  }
+
+  /**
+   * Move to the next slice for transcription
+   */
+  moveToNextTranscribeSlice(): void {
+    this.transcribeSliceIndex += 1
+  }
+
+  /**
+   * Get audio data for transcription (base64 encoded)
+   */
+  getAudioDataForTranscription(sliceIndex: number): Uint8Array | null {
+    const slice = this.slices.find((s) => s.index === sliceIndex)
+    if (!slice || slice.sampleCount === 0) {
+      return null
+    }
+
+    return slice.data.subarray(0, slice.sampleCount)
+  }
+
+  /**
+   * Get a slice by index
+   */
+  getSliceByIndex(sliceIndex: number): AudioSlice | null {
+    return this.slices.find((s) => s.index === sliceIndex) || null
+  }
+
+  /**
+   * Clean up old slices to manage memory
+   */
+  private cleanupOldSlices(): void {
+    if (this.slices.length <= this.maxSlicesInMemory) {
+      return
+    }
+
+    // Sort slices by index
+    this.slices.sort((a, b) => a.index - b.index)
+
+    // Keep only the most recent slices
+    const slicesToKeep = this.slices.slice(-this.maxSlicesInMemory)
+    const slicesToRemove = this.slices.slice(0, -this.maxSlicesInMemory)
+
+    // Release old slices
+    slicesToRemove.forEach((slice) => {
+      if (!slice.isReleased) {
+        slice.isReleased = true
+        // Clear the audio data to free memory
+        slice.data = new Uint8Array(0)
+      }
+    })
+
+    this.slices = slicesToKeep
+  }
+
+  /**
+   * Get memory usage statistics
+   */
+  getMemoryUsage(): MemoryUsage {
+    const activeSlices = this.slices.filter((s) => !s.isReleased)
+    const totalBytes = activeSlices.reduce(
+      (sum, slice) => sum + slice.sampleCount,
+      0,
+    )
+
+    // Estimate memory usage (Uint8Array = 1 byte per sample)
+    const estimatedMB = totalBytes / (1024 * 1024)
+
+    return {
+      slicesInMemory: activeSlices.length,
+      totalSamples: totalBytes / 2, // Convert bytes to samples (2 bytes per sample)
+      estimatedMB: Math.round(estimatedMB * 100) / 100, // Round to 2 decimal places
+    }
+  }
+
+  /**
+   * Reset all slices and indices
+   */
+  reset(): void {
+    // Release all slices
+    this.slices.forEach((slice) => {
+      slice.isReleased = true
+      // Clear the audio data to free memory
+      slice.data = new Uint8Array(0)
+    })
+
+    // Reset state
+    this.slices = []
+    this.currentSliceIndex = 0
+    this.transcribeSliceIndex = 0
+  }
+
+  /**
+   * Get current slice information
+   */
+  getCurrentSliceInfo() {
+    return {
+      currentSliceIndex: this.currentSliceIndex,
+      transcribeSliceIndex: this.transcribeSliceIndex,
+      totalSlices: this.slices.length,
+      memoryUsage: this.getMemoryUsage(),
+    }
+  }
+
+  /**
+   * Force move to the next slice, finalizing the current one regardless of capacity
+   */
+  forceNextSlice(): { slice?: AudioSlice } {
+    const currentSlice = this.slices.find(
+      (s) => s.index === this.currentSliceIndex,
+    )
+
+    if (currentSlice && currentSlice.sampleCount > 0) {
+      // Finalize current slice
+      this.finalizeCurrentSlice()
+
+      // Move to next slice
+      this.currentSliceIndex += 1
+
+      return { slice: currentSlice }
+    }
+
+    // If no current slice or it's empty, just move to next index
+    this.currentSliceIndex += 1
+    return {}
+  }
+}

package/src/realtime-transcription/adapters/AudioPcmStreamAdapter.ts

@@ -0,0 +1,143 @@
+/* eslint-disable import/no-extraneous-dependencies */
+// @ts-ignore
+import LiveAudioStream from '@fugood/react-native-audio-pcm-stream'
+import type { AudioStreamInterface, AudioStreamConfig, AudioStreamData } from '../types'
+import { base64ToUint8Array } from '../../utils/common'
+
+export class AudioPcmStreamAdapter implements AudioStreamInterface {
+  private isInitialized = false
+
+  private recording = false
+
+  private config: AudioStreamConfig | null = null
+
+  private dataCallback?: (data: AudioStreamData) => void
+
+  private errorCallback?: (error: string) => void
+
+  private statusCallback?: (isRecording: boolean) => void
+
+  async initialize(config: AudioStreamConfig): Promise<void> {
+    if (this.isInitialized) {
+      await this.release()
+    }
+
+    try {
+      this.config = config || null
+
+      // Initialize LiveAudioStream
+      LiveAudioStream.init({
+        sampleRate: config.sampleRate || 16000,
+        channels: config.channels || 1,
+        bitsPerSample: config.bitsPerSample || 16,
+        audioSource: config.audioSource || 6,
+        bufferSize: config.bufferSize || 16 * 1024,
+        wavFile: '', // We handle file writing separately
+      })
+
+      // Set up data listener
+      LiveAudioStream.on('data', this.handleAudioData.bind(this))
+
+      this.isInitialized = true
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : 'Unknown initialization error'
+      this.errorCallback?.(errorMessage)
+      throw new Error(`Failed to initialize LiveAudioStream: ${errorMessage}`)
+    }
+  }
+
+  async start(): Promise<void> {
+    if (!this.isInitialized) {
+      throw new Error('AudioStream not initialized')
+    }
+
+    if (this.recording) {
+      return
+    }
+
+    try {
+      LiveAudioStream.start()
+      this.recording = true
+      this.statusCallback?.(true)
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : 'Unknown start error'
+      this.errorCallback?.(errorMessage)
+      throw new Error(`Failed to start recording: ${errorMessage}`)
+    }
+  }
+
+  async stop(): Promise<void> {
+    if (!this.recording) {
+      return
+    }
+
+    try {
+      await LiveAudioStream.stop()
+      this.recording = false
+      this.statusCallback?.(false)
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : 'Unknown stop error'
+      this.errorCallback?.(errorMessage)
+      throw new Error(`Failed to stop recording: ${errorMessage}`)
+    }
+  }
+
+  isRecording(): boolean {
+    return this.recording
+  }
+
+  onData(callback: (data: AudioStreamData) => void): void {
+    this.dataCallback = callback
+  }
+
+  onError(callback: (error: string) => void): void {
+    this.errorCallback = callback
+  }
+
+  onStatusChange(callback: (isRecording: boolean) => void): void {
+    this.statusCallback = callback
+  }
+
+  async release(): Promise<void> {
+    if (this.recording) {
+      await this.stop()
+    }
+
+    try {
+      // LiveAudioStream doesn't have an explicit release method
+      // But we should remove listeners and reset state
+      this.isInitialized = false
+      this.config = null
+      this.dataCallback = undefined
+      this.errorCallback = undefined
+      this.statusCallback = undefined
+    } catch (error) {
+      console.warn('Error during LiveAudioStream release:', error)
+    }
+  }
+
+  /**
+   * Handle incoming audio data from LiveAudioStream
+   */
+  private handleAudioData(base64Data: string): void {
+    if (!this.dataCallback) {
+      return
+    }
+
+    try {
+      const audioData = base64ToUint8Array(base64Data)
+
+      const streamData: AudioStreamData = {
+        data: audioData,
+        sampleRate: this.config?.sampleRate || 16000,
+        channels: this.config?.channels || 1,
+        timestamp: Date.now(),
+      }
+
+      this.dataCallback(streamData)
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : 'Audio processing error'
+      this.errorCallback?.(errorMessage)
+    }
+  }
+}

package/src/realtime-transcription/adapters/JestAudioStreamAdapter.ts

@@ -0,0 +1,251 @@
+import type { AudioStreamInterface, AudioStreamConfig, AudioStreamData } from '../types'
+
+export interface JestAudioStreamAdapterOptions {
+  sampleRate?: number
+  channels?: number
+  bitsPerSample?: number
+  simulateLatency?: number // milliseconds
+  simulateErrors?: boolean
+  simulateStartErrorOnly?: boolean // only simulate errors on start, not initialize
+  chunkSize?: number // bytes per chunk
+  chunkInterval?: number // milliseconds between chunks
+  maxChunks?: number // maximum number of chunks to send
+  audioData?: Uint8Array // pre-defined audio data to stream
+  generateSilence?: boolean // generate silence if no audioData provided
+}
+
+export class JestAudioStreamAdapter implements AudioStreamInterface {
+  private config: AudioStreamConfig | null = null
+
+  private options: JestAudioStreamAdapterOptions
+
+  private isInitialized = false
+
+  private recording = false
+
+  private dataCallback?: (data: AudioStreamData) => void
+
+  private errorCallback?: (error: string) => void
+
+  private statusCallback?: (isRecording: boolean) => void
+
+  private streamInterval?: ReturnType<typeof setTimeout>
+
+  private chunksSent = 0
+
+  private startTime = 0
+
+  constructor(options: JestAudioStreamAdapterOptions = {}) {
+    this.options = {
+      sampleRate: 16000,
+      channels: 1,
+      bitsPerSample: 16,
+      simulateLatency: 0,
+      simulateErrors: false,
+      chunkSize: 3200, // 100ms at 16kHz, 16-bit, mono
+      chunkInterval: 100, // 100ms
+      maxChunks: -1, // unlimited
+      generateSilence: true,
+      ...options,
+    }
+  }
+
+  async initialize(config: AudioStreamConfig): Promise<void> {
+    if (this.isInitialized) {
+      await this.release()
+    }
+
+    if (this.options.simulateLatency! > 0) {
+      await JestAudioStreamAdapter.delay(this.options.simulateLatency!)
+    }
+
+    if (this.options.simulateErrors && !this.options.simulateStartErrorOnly) {
+      throw new Error('Simulated initialization error')
+    }
+
+    this.config = config
+    this.isInitialized = true
+  }
+
+  async start(): Promise<void> {
+    if (!this.isInitialized) {
+      throw new Error('AudioStream not initialized')
+    }
+
+    if (this.recording) {
+      return
+    }
+
+    if (this.options.simulateLatency! > 0) {
+      await JestAudioStreamAdapter.delay(this.options.simulateLatency!)
+    }
+
+    if (this.options.simulateErrors) {
+      throw new Error('Simulated start error')
+    }
+
+    this.recording = true
+    this.chunksSent = 0
+    this.startTime = Date.now()
+    this.statusCallback?.(true)
+    this.startStreaming()
+  }
+
+  async stop(): Promise<void> {
+    if (!this.recording) {
+      return
+    }
+
+    if (this.options.simulateLatency! > 0) {
+      await JestAudioStreamAdapter.delay(this.options.simulateLatency!)
+    }
+
+    this.recording = false
+    this.statusCallback?.(false)
+
+    if (this.streamInterval) {
+      clearTimeout(this.streamInterval)
+      this.streamInterval = undefined
+    }
+  }
+
+  isRecording(): boolean {
+    return this.recording
+  }
+
+  onData(callback: (data: AudioStreamData) => void): void {
+    this.dataCallback = callback
+  }
+
+  onError(callback: (error: string) => void): void {
+    this.errorCallback = callback
+  }
+
+  onStatusChange(callback: (isRecording: boolean) => void): void {
+    this.statusCallback = callback
+  }
+
+  async release(): Promise<void> {
+    if (this.recording) {
+      await this.stop()
+    }
+
+    this.isInitialized = false
+    this.config = null
+    this.dataCallback = undefined
+    this.errorCallback = undefined
+    this.statusCallback = undefined
+    this.chunksSent = 0
+  }
+
+  // Test helper methods
+  simulateError(error: string): void {
+    this.errorCallback?.(error)
+  }
+
+  simulateDataChunk(data: Uint8Array): void {
+    if (!this.dataCallback || !this.config) {
+      return
+    }
+
+    const streamData: AudioStreamData = {
+      data,
+      sampleRate: this.config.sampleRate || this.options.sampleRate!,
+      channels: this.config.channels || this.options.channels!,
+      timestamp: Date.now(),
+    }
+
+    this.dataCallback(streamData)
+  }
+
+  getChunksSent(): number {
+    return this.chunksSent
+  }
+
+  getTotalBytesStreamed(): number {
+    return this.chunksSent * this.options.chunkSize!
+  }
+
+  getStreamDuration(): number {
+    return this.recording ? Date.now() - this.startTime : 0
+  }
+
+    private startStreaming(): void {
+    if (!this.dataCallback || !this.config) {
+      return
+    }
+
+    const streamChunk = () => {
+      if (!this.recording) {
+        return
+      }
+
+      // Check if we've reached the maximum chunks
+      if (this.options.maxChunks! > 0 && this.chunksSent >= this.options.maxChunks!) {
+        this.stop()
+        return
+      }
+
+      // Generate or use provided audio data
+      const audioData = this.generateAudioChunk()
+
+      if (audioData) {
+        this.simulateDataChunk(audioData)
+        this.chunksSent += 1
+      }
+
+      // Schedule next chunk if still recording
+      if (this.recording) {
+        this.streamInterval = setTimeout(streamChunk, this.options.chunkInterval!)
+      }
+    }
+
+    // Start streaming after a short delay
+    this.streamInterval = setTimeout(streamChunk, this.options.chunkInterval!)
+  }
+
+  private generateAudioChunk(): Uint8Array | null {
+    // If we have pre-defined audio data, use it
+    if (this.options.audioData) {
+      const startByte = this.chunksSent * this.options.chunkSize!
+      const endByte = Math.min(startByte + this.options.chunkSize!, this.options.audioData.length)
+
+      if (startByte >= this.options.audioData.length) {
+        return null // No more data
+      }
+
+      return this.options.audioData.subarray(startByte, endByte)
+    }
+
+    // Generate silence or simple tone
+    const chunkSize = this.options.chunkSize!
+    const audioData = new Uint8Array(chunkSize)
+
+    if (this.options.generateSilence) {
+      // Generate silence (all zeros)
+      audioData.fill(0)
+    } else {
+      // Generate a simple sine wave tone for testing
+      const sampleRate = this.options.sampleRate!
+      const frequency = 440 // A4 note
+      const samplesPerChunk = chunkSize / 2 // 16-bit samples
+      const timeOffset = (this.chunksSent * samplesPerChunk) / sampleRate
+
+            for (let i = 0; i < samplesPerChunk; i += 1) {
+        const time = timeOffset + i / sampleRate
+        const amplitude = Math.sin(2 * Math.PI * frequency * time) * 0.5
+        const sample = Math.round(amplitude * 32767) // 16-bit signed sample
+
+        // Convert to little-endian bytes
+        audioData[i * 2] = sample % 256
+        audioData[i * 2 + 1] = Math.floor(sample / 256) % 256
+      }
+    }
+
+    return audioData
+  }
+
+  private static delay(ms: number): Promise<void> {
+    return new Promise(resolve => setTimeout(resolve, ms))
+  }
+}