react-native-sherpa-onnx 0.3.0 → 0.3.2

package/src/stt/streaming.ts

@@ -0,0 +1,361 @@
+import SherpaOnnx from '../NativeSherpaOnnx';
+import { resolveModelPath } from '../utils';
+import type {
+  OnlineSTTModelType,
+  StreamingSttEngine,
+  StreamingSttInitOptions,
+  StreamingSttResult,
+  SttStream,
+} from './streamingTypes';
+
+let streamingSttInstanceCounter = 0;
+
+/**
+ * Map detected STT model type (from detectSttModel) to an online (streaming) model type.
+ * Throws if the detected type has no streaming support.
+ */
+export function mapDetectedToOnlineType(
+  detectedType: string | undefined
+): OnlineSTTModelType {
+  const t = detectedType ?? '';
+  switch (t) {
+    case 'transducer':
+      return 'transducer';
+    case 'paraformer':
+      return 'paraformer';
+    case 'nemo_ctc':
+      return 'nemo_ctc';
+    case 'zipformer_ctc':
+    case 'ctc':
+      return 'zipformer2_ctc';
+    case 'tone_ctc':
+      return 'tone_ctc';
+    default:
+      throw new Error(
+        `Model type "${t}" is not supported for streaming STT. Use createSTT() for offline recognition, or pass a supported modelType: transducer, paraformer, zipformer2_ctc, nemo_ctc, tone_ctc.`
+      );
+  }
+}
+
+/**
+ * Returns the online (streaming) model type for a detected STT model type, or null if streaming is not supported.
+ * Use this to check whether the current model can be used with createStreamingSTT() (e.g. for live transcription).
+ */
+export function getOnlineTypeOrNull(
+  detectedType: string | undefined
+): OnlineSTTModelType | null {
+  try {
+    return mapDetectedToOnlineType(detectedType);
+  } catch {
+    return null;
+  }
+}
+let sttStreamCounter = 0;
+
+function normalizeStreamingResult(raw: {
+  text?: string;
+  tokens?: string[] | unknown;
+  timestamps?: number[] | unknown;
+}): StreamingSttResult {
+  return {
+    text: typeof raw.text === 'string' ? raw.text : '',
+    tokens: Array.isArray(raw.tokens) ? (raw.tokens as string[]) : [],
+    timestamps: Array.isArray(raw.timestamps)
+      ? (raw.timestamps as number[])
+      : [],
+  };
+}
+
+/**
+ * Flatten StreamingSttInitOptions to native initializeOnlineStt parameters.
+ * EndpointConfig (rule1, rule2, rule3) is expanded to 9 flat params.
+ */
+function flattenInitOptionsForNative(options: StreamingSttInitOptions): {
+  modelDir: string;
+  modelType: string;
+  enableEndpoint: boolean;
+  decodingMethod: string;
+  maxActivePaths: number;
+  hotwordsFile?: string;
+  hotwordsScore?: number;
+  numThreads?: number;
+  provider?: string;
+  ruleFsts?: string;
+  ruleFars?: string;
+  blankPenalty?: number;
+  debug?: boolean;
+  rule1MustContainNonSilence?: boolean;
+  rule1MinTrailingSilence?: number;
+  rule1MinUtteranceLength?: number;
+  rule2MustContainNonSilence?: boolean;
+  rule2MinTrailingSilence?: number;
+  rule2MinUtteranceLength?: number;
+  rule3MustContainNonSilence?: boolean;
+  rule3MinTrailingSilence?: number;
+  rule3MinUtteranceLength?: number;
+} {
+  const ep = options.endpointConfig;
+  return {
+    modelDir: '', // filled by caller after resolveModelPath
+    modelType: options.modelType,
+    enableEndpoint: options.enableEndpoint ?? true,
+    decodingMethod: options.decodingMethod ?? 'greedy_search',
+    maxActivePaths: options.maxActivePaths ?? 4,
+    hotwordsFile: options.hotwordsFile,
+    hotwordsScore: options.hotwordsScore,
+    numThreads: options.numThreads,
+    provider: options.provider,
+    ruleFsts: options.ruleFsts,
+    ruleFars: options.ruleFars,
+    blankPenalty: options.blankPenalty,
+    debug: options.debug,
+    rule1MustContainNonSilence: ep?.rule1?.mustContainNonSilence,
+    rule1MinTrailingSilence: ep?.rule1?.minTrailingSilence,
+    rule1MinUtteranceLength: ep?.rule1?.minUtteranceLength,
+    rule2MustContainNonSilence: ep?.rule2?.mustContainNonSilence,
+    rule2MinTrailingSilence: ep?.rule2?.minTrailingSilence,
+    rule2MinUtteranceLength: ep?.rule2?.minUtteranceLength,
+    rule3MustContainNonSilence: ep?.rule3?.mustContainNonSilence,
+    rule3MinTrailingSilence: ep?.rule3?.minTrailingSilence,
+    rule3MinUtteranceLength: ep?.rule3?.minUtteranceLength,
+  };
+}
+
+/**
+ * Create a streaming (online) STT engine. Use this for real-time recognition with
+ * partial results and endpoint detection. Call destroy() when done.
+ *
+ * @param options - Streaming STT init options (modelPath required; modelType optional, use 'auto' to detect from directory)
+ * @returns Promise resolving to a StreamingSttEngine
+ * @example
+ * ```typescript
+ * // With explicit model type
+ * const engine = await createStreamingSTT({
+ *   modelPath: { type: 'asset', path: 'models/streaming-zipformer-en' },
+ *   modelType: 'transducer',
+ * });
+ * // With auto-detection
+ * const engine = await createStreamingSTT({
+ *   modelPath: { type: 'asset', path: 'models/sherpa-onnx-streaming-t-one-russian-2025-09-08' },
+ *   modelType: 'auto',
+ * });
+ * const stream = await engine.createStream();
+ * await stream.acceptWaveform(samples, 16000);
+ * if (await stream.isReady()) {
+ *   await stream.decode();
+ *   const result = await stream.getResult();
+ *   console.log(result.text);
+ * }
+ * await stream.release();
+ * await engine.destroy();
+ * ```
+ */
+export async function createStreamingSTT(
+  options: StreamingSttInitOptions
+): Promise<StreamingSttEngine> {
+  const instanceId = `streaming_stt_${++streamingSttInstanceCounter}`;
+  const resolvedPath = await resolveModelPath(options.modelPath);
+
+  let effectiveModelType: OnlineSTTModelType;
+  if (options.modelType === 'auto' || options.modelType === undefined) {
+    const detectResult = await SherpaOnnx.detectSttModel(
+      resolvedPath,
+      undefined,
+      undefined
+    );
+    if (!detectResult.success) {
+      const errMsg =
+        'error' in detectResult &&
+        typeof (detectResult as { error?: string }).error === 'string'
+          ? (detectResult as { error: string }).error
+          : 'Unknown error';
+      throw new Error(
+        `Streaming STT auto-detection failed for ${resolvedPath}. ${errMsg}`
+      );
+    }
+    effectiveModelType = mapDetectedToOnlineType(detectResult.modelType);
+  } else {
+    effectiveModelType = options.modelType;
+  }
+
+  const optionsWithResolvedType = { ...options, modelType: effectiveModelType };
+  const flat = flattenInitOptionsForNative(optionsWithResolvedType);
+  flat.modelDir = resolvedPath;
+
+  // Build options with only defined values (no undefined) to avoid iOS TurboModule marshalling crash when options contain undefined.
+  const nativeOptions: Parameters<
+    typeof SherpaOnnx.initializeOnlineSttWithOptions
+  >[1] = {
+    modelDir: flat.modelDir,
+    modelType: flat.modelType,
+    enableEndpoint: flat.enableEndpoint,
+    decodingMethod: flat.decodingMethod,
+    maxActivePaths: flat.maxActivePaths,
+  };
+  if (flat.hotwordsFile !== undefined)
+    nativeOptions.hotwordsFile = flat.hotwordsFile;
+  if (flat.hotwordsScore !== undefined)
+    nativeOptions.hotwordsScore = flat.hotwordsScore;
+  if (flat.numThreads !== undefined) nativeOptions.numThreads = flat.numThreads;
+  if (flat.provider !== undefined) nativeOptions.provider = flat.provider;
+  if (flat.ruleFsts !== undefined) nativeOptions.ruleFsts = flat.ruleFsts;
+  if (flat.ruleFars !== undefined) nativeOptions.ruleFars = flat.ruleFars;
+  if (flat.blankPenalty !== undefined)
+    nativeOptions.blankPenalty = flat.blankPenalty;
+  if (flat.debug !== undefined) nativeOptions.debug = flat.debug;
+  if (flat.rule1MustContainNonSilence !== undefined)
+    nativeOptions.rule1MustContainNonSilence = flat.rule1MustContainNonSilence;
+  if (flat.rule1MinTrailingSilence !== undefined)
+    nativeOptions.rule1MinTrailingSilence = flat.rule1MinTrailingSilence;
+  if (flat.rule1MinUtteranceLength !== undefined)
+    nativeOptions.rule1MinUtteranceLength = flat.rule1MinUtteranceLength;
+  if (flat.rule2MustContainNonSilence !== undefined)
+    nativeOptions.rule2MustContainNonSilence = flat.rule2MustContainNonSilence;
+  if (flat.rule2MinTrailingSilence !== undefined)
+    nativeOptions.rule2MinTrailingSilence = flat.rule2MinTrailingSilence;
+  if (flat.rule2MinUtteranceLength !== undefined)
+    nativeOptions.rule2MinUtteranceLength = flat.rule2MinUtteranceLength;
+  if (flat.rule3MustContainNonSilence !== undefined)
+    nativeOptions.rule3MustContainNonSilence = flat.rule3MustContainNonSilence;
+  if (flat.rule3MinTrailingSilence !== undefined)
+    nativeOptions.rule3MinTrailingSilence = flat.rule3MinTrailingSilence;
+  if (flat.rule3MinUtteranceLength !== undefined)
+    nativeOptions.rule3MinUtteranceLength = flat.rule3MinUtteranceLength;
+
+  const result = await SherpaOnnx.initializeOnlineSttWithOptions(
+    instanceId,
+    nativeOptions
+  );
+
+  if (!result.success) {
+    throw new Error(`Streaming STT initialization failed for ${instanceId}`);
+  }
+
+  const enableInputNormalization = options.enableInputNormalization !== false;
+  let destroyed = false;
+
+  const guard = () => {
+    if (destroyed) {
+      throw new Error(
+        `Streaming STT engine ${instanceId} has been destroyed; cannot call methods on it.`
+      );
+    }
+  };
+
+  const engine: StreamingSttEngine = {
+    get instanceId() {
+      return instanceId;
+    },
+
+    async createStream(hotwords?: string): Promise<SttStream> {
+      guard();
+      const streamId = `stt_stream_${++sttStreamCounter}`;
+      await SherpaOnnx.createSttStream(instanceId, streamId, hotwords);
+
+      let released = false;
+      const streamGuard = () => {
+        if (destroyed) {
+          throw new Error(
+            `Streaming STT engine ${instanceId} has been destroyed.`
+          );
+        }
+        if (released) {
+          throw new Error(
+            `Stream ${streamId} has been released; cannot call methods on it.`
+          );
+        }
+      };
+
+      const stream: SttStream = {
+        get streamId() {
+          return streamId;
+        },
+
+        async acceptWaveform(
+          samples: number[],
+          sampleRate: number
+        ): Promise<void> {
+          streamGuard();
+          await SherpaOnnx.acceptSttWaveform(streamId, samples, sampleRate);
+        },
+
+        async inputFinished(): Promise<void> {
+          streamGuard();
+          await SherpaOnnx.sttStreamInputFinished(streamId);
+        },
+
+        async decode(): Promise<void> {
+          streamGuard();
+          await SherpaOnnx.decodeSttStream(streamId);
+        },
+
+        async isReady(): Promise<boolean> {
+          streamGuard();
+          return SherpaOnnx.isSttStreamReady(streamId);
+        },
+
+        async getResult(): Promise<StreamingSttResult> {
+          streamGuard();
+          const raw = await SherpaOnnx.getSttStreamResult(streamId);
+          return normalizeStreamingResult(raw);
+        },
+
+        async isEndpoint(): Promise<boolean> {
+          streamGuard();
+          return SherpaOnnx.isSttStreamEndpoint(streamId);
+        },
+
+        async reset(): Promise<void> {
+          streamGuard();
+          await SherpaOnnx.resetSttStream(streamId);
+        },
+
+        async release(): Promise<void> {
+          if (released) return;
+          released = true;
+          await SherpaOnnx.releaseSttStream(streamId);
+        },
+
+        async processAudioChunk(
+          samples: number[],
+          sampleRate: number
+        ): Promise<{ result: StreamingSttResult; isEndpoint: boolean }> {
+          streamGuard();
+          let toSend: number[] = samples;
+          if (enableInputNormalization && samples.length > 0) {
+            let maxAbs = 1e-10;
+            for (let i = 0; i < samples.length; i++) {
+              const abs = Math.abs(samples[i]!);
+              if (abs > maxAbs) maxAbs = abs;
+            }
+            const scale = maxAbs < 0.01 ? 80 : Math.min(80, 0.8 / maxAbs);
+            toSend = new Array(samples.length);
+            for (let i = 0; i < samples.length; i++) {
+              const v = samples[i]! * scale;
+              toSend[i] = v < -1 ? -1 : v > 1 ? 1 : v;
+            }
+          }
+          const raw = await SherpaOnnx.processSttAudioChunk(
+            streamId,
+            toSend,
+            sampleRate
+          );
+          return {
+            result: normalizeStreamingResult(raw),
+            isEndpoint: Boolean(raw.isEndpoint),
+          };
+        },
+      };
+
+      return stream;
+    },
+
+    async destroy(): Promise<void> {
+      if (destroyed) return;
+      destroyed = true;
+      await SherpaOnnx.unloadOnlineStt(instanceId);
+    },
+  };
+
+  return engine;
+}

package/src/stt/streamingTypes.ts

@@ -0,0 +1,151 @@
+import type { ModelPathConfig } from '../types';
+
+/**
+ * Online (streaming) STT model types.
+ * These models use OnlineRecognizer + OnlineStream in sherpa-onnx.
+ * Must match the native OnlineRecognizer model config (transducer, paraformer, zipformer2_ctc, nemo_ctc, tone_ctc).
+ */
+export type OnlineSTTModelType =
+  | 'transducer'
+  | 'paraformer'
+  | 'zipformer2_ctc'
+  | 'nemo_ctc'
+  | 'tone_ctc';
+
+/** Runtime list of supported online STT model types. */
+export const ONLINE_STT_MODEL_TYPES: readonly OnlineSTTModelType[] = [
+  'transducer',
+  'paraformer',
+  'zipformer2_ctc',
+  'nemo_ctc',
+  'tone_ctc',
+] as const;
+
+/**
+ * Single endpoint rule (Kotlin EndpointRule).
+ * Used to detect end of utterance in streaming recognition.
+ */
+export interface EndpointRule {
+  /** If true, rule only matches when the segment contains non-silence. */
+  mustContainNonSilence: boolean;
+  /** Minimum trailing silence in seconds. */
+  minTrailingSilence: number;
+  /** Minimum utterance length in seconds (e.g. max length cap). */
+  minUtteranceLength: number;
+}
+
+/**
+ * Endpoint detection config (Kotlin EndpointConfig).
+ * Three rules; first match determines end of utterance.
+ */
+export interface EndpointConfig {
+  /** Rule 1: e.g. 2.4s trailing silence, no speech required. */
+  rule1?: EndpointRule;
+  /** Rule 2: e.g. 1.4s trailing silence, speech required. */
+  rule2?: EndpointRule;
+  /** Rule 3: e.g. max utterance length 20s. */
+  rule3?: EndpointRule;
+}
+
+/**
+ * Options for initializing the streaming (online) STT engine.
+ */
+export interface StreamingSttInitOptions {
+  /** Model path configuration (asset, file, or auto). */
+  modelPath: ModelPathConfig;
+  /** Online model type. Use 'auto' to detect from model directory (calls detectSttModel and maps to an online type). */
+  modelType: OnlineSTTModelType | 'auto';
+  /** Enable endpoint detection. Default: true. */
+  enableEndpoint?: boolean;
+  /** Endpoint rules. Defaults match Kotlin (rule1: 2.4s silence, rule2: 1.4s + speech, rule3: 20s max). */
+  endpointConfig?: EndpointConfig;
+  /** Decoding method. Default: "greedy_search". */
+  decodingMethod?: 'greedy_search' | 'modified_beam_search';
+  /** Max active paths for beam search. Default: 4. */
+  maxActivePaths?: number;
+  /** Path to hotwords file (transducer/nemo_transducer). */
+  hotwordsFile?: string;
+  /** Hotwords score. Default: 1.5. */
+  hotwordsScore?: number;
+  /** Number of threads for inference. Default: 1. */
+  numThreads?: number;
+  /** Execution provider (e.g. "cpu"). */
+  provider?: string;
+  /** Path(s) to rule FSTs for ITN. */
+  ruleFsts?: string;
+  /** Path(s) to rule FARs for ITN. */
+  ruleFars?: string;
+  /** Blank penalty. */
+  blankPenalty?: number;
+  /** Enable debug logging. Default: false. */
+  debug?: boolean;
+  /**
+   * Enable adaptive input normalization for audio chunks in processAudioChunk().
+   * When true (default), input is scaled so the peak is ~0.8 to handle varying device levels (e.g. quiet mics on iOS).
+   * Set to false if your audio is already in the expected range [-1, 1] and you want to pass it through unchanged.
+   */
+  enableInputNormalization?: boolean;
+}
+
+/**
+ * Partial or final recognition result from streaming STT (maps to Kotlin OnlineRecognizerResult).
+ */
+export interface StreamingSttResult {
+  text: string;
+  tokens: string[];
+  timestamps: number[];
+}
+
+/**
+ * Streaming STT stream. Created by StreamingSttEngine.createStream().
+ * Feeds audio via acceptWaveform, then decode / getResult / isEndpoint.
+ */
+export interface SttStream {
+  readonly streamId: string;
+
+  /** Feed PCM samples (float in [-1, 1]) to the stream. */
+  acceptWaveform(samples: number[], sampleRate: number): Promise<void>;
+
+  /** Signal that no more audio will be fed. */
+  inputFinished(): Promise<void>;
+
+  /** Run decoding on accumulated audio (call when isReady() is true). */
+  decode(): Promise<void>;
+
+  /** True if there is enough audio to decode. */
+  isReady(): Promise<boolean>;
+
+  /** Get current partial or final result. Call after decode(). */
+  getResult(): Promise<StreamingSttResult>;
+
+  /** True if endpoint (end of utterance) was detected. */
+  isEndpoint(): Promise<boolean>;
+
+  /** Reset stream state for reuse. */
+  reset(): Promise<void>;
+
+  /** Release native stream; do not use after this. */
+  release(): Promise<void>;
+
+  /**
+   * Convenience: feed audio, auto-decode while ready, return result and endpoint status.
+   * Reduces bridge round-trips from 5 to 1 per chunk.
+   */
+  processAudioChunk(
+    samples: number[],
+    sampleRate: number
+  ): Promise<{ result: StreamingSttResult; isEndpoint: boolean }>;
+}
+
+/**
+ * Streaming STT engine (OnlineRecognizer). Create via createStreamingSTT().
+ */
+export interface StreamingSttEngine {
+  readonly instanceId: string;
+
+  /** Create a new stream for this recognizer. Optional hotwords string. */
+  createStream(hotwords?: string): Promise<SttStream>;
+
+  /** Release native recognizer and all streams. */
+  destroy(): Promise<void>;
+}

package/src/tts/index.ts

@@ -1,4 +1,3 @@
-import { DeviceEventEmitter } from 'react-native';
 import SherpaOnnx from '../NativeSherpaOnnx';
 import type {
   TTSInitializeOptions,
@@ -10,10 +9,6 @@ import type {
   GeneratedAudioWithTimestamps,
   TTSModelInfo,
   TtsEngine,
-  TtsStreamChunk,
-  TtsStreamEnd,
-  TtsStreamError,
-  TtsStreamHandlers,
 } from './types';
 import type { ModelPathConfig } from '../types';
 import { resolveModelPath } from '../utils';
@@ -135,7 +130,7 @@ function toNativeTtsOptions(
 }
 
 // TTS stream events are sent from native via sendEventWithName; use DeviceEventEmitter
-// so we don't need NativeEventEmitter (which expects addListener/removeListeners on the module).
+
 /**
  * Create a TTS engine instance. Call destroy() on the returned engine when done to free native resources.
  *
@@ -260,66 +255,6 @@ export async function createTTS(
       );
     },
 
-    async generateSpeechStream(
-      text: string,
-      opts: TtsGenerationOptions | undefined,
-      handlers: TtsStreamHandlers
-    ): Promise<() => void> {
-      guard();
-      const subscriptions = [
-        DeviceEventEmitter.addListener('ttsStreamChunk', (event: unknown) => {
-          const e = event as TtsStreamChunk;
-          if (e.instanceId != null && e.instanceId !== instanceId) return;
-          handlers.onChunk?.(e);
-        }),
-        DeviceEventEmitter.addListener('ttsStreamEnd', (event: unknown) => {
-          const e = event as TtsStreamEnd;
-          if (e.instanceId != null && e.instanceId !== instanceId) return;
-          handlers.onEnd?.(e);
-        }),
-        DeviceEventEmitter.addListener('ttsStreamError', (event: unknown) => {
-          const e = event as TtsStreamError;
-          if (e.instanceId != null && e.instanceId !== instanceId) return;
-          handlers.onError?.(e);
-        }),
-      ];
-
-      try {
-        await SherpaOnnx.generateTtsStream(
-          instanceId,
-          text,
-          toNativeTtsOptions(opts)
-        );
-      } catch (error) {
-        subscriptions.forEach((sub) => sub.remove());
-        throw error;
-      }
-
-      return () => {
-        subscriptions.forEach((sub) => sub.remove());
-      };
-    },
-
-    async cancelSpeechStream(): Promise<void> {
-      guard();
-      return SherpaOnnx.cancelTtsStream(instanceId);
-    },
-
-    async startPcmPlayer(sampleRate: number, channels: number): Promise<void> {
-      guard();
-      return SherpaOnnx.startTtsPcmPlayer(instanceId, sampleRate, channels);
-    },
-
-    async writePcmChunk(samples: number[]): Promise<void> {
-      guard();
-      return SherpaOnnx.writeTtsPcmChunk(instanceId, samples);
-    },
-
-    async stopPcmPlayer(): Promise<void> {
-      guard();
-      return SherpaOnnx.stopTtsPcmPlayer(instanceId);
-    },
-
     async updateParams(opts: TtsUpdateOptions): Promise<{
       success: boolean;
       detectedModels: Array<{ type: string; modelDir: string }>;
@@ -445,6 +380,10 @@ export function shareAudioFile(
   return SherpaOnnx.shareTtsAudio(fileUri, mimeType);
 }
 
+// Streaming TTS (separate engine; use createStreamingTTS for chunk callbacks and PCM playback)
+export { createStreamingTTS } from './streaming';
+export type { StreamingTtsEngine } from './streamingTypes';
+
 // Export types and runtime type list
 export type {
   TTSInitializeOptions,
@@ -462,6 +401,7 @@ export type {
   TtsSubtitleItem,
   TTSModelInfo,
   TtsEngine,
+  TtsStreamController,
   TtsStreamHandlers,
   TtsStreamChunk,
   TtsStreamEnd,