react-native-sherpa-onnx 0.3.0 → 0.3.2

package/src/tts/streaming.ts

@@ -0,0 +1,336 @@
+import { DeviceEventEmitter } from 'react-native';
+import SherpaOnnx from '../NativeSherpaOnnx';
+import type {
+  TTSInitializeOptions,
+  TTSModelType,
+  TtsModelOptions,
+  TtsGenerationOptions,
+  TtsStreamChunk,
+  TtsStreamEnd,
+  TtsStreamError,
+  TtsStreamHandlers,
+  TtsStreamController,
+  TTSModelInfo,
+} from './types';
+import type { StreamingTtsEngine } from './streamingTypes';
+import type { ModelPathConfig } from '../types';
+import { resolveModelPath } from '../utils';
+
+let streamingTtsInstanceCounter = 0;
+let ttsRequestIdCounter = 0;
+
+/**
+ * Flatten model-specific options for the given model type to native init params.
+ */
+function flattenTtsModelOptionsForNative(
+  modelType: TTSModelType | undefined,
+  modelOptions: TtsModelOptions | undefined
+): {
+  noiseScale: number | undefined;
+  noiseScaleW: number | undefined;
+  lengthScale: number | undefined;
+} {
+  if (
+    !modelOptions ||
+    !modelType ||
+    modelType === 'auto' ||
+    modelType === 'zipvoice'
+  )
+    return {
+      noiseScale: undefined,
+      noiseScaleW: undefined,
+      lengthScale: undefined,
+    };
+  const block =
+    modelType === 'vits'
+      ? modelOptions.vits
+      : modelType === 'matcha'
+      ? modelOptions.matcha
+      : modelType === 'kokoro'
+      ? modelOptions.kokoro
+      : modelType === 'kitten'
+      ? modelOptions.kitten
+      : modelType === 'pocket'
+      ? modelOptions.pocket
+      : undefined;
+  if (!block)
+    return {
+      noiseScale: undefined,
+      noiseScaleW: undefined,
+      lengthScale: undefined,
+    };
+  const n = block as {
+    noiseScale?: number;
+    noiseScaleW?: number;
+    lengthScale?: number;
+  };
+  return {
+    noiseScale:
+      n.noiseScale !== undefined && typeof n.noiseScale === 'number'
+        ? n.noiseScale
+        : undefined,
+    noiseScaleW:
+      n.noiseScaleW !== undefined && typeof n.noiseScaleW === 'number'
+        ? n.noiseScaleW
+        : undefined,
+    lengthScale:
+      n.lengthScale !== undefined && typeof n.lengthScale === 'number'
+        ? n.lengthScale
+        : undefined,
+  };
+}
+
+function toNativeTtsOptions(
+  options?: TtsGenerationOptions
+): Record<string, unknown> {
+  if (options == null) return {};
+  const out: Record<string, unknown> = {};
+  if (options.sid !== undefined) out.sid = options.sid;
+  if (options.speed !== undefined) out.speed = options.speed;
+  if (options.silenceScale !== undefined)
+    out.silenceScale = options.silenceScale;
+  if (options.referenceAudio != null) {
+    out.referenceAudio = options.referenceAudio.samples;
+    out.referenceSampleRate = options.referenceAudio.sampleRate;
+  }
+  if (options.referenceText !== undefined)
+    out.referenceText = options.referenceText;
+  if (options.numSteps !== undefined) out.numSteps = options.numSteps;
+  if (options.extra != null && Object.keys(options.extra).length > 0)
+    out.extra = options.extra;
+  return out;
+}
+
+/**
+ * Create a streaming TTS engine instance. Use for incremental generation with
+ * chunk callbacks and PCM playback. Call destroy() when done.
+ *
+ * @param options - TTS initialization options or model path configuration
+ * @returns Promise resolving to a StreamingTtsEngine instance
+ * @example
+ * ```typescript
+ * const tts = await createStreamingTTS({
+ *   modelPath: { type: 'asset', path: 'models/vits-piper-en' },
+ *   modelType: 'vits',
+ * });
+ * const controller = await tts.generateSpeechStream('Hello', undefined, {
+ *   onChunk: (chunk) => playPcm(chunk.samples, chunk.sampleRate),
+ *   onEnd: () => {},
+ * });
+ * await tts.destroy();
+ * ```
+ */
+export async function createStreamingTTS(
+  options: TTSInitializeOptions | ModelPathConfig
+): Promise<StreamingTtsEngine> {
+  const instanceId = `streaming_tts_${++streamingTtsInstanceCounter}`;
+
+  let modelPath: ModelPathConfig;
+  let modelType: TTSModelType | undefined;
+  let provider: string | undefined;
+  let numThreads: number | undefined;
+  let debug: boolean | undefined;
+  let modelOptions: TtsModelOptions | undefined;
+  let ruleFsts: string | undefined;
+  let ruleFars: string | undefined;
+  let maxNumSentences: number | undefined;
+  let silenceScale: number | undefined;
+
+  if ('modelPath' in options) {
+    modelPath = options.modelPath;
+    modelType = options.modelType;
+    provider = options.provider;
+    numThreads = options.numThreads;
+    debug = options.debug;
+    modelOptions = options.modelOptions;
+    ruleFsts = options.ruleFsts;
+    ruleFars = options.ruleFars;
+    maxNumSentences = options.maxNumSentences;
+    silenceScale = options.silenceScale;
+  } else {
+    modelPath = options;
+    modelType = undefined;
+    provider = undefined;
+    numThreads = undefined;
+    debug = undefined;
+    modelOptions = undefined;
+    ruleFsts = undefined;
+    ruleFars = undefined;
+    maxNumSentences = undefined;
+    silenceScale = undefined;
+  }
+
+  const flat = flattenTtsModelOptionsForNative(modelType, modelOptions);
+  const resolvedPath = await resolveModelPath(modelPath);
+
+  const result = await SherpaOnnx.initializeTts(
+    instanceId,
+    resolvedPath,
+    modelType ?? 'auto',
+    numThreads ?? 2,
+    debug ?? false,
+    flat.noiseScale,
+    flat.noiseScaleW,
+    flat.lengthScale,
+    ruleFsts,
+    ruleFars,
+    maxNumSentences,
+    silenceScale,
+    provider
+  );
+
+  if (!result.success) {
+    throw new Error(
+      `TTS initialization failed: ${JSON.stringify(
+        result.detectedModels ?? []
+      )}`
+    );
+  }
+
+  let destroyed = false;
+
+  const guard = () => {
+    if (destroyed) {
+      throw new Error(
+        `Streaming TTS instance ${instanceId} has been destroyed; cannot call methods on it.`
+      );
+    }
+  };
+
+  const engine: StreamingTtsEngine = {
+    get instanceId() {
+      return instanceId;
+    },
+
+    async generateSpeechStream(
+      text: string,
+      opts: TtsGenerationOptions | undefined,
+      handlers: TtsStreamHandlers
+    ): Promise<TtsStreamController> {
+      guard();
+      const requestId = `tts_req_${++ttsRequestIdCounter}`;
+      const subscriptions: Array<{ remove: () => void }> = [];
+      let unsubscribed = false;
+
+      const unsubscribe = () => {
+        if (unsubscribed) return;
+        unsubscribed = true;
+        subscriptions.forEach((sub) => sub.remove());
+      };
+
+      const matchesRequest = (e: { instanceId?: string; requestId?: string }) =>
+        (e.instanceId == null || e.instanceId === instanceId) &&
+        (e.requestId == null || e.requestId === requestId);
+
+      subscriptions.push(
+        DeviceEventEmitter.addListener('ttsStreamChunk', (event: unknown) => {
+          const e = event as TtsStreamChunk;
+          if (!matchesRequest(e)) {
+            return;
+          }
+          handlers.onChunk?.(e);
+        }),
+        DeviceEventEmitter.addListener('ttsStreamEnd', (event: unknown) => {
+          const e = event as TtsStreamEnd;
+          if (!matchesRequest(e)) {
+            return;
+          }
+          try {
+            handlers.onEnd?.(e);
+          } finally {
+            unsubscribe();
+          }
+        }),
+        DeviceEventEmitter.addListener('ttsStreamError', (event: unknown) => {
+          const e = event as TtsStreamError;
+          if (!matchesRequest(e)) {
+            return;
+          }
+          try {
+            handlers.onError?.(e);
+          } finally {
+            unsubscribe();
+          }
+        })
+      );
+
+      // Yield so the bridge can register listeners before native emits (avoids "no listeners" / "already in progress")
+      await new Promise<void>((resolve) => {
+        if (typeof setImmediate === 'function') {
+          setImmediate(resolve);
+        } else {
+          setTimeout(resolve, 0);
+        }
+      });
+
+      try {
+        await SherpaOnnx.generateTtsStream(
+          instanceId,
+          requestId,
+          text,
+          toNativeTtsOptions(opts)
+        );
+      } catch (error) {
+        unsubscribe();
+        throw error;
+      }
+
+      const controller: TtsStreamController = {
+        async cancel(): Promise<void> {
+          guard();
+          await SherpaOnnx.cancelTtsStream(instanceId);
+          unsubscribe();
+        },
+        unsubscribe,
+      };
+      return controller;
+    },
+
+    async cancelSpeechStream(): Promise<void> {
+      guard();
+      return SherpaOnnx.cancelTtsStream(instanceId);
+    },
+
+    async startPcmPlayer(sampleRate: number, channels: number): Promise<void> {
+      guard();
+      return SherpaOnnx.startTtsPcmPlayer(instanceId, sampleRate, channels);
+    },
+
+    async writePcmChunk(samples: number[]): Promise<void> {
+      guard();
+      return SherpaOnnx.writeTtsPcmChunk(instanceId, samples);
+    },
+
+    async stopPcmPlayer(): Promise<void> {
+      guard();
+      return SherpaOnnx.stopTtsPcmPlayer(instanceId);
+    },
+
+    async getModelInfo(): Promise<TTSModelInfo> {
+      guard();
+      const [sampleRate, numSpeakers] = await Promise.all([
+        SherpaOnnx.getTtsSampleRate(instanceId),
+        SherpaOnnx.getTtsNumSpeakers(instanceId),
+      ]);
+      return { sampleRate, numSpeakers };
+    },
+
+    async getSampleRate(): Promise<number> {
+      guard();
+      return SherpaOnnx.getTtsSampleRate(instanceId);
+    },
+
+    async getNumSpeakers(): Promise<number> {
+      guard();
+      return SherpaOnnx.getTtsNumSpeakers(instanceId);
+    },
+
+    async destroy(): Promise<void> {
+      if (destroyed) return;
+      destroyed = true;
+      await SherpaOnnx.unloadTts(instanceId);
+    },
+  };
+
+  return engine;
+}

package/src/tts/streamingTypes.ts

@@ -0,0 +1,54 @@
+import type {
+  TtsStreamHandlers,
+  TtsStreamController,
+  TtsGenerationOptions,
+  TTSModelInfo,
+} from './types';
+
+// Re-export streaming event types for consumers who import from streamingTypes
+export type {
+  TtsStreamChunk,
+  TtsStreamEnd,
+  TtsStreamError,
+  TtsStreamHandlers,
+  TtsStreamController,
+  TtsGenerationOptions,
+  TTSModelInfo,
+} from './types';
+
+/**
+ * Streaming-only TTS engine returned by createStreamingTTS().
+ * Use for incremental generation with chunk callbacks and PCM playback.
+ * Call destroy() when done to free native resources.
+ */
+export interface StreamingTtsEngine {
+  readonly instanceId: string;
+
+  /** Generate speech in streaming mode; audio delivered via handlers. */
+  generateSpeechStream(
+    text: string,
+    options: TtsGenerationOptions | undefined,
+    handlers: TtsStreamHandlers
+  ): Promise<TtsStreamController>;
+
+  /** Cancel the current streaming generation. */
+  cancelSpeechStream(): Promise<void>;
+
+  /** Start built-in PCM playback (e.g. for play-while-generating). */
+  startPcmPlayer(sampleRate: number, channels: number): Promise<void>;
+
+  /** Write float PCM samples to the player. Use from onChunk. */
+  writePcmChunk(samples: number[]): Promise<void>;
+
+  /** Stop and release the PCM player. */
+  stopPcmPlayer(): Promise<void>;
+
+  /** Model sample rate and number of speakers. */
+  getModelInfo(): Promise<TTSModelInfo>;
+
+  getSampleRate(): Promise<number>;
+  getNumSpeakers(): Promise<number>;
+
+  /** Release native TTS resources. Do not use the engine after this. */
+  destroy(): Promise<void>;
+}

package/src/tts/types.ts

@@ -289,6 +289,8 @@ export interface GeneratedAudioWithTimestamps extends GeneratedAudio {
 export interface TtsStreamChunk {
   /** Instance ID (set by native for multi-instance routing). */
   instanceId?: string;
+  /** Request ID for this generation (distinguishes concurrent streams on same instance). */
+  requestId?: string;
   samples: number[];
   sampleRate: number;
   progress: number;
@@ -301,6 +303,8 @@ export interface TtsStreamChunk {
 export interface TtsStreamEnd {
   /** Instance ID (set by native for multi-instance routing). */
   instanceId?: string;
+  /** Request ID for this generation. */
+  requestId?: string;
   cancelled: boolean;
 }
 
@@ -310,9 +314,22 @@ export interface TtsStreamEnd {
 export interface TtsStreamError {
   /** Instance ID (set by native for multi-instance routing). */
   instanceId?: string;
+  /** Request ID for this generation. */
+  requestId?: string;
   message: string;
 }
 
+/**
+ * Controller returned by generateSpeechStream().
+ * Use cancel() to stop generation, unsubscribe() to remove event listeners.
+ */
+export interface TtsStreamController {
+  /** Cancel the ongoing TTS generation. */
+  cancel(): Promise<void>;
+  /** Remove event listeners (called automatically on end/error, or manually). */
+  unsubscribe(): void;
+}
+
 /**
  * Handlers for TTS streaming generation (chunk, end, error).
  */
@@ -323,7 +340,9 @@ export interface TtsStreamHandlers {
 }
 
 /**
- * Instance-based TTS engine returned by createTTS().
+ * Instance-based batch TTS engine returned by createTTS().
+ * Use for one-shot synthesis (generateSpeech, generateSpeechWithTimestamps).
+ * For streaming, use createStreamingTTS() and StreamingTtsEngine instead.
  * Call destroy() when done to free native resources.
  */
 export interface TtsEngine {
@@ -336,15 +355,6 @@ export interface TtsEngine {
     text: string,
     options?: TtsGenerationOptions
   ): Promise<GeneratedAudioWithTimestamps>;
-  generateSpeechStream(
-    text: string,
-    options: TtsGenerationOptions | undefined,
-    handlers: TtsStreamHandlers
-  ): Promise<() => void>;
-  cancelSpeechStream(): Promise<void>;
-  startPcmPlayer(sampleRate: number, channels: number): Promise<void>;
-  writePcmChunk(samples: number[]): Promise<void>;
-  stopPcmPlayer(): Promise<void>;
   updateParams(options: TtsUpdateOptions): Promise<{
     success: boolean;
     detectedModels: Array<{ type: string; modelDir: string }>;

package/android/codegen.gradle

@@ -1,57 +0,0 @@
-// With includesGeneratedCode=false, RNGP does not run codegen for this library when it
-// is built as a dependency (e.g. example app). Run codegen whenever this module is built
-// (standalone AAR or as part of an app) so that NativeSherpaOnnxSpec exists for Kotlin compile.
-// Uses Exec + node (no Node Gradle plugin) to avoid classloader/DSL issues in applied script.
-def libraryRoot = project.projectDir.parentFile
-def outDir = file("${project.buildDir}/generated/source/codegen")
-def codegenJavaDir = file("${project.buildDir}/generated/source/codegen/java")
-
-codegenJavaDir.mkdirs()
-
-def codegenScript = file("${libraryRoot}/node_modules/react-native/scripts/generate-codegen-artifacts.js")
-def outDirPath = outDir.absolutePath.replace("\\", "/")
-def libraryRootPath = libraryRoot.absolutePath.replace("\\", "/")
-
-tasks.register("generateCodegenSpecNode", Exec) {
-  onlyIf { !project.hasProperty('useNpx') || project.property('useNpx') != 'true' }
-  workingDir = libraryRoot
-  inputs.dir(file("${libraryRoot}/src"))
-  inputs.file(file("${libraryRoot}/package.json"))
-  outputs.dir(codegenJavaDir)
-
-  doFirst {
-    if (!file("${libraryRoot}/node_modules/react-native").exists()) {
-      throw new RuntimeException(
-        "Codegen requires node_modules at library root. Run 'yarn install' (or npm install) in ${libraryRoot}, then rebuild."
-      )
-    }
-    outDir.mkdirs()
-  }
-
-  commandLine(
-    'node',
-    codegenScript.absolutePath,
-    '-p', libraryRootPath,
-    '-t', 'android',
-    '-o', outDirPath,
-    '-s', 'library'
-  )
-  environment 'CI': 'true'
-
-  doLast {
-    def nestedJava = file("${outDir}/android/app/build/generated/source/codegen/java")
-    if (nestedJava.exists()) {
-      project.copy {
-        from nestedJava
-        into codegenJavaDir
-      }
-      println "[codegen] Normalised spec output -> ${codegenJavaDir}"
-    } else {
-      println "[codegen] WARNING: expected nested output at ${nestedJava} not found"
-    }
-  }
-}
-
-tasks.register('generateCodegenSpec') {
-  dependsOn tasks.named('generateCodegenSpecNode')
-}