react-native-sherpa-onnx 0.2.0 → 0.3.0

package/src/stt/index.ts

@@ -1,89 +1,249 @@
-import SherpaOnnx from '../NativeSherpaOnnx';
-import type { STTInitializeOptions } from './types';
-import type { InitializeOptions } from '../types';
-import { resolveModelPath } from '../utils';
-
-/**
- * Initialize Speech-to-Text (STT) with model directory.
- *
- * Supports multiple model source types:
- * - Asset models (bundled in app)
- * - File system models (downloaded or user-provided)
- * - Auto-detection (tries asset first, then file system)
- *
- * @param options - STT initialization options or model path configuration
- * @returns Object with success status and array of detected models (each with type and modelDir)
- * @example
- * ```typescript
- * // Simple string (auto-detect)
- * const result = await initializeSTT('models/sherpa-onnx-model');
- * console.log('Detected models:', result.detectedModels);
- * // result.detectedModels = [{ type: 'transducer', modelDir: '/path/to/model' }]
- *
- * // Asset model
- * const result = await initializeSTT({
- *   modelPath: { type: 'asset', path: 'models/sherpa-onnx-model' }
- * });
- *
- * // File system model with preferInt8 option
- * const result = await initializeSTT({
- *   modelPath: { type: 'file', path: '/path/to/model' },
- *   preferInt8: true  // Prefer quantized int8 models (smaller, faster)
- * });
- *
- * // With explicit model type
- * const result = await initializeSTT({
- *   modelPath: { type: 'asset', path: 'models/sherpa-onnx-nemo-parakeet-tdt-ctc-en' },
- *   modelType: 'nemo_ctc'
- * });
- * ```
- */
-export async function initializeSTT(
-  options: STTInitializeOptions | InitializeOptions['modelPath']
-): Promise<{
-  success: boolean;
-  detectedModels: Array<{ type: string; modelDir: string }>;
-}> {
-  // Handle both object syntax and direct path syntax
-  let modelPath: InitializeOptions['modelPath'];
-  let preferInt8: boolean | undefined;
-  let modelType: string | undefined;
-
-  if (typeof options === 'object' && 'modelPath' in options) {
-    modelPath = options.modelPath;
-    preferInt8 = options.preferInt8;
-    modelType = options.modelType;
-  } else {
-    modelPath = options as InitializeOptions['modelPath'];
-    preferInt8 = undefined;
-    modelType = undefined;
-  }
-
-  const resolvedPath = await resolveModelPath(modelPath);
-  return SherpaOnnx.initializeSherpaOnnx(resolvedPath, preferInt8, modelType);
-}
-
-/**
- * Transcribe an audio file.
- *
- * @param filePath - Path to WAV file (16kHz, mono, 16-bit PCM)
- * @returns Promise resolving to transcribed text
- * @example
- * ```typescript
- * const transcription = await transcribeFile('path/to/audio.wav');
- * console.log('Transcription:', transcription);
- * ```
- */
-export function transcribeFile(filePath: string): Promise<string> {
-  return SherpaOnnx.transcribeFile(filePath);
-}
-
-/**
- * Release STT resources.
- */
-export function unloadSTT(): Promise<void> {
-  return SherpaOnnx.unloadSherpaOnnx();
-}
-
-// Export types
-export type { STTInitializeOptions, TranscriptionResult } from './types';
+import SherpaOnnx from '../NativeSherpaOnnx';
+import type {
+  STTInitializeOptions,
+  STTModelType,
+  SttEngine,
+  SttModelOptions,
+  SttRecognitionResult,
+  SttRuntimeConfig,
+} from './types';
+import type { ModelPathConfig } from '../types';
+import { resolveModelPath } from '../utils';
+
+let sttInstanceCounter = 0;
+
+function normalizeSttResult(raw: {
+  text?: string;
+  tokens?: string[] | unknown;
+  timestamps?: number[] | unknown;
+  lang?: string;
+  emotion?: string;
+  event?: string;
+  durations?: number[] | unknown;
+}): SttRecognitionResult {
+  return {
+    text: typeof raw.text === 'string' ? raw.text : '',
+    tokens: Array.isArray(raw.tokens) ? (raw.tokens as string[]) : [],
+    timestamps: Array.isArray(raw.timestamps)
+      ? (raw.timestamps as number[])
+      : [],
+    lang: typeof raw.lang === 'string' ? raw.lang : '',
+    emotion: typeof raw.emotion === 'string' ? raw.emotion : '',
+    event: typeof raw.event === 'string' ? raw.event : '',
+    durations: Array.isArray(raw.durations) ? (raw.durations as number[]) : [],
+  };
+}
+
+/**
+ * Detect STT model type and structure without initializing the recognizer.
+ * Uses the same native file-based detection as createSTT. Stateless; no instance required.
+ *
+ * @param modelPath - Model path configuration (asset, file, or auto)
+ * @param options - Optional preferInt8 and modelType (default: auto)
+ * @returns Object with success, detectedModels (array of { type, modelDir }), and modelType (primary detected type)
+ * @example
+ * ```typescript
+ * const path = { type: 'asset' as const, path: 'models/sherpa-onnx-whisper-tiny-en' };
+ * const result = await detectSttModel(path);
+ * if (result.success && result.detectedModels.length > 0) {
+ *   console.log('Detected type:', result.modelType, result.detectedModels);
+ * }
+ * ```
+ */
+export async function detectSttModel(
+  modelPath: ModelPathConfig,
+  options?: { preferInt8?: boolean; modelType?: STTModelType }
+): Promise<{
+  success: boolean;
+  detectedModels: Array<{ type: string; modelDir: string }>;
+  modelType?: string;
+}> {
+  const resolvedPath = await resolveModelPath(modelPath);
+  return SherpaOnnx.detectSttModel(
+    resolvedPath,
+    options?.preferInt8,
+    options?.modelType
+  );
+}
+
+/**
+ * Create an STT engine instance. Call destroy() on the returned engine when done to free native resources.
+ *
+ * @param options - STT initialization options or model path configuration
+ * @returns Promise resolving to an SttEngine instance
+ * @example
+ * ```typescript
+ * const stt = await createSTT({
+ *   modelPath: { type: 'asset', path: 'models/whisper-tiny' },
+ * });
+ * const result = await stt.transcribeFile('/path/to/audio.wav');
+ * console.log(result.text);
+ * await stt.destroy();
+ * ```
+ */
+export async function createSTT(
+  options: STTInitializeOptions | ModelPathConfig
+): Promise<SttEngine> {
+  const instanceId = `stt_${++sttInstanceCounter}`;
+
+  let modelPath: ModelPathConfig;
+  let preferInt8: boolean | undefined;
+  let modelType: STTModelType | undefined;
+  let hotwordsFile: string | undefined;
+  let hotwordsScore: number | undefined;
+  let numThreads: number | undefined;
+  let provider: string | undefined;
+  let ruleFsts: string | undefined;
+  let ruleFars: string | undefined;
+  let dither: number | undefined;
+  let modelOptions: SttModelOptions | undefined;
+  let modelingUnit: string | undefined;
+  let bpeVocab: string | undefined;
+
+  if ('modelPath' in options) {
+    modelPath = options.modelPath;
+    preferInt8 = options.preferInt8;
+    modelType = options.modelType;
+    hotwordsFile = options.hotwordsFile;
+    hotwordsScore = options.hotwordsScore;
+    numThreads = options.numThreads;
+    provider = options.provider;
+    ruleFsts = options.ruleFsts;
+    ruleFars = options.ruleFars;
+    dither = options.dither;
+    modelOptions = options.modelOptions;
+    modelingUnit = options.modelingUnit;
+    bpeVocab = options.bpeVocab;
+  } else {
+    modelPath = options;
+    preferInt8 = undefined;
+    modelType = undefined;
+    hotwordsFile = undefined;
+    hotwordsScore = undefined;
+    numThreads = undefined;
+    provider = undefined;
+    ruleFsts = undefined;
+    ruleFars = undefined;
+    dither = undefined;
+    modelOptions = undefined;
+    modelingUnit = undefined;
+    bpeVocab = undefined;
+  }
+
+  const debug = 'modelPath' in options ? options.debug : undefined;
+  const resolvedPath = await resolveModelPath(modelPath);
+
+  const result = await SherpaOnnx.initializeStt(
+    instanceId,
+    resolvedPath,
+    preferInt8,
+    modelType,
+    debug,
+    hotwordsFile,
+    hotwordsScore,
+    numThreads,
+    provider,
+    ruleFsts,
+    ruleFars,
+    dither,
+    modelOptions,
+    modelingUnit,
+    bpeVocab
+  );
+
+  if (!result.success) {
+    throw new Error(
+      `STT initialization failed: ${JSON.stringify(
+        result.detectedModels ?? []
+      )}`
+    );
+  }
+
+  let destroyed = false;
+
+  const guard = () => {
+    if (destroyed) {
+      throw new Error(
+        `STT instance ${instanceId} has been destroyed; cannot call methods on it.`
+      );
+    }
+  };
+
+  const engine: SttEngine = {
+    get instanceId() {
+      return instanceId;
+    },
+
+    async transcribeFile(filePath: string): Promise<SttRecognitionResult> {
+      guard();
+      const raw = await SherpaOnnx.transcribeFile(instanceId, filePath);
+      return normalizeSttResult(raw);
+    },
+
+    async transcribeSamples(
+      samples: number[],
+      sampleRate: number
+    ): Promise<SttRecognitionResult> {
+      guard();
+      const raw = await SherpaOnnx.transcribeSamples(
+        instanceId,
+        samples,
+        sampleRate
+      );
+      return normalizeSttResult(raw);
+    },
+
+    async setConfig(config: SttRuntimeConfig): Promise<void> {
+      guard();
+      const map: Record<string, string | number> = {};
+      if (config.decodingMethod != null)
+        map.decodingMethod = config.decodingMethod;
+      if (config.maxActivePaths != null)
+        map.maxActivePaths = config.maxActivePaths;
+      if (config.hotwordsFile != null) map.hotwordsFile = config.hotwordsFile;
+      if (config.hotwordsScore != null)
+        map.hotwordsScore = config.hotwordsScore;
+      if (config.blankPenalty != null) map.blankPenalty = config.blankPenalty;
+      if (config.ruleFsts != null) map.ruleFsts = config.ruleFsts;
+      if (config.ruleFars != null) map.ruleFars = config.ruleFars;
+      return SherpaOnnx.setSttConfig(instanceId, map);
+    },
+
+    async destroy(): Promise<void> {
+      if (destroyed) return;
+      destroyed = true;
+      await SherpaOnnx.unloadStt(instanceId);
+    },
+  };
+
+  return engine;
+}
+
+// Export types and runtime type list
+export type {
+  STTInitializeOptions,
+  STTModelType,
+  SttModelOptions,
+  SttRecognitionResult,
+  SttRuntimeConfig,
+  SttEngine,
+  SttInitResult,
+} from './types';
+export {
+  STT_MODEL_TYPES,
+  STT_HOTWORDS_MODEL_TYPES,
+  sttSupportsHotwords,
+} from './types';
+export {
+  getWhisperLanguages,
+  WHISPER_LANGUAGES,
+  getSenseVoiceLanguages,
+  SENSEVOICE_LANGUAGES,
+  getCanaryLanguages,
+  CANARY_LANGUAGES,
+  getFunasrNanoLanguages,
+  FUNASR_NANO_LANGUAGES,
+  getFunasrMltNanoLanguages,
+  FUNASR_MLT_NANO_LANGUAGES,
+} from './sttModelLanguages';
+export type { SttModelLanguage, WhisperLanguage } from './sttModelLanguages';

package/src/stt/sttModelLanguages.ts

@@ -0,0 +1,237 @@
+/**
+ * STT model language codes and display names.
+ * Per-model lists for Whisper, SenseVoice, and others. Use these for language-hint
+ * dropdowns so users only pick valid codes (invalid codes can crash the app, e.g. Whisper).
+ */
+
+export interface SttModelLanguage {
+  /**
+   * Value to pass as language (e.g. "en" for Whisper, "中文" for FunASR Nano).
+   * Use as modelOptions.<model>.language (or srcLang/tgtLang where applicable).
+   */
+  id: string;
+  /** Display name in English (e.g. "english", "chinese"). */
+  name: string;
+}
+
+/** @deprecated Use SttModelLanguage. Kept for backward compatibility. */
+export type WhisperLanguage = SttModelLanguage;
+
+// ========== Whisper ==========
+// https://github.com/ggml-org/whisper.cpp/blob/d682e150908e10caa4c15883c633d7902d385237/src/whisper.cpp#L248
+
+/** Ordered list of all Whisper-supported language codes and names. */
+export const WHISPER_LANGUAGES: readonly SttModelLanguage[] = [
+  { id: 'en', name: 'english' },
+  { id: 'zh', name: 'chinese' },
+  { id: 'de', name: 'german' },
+  { id: 'es', name: 'spanish' },
+  { id: 'ru', name: 'russian' },
+  { id: 'ko', name: 'korean' },
+  { id: 'fr', name: 'french' },
+  { id: 'ja', name: 'japanese' },
+  { id: 'pt', name: 'portuguese' },
+  { id: 'tr', name: 'turkish' },
+  { id: 'pl', name: 'polish' },
+  { id: 'ca', name: 'catalan' },
+  { id: 'nl', name: 'dutch' },
+  { id: 'ar', name: 'arabic' },
+  { id: 'sv', name: 'swedish' },
+  { id: 'it', name: 'italian' },
+  { id: 'id', name: 'indonesian' },
+  { id: 'hi', name: 'hindi' },
+  { id: 'fi', name: 'finnish' },
+  { id: 'vi', name: 'vietnamese' },
+  { id: 'he', name: 'hebrew' },
+  { id: 'uk', name: 'ukrainian' },
+  { id: 'el', name: 'greek' },
+  { id: 'ms', name: 'malay' },
+  { id: 'cs', name: 'czech' },
+  { id: 'ro', name: 'romanian' },
+  { id: 'da', name: 'danish' },
+  { id: 'hu', name: 'hungarian' },
+  { id: 'ta', name: 'tamil' },
+  { id: 'no', name: 'norwegian' },
+  { id: 'th', name: 'thai' },
+  { id: 'ur', name: 'urdu' },
+  { id: 'hr', name: 'croatian' },
+  { id: 'bg', name: 'bulgarian' },
+  { id: 'lt', name: 'lithuanian' },
+  { id: 'la', name: 'latin' },
+  { id: 'mi', name: 'maori' },
+  { id: 'ml', name: 'malayalam' },
+  { id: 'cy', name: 'welsh' },
+  { id: 'sk', name: 'slovak' },
+  { id: 'te', name: 'telugu' },
+  { id: 'fa', name: 'persian' },
+  { id: 'lv', name: 'latvian' },
+  { id: 'bn', name: 'bengali' },
+  { id: 'sr', name: 'serbian' },
+  { id: 'az', name: 'azerbaijani' },
+  { id: 'sl', name: 'slovenian' },
+  { id: 'kn', name: 'kannada' },
+  { id: 'et', name: 'estonian' },
+  { id: 'mk', name: 'macedonian' },
+  { id: 'br', name: 'breton' },
+  { id: 'eu', name: 'basque' },
+  { id: 'is', name: 'icelandic' },
+  { id: 'hy', name: 'armenian' },
+  { id: 'ne', name: 'nepali' },
+  { id: 'mn', name: 'mongolian' },
+  { id: 'bs', name: 'bosnian' },
+  { id: 'kk', name: 'kazakh' },
+  { id: 'sq', name: 'albanian' },
+  { id: 'sw', name: 'swahili' },
+  { id: 'gl', name: 'galician' },
+  { id: 'mr', name: 'marathi' },
+  { id: 'pa', name: 'punjabi' },
+  { id: 'si', name: 'sinhala' },
+  { id: 'km', name: 'khmer' },
+  { id: 'sn', name: 'shona' },
+  { id: 'yo', name: 'yoruba' },
+  { id: 'so', name: 'somali' },
+  { id: 'af', name: 'afrikaans' },
+  { id: 'oc', name: 'occitan' },
+  { id: 'ka', name: 'georgian' },
+  { id: 'be', name: 'belarusian' },
+  { id: 'tg', name: 'tajik' },
+  { id: 'sd', name: 'sindhi' },
+  { id: 'gu', name: 'gujarati' },
+  { id: 'am', name: 'amharic' },
+  { id: 'yi', name: 'yiddish' },
+  { id: 'lo', name: 'lao' },
+  { id: 'uz', name: 'uzbek' },
+  { id: 'fo', name: 'faroese' },
+  { id: 'ht', name: 'haitian creole' },
+  { id: 'ps', name: 'pashto' },
+  { id: 'tk', name: 'turkmen' },
+  { id: 'nn', name: 'nynorsk' },
+  { id: 'mt', name: 'maltese' },
+  { id: 'sa', name: 'sanskrit' },
+  { id: 'lb', name: 'luxembourgish' },
+  { id: 'my', name: 'myanmar' },
+  { id: 'bo', name: 'tibetan' },
+  { id: 'tl', name: 'tagalog' },
+  { id: 'mg', name: 'malagasy' },
+  { id: 'as', name: 'assamese' },
+  { id: 'tt', name: 'tatar' },
+  { id: 'haw', name: 'hawaiian' },
+  { id: 'ln', name: 'lingala' },
+  { id: 'ha', name: 'hausa' },
+  { id: 'ba', name: 'bashkir' },
+  { id: 'jw', name: 'javanese' },
+  { id: 'su', name: 'sundanese' },
+  { id: 'yue', name: 'cantonese' },
+] as const;
+
+/**
+ * Returns the list of Whisper-supported language codes and display names.
+ * Use for building a language-hint dropdown so users only pick valid codes (invalid codes can crash the app).
+ */
+export function getWhisperLanguages(): readonly SttModelLanguage[] {
+  return WHISPER_LANGUAGES;
+}
+
+// ========== SenseVoice ==========
+// https://github.com/FunAudioLLM/SenseVoice/blob/1a90d46cb933ef9e213b7d90292b9301b3e20f40/api.py#L22
+
+/** Ordered list of SenseVoice-supported language codes and names. */
+export const SENSEVOICE_LANGUAGES: readonly SttModelLanguage[] = [
+  { id: 'auto', name: 'auto' },
+  { id: 'zh', name: 'chinese' },
+  { id: 'en', name: 'english' },
+  { id: 'yue', name: 'cantonese' },
+  { id: 'ja', name: 'japanese' },
+  { id: 'ko', name: 'korean' },
+] as const;
+
+/**
+ * Returns the list of SenseVoice-supported language codes and display names.
+ * Use for modelOptions.senseVoice.language so users only pick valid codes.
+ */
+export function getSenseVoiceLanguages(): readonly SttModelLanguage[] {
+  return SENSEVOICE_LANGUAGES;
+}
+
+// ========== Canary ==========
+// Used for modelOptions.canary.srcLang and modelOptions.canary.tgtLang.
+// sherpa-onnx canary only supports 4 languages as it is the 180m model. The 1b model supports 25 languages.
+// https://build.nvidia.com/nvidia/canary-1b-asr/modelcard
+
+/** Canary: en, es, de, fr. */
+export const CANARY_LANGUAGES: readonly SttModelLanguage[] = [
+  { id: 'en', name: 'english' },
+  { id: 'es', name: 'spanish' },
+  { id: 'de', name: 'german' },
+  { id: 'fr', name: 'french' },
+] as const;
+
+/**
+ * Returns the list of Canary-supported language codes and display names.
+ * Use for modelOptions.canary.srcLang and modelOptions.canary.tgtLang.
+ */
+export function getCanaryLanguages(): readonly SttModelLanguage[] {
+  return CANARY_LANGUAGES;
+}
+
+// ========== FunASR Nano ==========
+// Ids are the values passed to model.generate(..., language="中文"). Names are English display names.
+// https://github.com/FunAudioLLM/Fun-ASR/blob/7dfdb6639e2ba861d3311a8d8c0e3578a8d24122/README.md?plain=1#L99
+
+/** Fun-ASR-Nano-2512: Chinese, English, Japanese. */
+export const FUNASR_NANO_LANGUAGES: readonly SttModelLanguage[] = [
+  { id: '中文', name: 'chinese' },
+  { id: '英文', name: 'english' },
+  { id: '日文', name: 'japanese' },
+] as const;
+
+/** Fun-ASR-MLT-Nano-2512: multilingual list. */
+export const FUNASR_MLT_NANO_LANGUAGES: readonly SttModelLanguage[] = [
+  { id: '中文', name: 'chinese' },
+  { id: '英文', name: 'english' },
+  { id: '粤语', name: 'cantonese' },
+  { id: '日文', name: 'japanese' },
+  { id: '韩文', name: 'korean' },
+  { id: '越南语', name: 'vietnamese' },
+  { id: '印尼语', name: 'indonesian' },
+  { id: '泰语', name: 'thai' },
+  { id: '马来语', name: 'malay' },
+  { id: '菲律宾语', name: 'filipino' },
+  { id: '阿拉伯语', name: 'arabic' },
+  { id: '印地语', name: 'hindi' },
+  { id: '保加利亚语', name: 'bulgarian' },
+  { id: '克罗地亚语', name: 'croatian' },
+  { id: '捷克语', name: 'czech' },
+  { id: '丹麦语', name: 'danish' },
+  { id: '荷兰语', name: 'dutch' },
+  { id: '爱沙尼亚语', name: 'estonian' },
+  { id: '芬兰语', name: 'finnish' },
+  { id: '希腊语', name: 'greek' },
+  { id: '匈牙利语', name: 'hungarian' },
+  { id: '爱尔兰语', name: 'irish' },
+  { id: '拉脱维亚语', name: 'latvian' },
+  { id: '立陶宛语', name: 'lithuanian' },
+  { id: '马耳他语', name: 'maltese' },
+  { id: '波兰语', name: 'polish' },
+  { id: '葡萄牙语', name: 'portuguese' },
+  { id: '罗马尼亚语', name: 'romanian' },
+  { id: '斯洛伐克语', name: 'slovak' },
+  { id: '斯洛文尼亚语', name: 'slovenian' },
+  { id: '瑞典语', name: 'swedish' },
+] as const;
+
+/**
+ * Returns languages for Fun-ASR-Nano-2512 (中文, 英文, 日文).
+ * Id is the value for modelOptions.funasrNano.language (e.g. "中文").
+ */
+export function getFunasrNanoLanguages(): readonly SttModelLanguage[] {
+  return FUNASR_NANO_LANGUAGES;
+}
+
+/**
+ * Returns languages for Fun-ASR-MLT-Nano-2512 (multilingual).
+ * Id is the value for modelOptions.funasrNano.language (e.g. "中文").
+ */
+export function getFunasrMltNanoLanguages(): readonly SttModelLanguage[] {
+  return FUNASR_MLT_NANO_LANGUAGES;
+}