parakeet.js 0.0.3 → 1.0.0

package/README.md

@@ -7,7 +7,27 @@ Runs entirely in the browser on **WebGPU** or **WASM** via
 > **Parakeet.js** offers a high-performance, browser-first implementation for NVIDIA's Parakeet-TDT speech-to-text models, running entirely client-side via WebGPU and WASM. Powered by ONNX Runtime Web, this library makes it simple to integrate state-of-the-art transcription into any web application.
 
 > **Status:** Early preview – API is subject to change while things stabilise.
-> **Note:** Currently only supports the Parakeet-TDT model architecture.
+> **Note:** Currently supports Parakeet-TDT v2 (English) and v3 (Multilingual) model architectures.
+
+---
+
+## What's New (v0.3.x)
+
+### 🌐 Parakeet TDT v3 Multilingual Support
+- Added support for **Parakeet TDT 0.6B v3** with 13 languages: English, French, German, Spanish, Italian, Portuguese, Dutch, Polish, Russian, Ukrainian, Japanese, Korean, Chinese
+- Both v2 (English-only) and v3 (Multilingual) models now work out of the box
+- Use model keys for easier loading: `'parakeet-tdt-0.6b-v2'` or `'parakeet-tdt-0.6b-v3'`
+
+### 🎛️ Model Configuration API
+- New `MODELS` export with model metadata (supported languages, vocab size, etc.)
+- `getModelConfig()` for programmatic model introspection
+- `supportsLanguage()` helper to check language compatibility
+
+### 🧪 Demo App Improvements
+- **Model selector** dropdown to switch between v2 and v3
+- **Language selector** (context-aware, shows only supported languages)
+- **Quick Test** feature with HuggingFace speech datasets (People's Speech, MLS)
+- **Reference text** display for comparing transcription accuracy
 
 ---
 
@@ -29,23 +49,29 @@ yarn add parakeet.js onnxruntime-web
 
 We host ready-to-use ONNX exports on the HuggingFace Hub:
 
-```
-istupakov/parakeet-tdt-0.6b-v2-onnx
-```
+| Model | Languages | Repo ID |
+|-------|-----------|---------|
+| Parakeet TDT 0.6B v2 | English | `istupakov/parakeet-tdt-0.6b-v2-onnx` |
+| Parakeet TDT 0.6B v3 | 13 languages | `istupakov/parakeet-tdt-0.6b-v3-onnx` |
 
 The helper `getParakeetModel()` downloads all required files and caches them in **IndexedDB**:
 
 ```js
-import { getParakeetModel } from 'parakeet.js';
+import { getParakeetModel, MODELS } from 'parakeet.js';
 
-const repoId = 'istupakov/parakeet-tdt-0.6b-v2-onnx';
-const { urls, filenames } = await getParakeetModel(repoId, {
-  backend: 'webgpu', // 'webgpu' or 'wasm'
-  encoderQuant: 'fp32',    // 'fp32' or 'int8'
-  decoderQuant: 'int8',    // 'fp32' or 'int8'
-  preprocessor: 'nemo128',
+// Option 1: Use model key (recommended)
+const { urls, filenames, modelConfig } = await getParakeetModel('parakeet-tdt-0.6b-v3', {
+  backend: 'webgpu',
   progress: ({file,loaded,total}) => console.log(file, loaded/total)
 });
+
+// Option 2: Use repo ID directly
+const { urls, filenames } = await getParakeetModel('istupakov/parakeet-tdt-0.6b-v2-onnx', {
+  backend: 'webgpu',
+  encoderQuant: 'fp32',
+  decoderQuant: 'int8',
+  preprocessor: 'nemo128',
+});
 ```
 
 Returned structure:
@@ -109,7 +135,7 @@ Extra options:
 
 | Option | Default | Description |
 |--------|---------|-------------|
-| `temperature` | 1.2 | Softmax temperature for decoding |
+| `temperature` | 1.0 | Softmax temperature for decoding (1.0 = greedy, >1.0 = sampling) |
 | `frameStride` | 1 | Advance decoder by *n* encoder frames per step |
 
 ### Result schema
@@ -170,25 +196,83 @@ if (utterance_text.toLowerCase().includes(expected)) {
 
 ---
 
+## Model Configuration API
+
+Query model metadata programmatically:
+
+```js
+import { MODELS, LANGUAGE_NAMES, getModelConfig, supportsLanguage } from 'parakeet.js';
+
+// List all available models
+console.log(Object.keys(MODELS)); 
+// ['parakeet-tdt-0.6b-v2', 'parakeet-tdt-0.6b-v3']
+
+// Get model config
+const config = getModelConfig('parakeet-tdt-0.6b-v3');
+console.log(config.languages); // ['en', 'fr', 'de', 'es', ...]
+console.log(config.displayName); // 'Parakeet TDT 0.6B v3 (Multilingual)'
+
+// Check language support
+supportsLanguage('parakeet-tdt-0.6b-v3', 'fr'); // true
+supportsLanguage('parakeet-tdt-0.6b-v2', 'fr'); // false
+
+// Get language display names
+console.log(LANGUAGE_NAMES['fr']); // 'French'
+```
+
+---
+
 ## Using the React demo as a template
 
-Located at `examples/react-demo`.
+Located at `examples/react-demo` (production) and `examples/react-demo-dev` (development).
 
 Quick start:
 
 ```bash
-cd examples/react-demo
+cd examples/react-demo-dev
 npm i
 npm run dev  # Vite => http://localhost:5173
 ```
 
-Key components:
+### Demo Features
+
+The development demo (`react-demo-dev`) includes advanced features:
+
+- **Model Selector**: Switch between v2 (English) and v3 (Multilingual)
+- **Language Selector**: Context-aware dropdown showing only supported languages
+- **Quick Test**: Load random samples from HuggingFace speech datasets
+- **Reference Text**: Compare transcription against ground truth
+
+### Speech Dataset Utilities (Demo Only)
+
+The demo includes reusable utilities for testing with HuggingFace datasets:
+
+```js
+// Located in: examples/react-demo-dev/src/utils/speechDatasets.js
+import { fetchRandomSample, hasTestSamples, SPEECH_DATASETS } from './utils/speechDatasets';
+
+// Check if test samples are available for a language
+if (hasTestSamples('fr')) {
+  // Fetch a random French audio sample with transcription
+  const sample = await fetchRandomSample('fr', {
+    targetSampleRate: 16000,
+    onProgress: ({ message }) => console.log(message),
+  });
+  
+  console.log(sample.transcription); // Ground truth text
+  console.log(sample.pcm);           // Float32Array audio
+  console.log(sample.duration);      // Duration in seconds
+}
+```
+
+**Supported languages for testing:** English (People's Speech), French, German, Spanish, Italian, Portuguese, Dutch, Polish (Multilingual LibriSpeech)
+
+### Key Files
 
 | File | Purpose |
 |------|---------|
-| `App.jsx` | Complete end-to-end reference UI. Shows how to load a model with progress bars, perform a warm-up/verification step, display performance metrics (RTF, timings), and manage transcription history. |
-| `parakeet.js` | Library entry; houses the model wrapper and performance instrumentation. |
-| `hub.js` | Lightweight HuggingFace Hub helper – downloads and caches model binaries. |
+| `App.jsx` | Complete end-to-end reference UI with model/language selection, performance metrics, and transcription history |
+| `utils/speechDatasets.js` | Reusable utilities for fetching test samples from HuggingFace datasets |
 
 Copy-paste the `loadModel()` and `transcribeFile()` functions into your app, adjust UI bindings, and you are ready to go.
 
@@ -223,7 +307,19 @@ The demo is also available locally at `examples/hf-spaces-demo` and can be deplo
 
 ## Changelog
 
-See `OPTIMIZATION_PLAN.md` for a timeline of performance tweaks and planned features.
+### v0.3.x (January 2026)
+- ✨ **Multilingual Support**: Added Parakeet TDT 0.6B v3 with 13 languages
+- 🎛️ **Model Config API**: New `MODELS`, `LANGUAGE_NAMES`, `getModelConfig()`, `supportsLanguage()` exports
+- 🧪 **Demo Enhancements**: Model/language selectors, HuggingFace dataset testing
+- 🔧 **TDT Decoding Fix**: Aligned decoding logic with NeMo framework for improved accuracy
+- 🌊 **Streaming Support**: Added incremental transcription capabilities
+
+### v0.2.x
+- Initial WebGPU/WASM hybrid backend
+- IndexedDB model caching
+- Performance instrumentation (RTF, timing metrics)
+
+See `OPTIMIZATION_PLAN.md` for detailed performance notes.
 
 ---
 
@@ -232,9 +328,10 @@ See `OPTIMIZATION_PLAN.md` for a timeline of performance tweaks and planned feat
 This project builds upon the excellent work of:
 
 - **[istupakov](https://github.com/istupakov)** - For providing the [ONNX-ASR](https://github.com/istupakov/onnx-asr) repository, which served as the foundation and starting point for this JavaScript implementation
-- **[istupakov/parakeet-tdt-0.6b-v2-onnx](https://huggingface.co/istupakov/parakeet-tdt-0.6b-v2-onnx)** - For the ONNX model exports and preprocessor implementations that made this library possible.
+- **[istupakov/parakeet-tdt-0.6b-v2-onnx](https://huggingface.co/istupakov/parakeet-tdt-0.6b-v2-onnx)** - English model exports
+- **[istupakov/parakeet-tdt-0.6b-v3-onnx](https://huggingface.co/istupakov/parakeet-tdt-0.6b-v3-onnx)** - Multilingual model exports
 - **ONNX Runtime Web** - For powering the browser-based inference engine
-- **ONNX Runtime Node** - For enabling high-performance server-side inference
+- **HuggingFace Datasets** - People's Speech, Multilingual LibriSpeech for testing
 
 The Python-based ONNX-ASR project provided crucial insights into model handling, preprocessing pipelines, and served as a reference implementation during the development of this browser-compatible version.
 

package/package.json

@@ -1,18 +1,25 @@
 {
   "name": "parakeet.js",
-  "version": "0.0.3",
+  "version": "1.0.0",
   "description": "NVIDIA Parakeet speech recognition for the browser (WebGPU/WASM) powered by ONNX Runtime Web.",
   "type": "module",
   "exports": {
     ".": "./src/index.js"
   },
+  "files": [
+    "src",
+    "README.md",
+    "LICENSE"
+  ],
   "keywords": [
     "parakeet",
     "speech",
     "onnx",
     "webgpu",
     "wasm",
-    "transcription"
+    "transcription",
+    "multilingual",
+    "asr"
   ],
   "dependencies": {
     "onnxruntime-web": "1.22.0-dev.20250409-89f8206ba4"

package/src/backend.js

@@ -1,99 +1,107 @@
-// Back-end initialisation helper for ONNX Runtime Web.
-// At runtime the caller can specify preferred backend ("webgpu", "wasm").
-// The function resolves once ONNX Runtime is ready and returns the `ort` module.
-
-/**
- * Initialise ONNX Runtime Web and pick the execution provider.
- * If WebGPU is requested but not supported, we transparently fall back to WASM.
- * @param {Object} opts
- * @param {('webgpu'|'wasm')} [opts.backend='webgpu'] Desired backend.
- * @param {string} [opts.wasmPaths] Optional path prefix for WASM binaries.
- * @returns {Promise<typeof import('onnxruntime-web').default>}
- */
-export async function initOrt({ backend = 'webgpu', wasmPaths, numThreads } = {}) {
-  // Dynamic import to handle Vite bundling issues
-  let ort;
-  
-  try {
-    const ortModule = await import('onnxruntime-web');
-    ort = ortModule.default || ortModule;
-    
-    // Debug: Check the structure of ort
-    console.log('[Parakeet.js] ORT structure:', { 
-      hasDefault: !!ortModule.default, 
-      hasEnv: !!ort.env, 
-      hasWasm: !!ort.env?.wasm,
-      hasWebgpu: !!ort.env?.webgpu,
-      keys: Object.keys(ort).slice(0, 10) // Show first 10 keys
-    });
-    
-    // If still no env, try accessing it differently
-    if (!ort.env) {
-      console.log('[Parakeet.js] Trying alternative access patterns...');
-      console.log('[Parakeet.js] ortModule keys:', Object.keys(ortModule));
-      
-      // Sometimes the module structure is nested
-      if (ortModule.ort) {
-        ort = ortModule.ort;
-        console.log('[Parakeet.js] Found ort in ortModule.ort');
-      }
-    }
-  } catch (e) {
-    console.error('[Parakeet.js] Failed to import onnxruntime-web:', e);
-    throw new Error('Failed to load ONNX Runtime Web. Please check your network connection.');
-  }
-  
-  if (!ort || !ort.env) {
-    throw new Error('ONNX Runtime Web loaded but env is not available. This might be a bundling issue.');
-  }
-  
-  // Set up WASM paths first (needed for all backends)
-  if (!ort.env.wasm.wasmPaths) {
-    // Use the same version as in package.json
-    const ver = '1.22.0-dev.20250409-89f8206ba4';
-    ort.env.wasm.wasmPaths = `https://cdn.jsdelivr.net/npm/onnxruntime-web@${ver}/dist/`;
-  }
-
-  // Configure WASM for better performance
-  if (backend === 'wasm' || backend === 'webgpu') {
-    // Enable multi-threading if supported
-    if (typeof SharedArrayBuffer !== 'undefined') {
-      ort.env.wasm.numThreads = numThreads || navigator.hardwareConcurrency || 4;
-      ort.env.wasm.simd = true;
-      console.log(`[Parakeet.js] WASM configured with ${ort.env.wasm.numThreads} threads, SIMD enabled`);
-    } else {
-      console.warn('[Parakeet.js] SharedArrayBuffer not available - using single-threaded WASM');
-      ort.env.wasm.numThreads = 1;
-    }
-    
-    // Enable other WASM optimizations
-    ort.env.wasm.proxy = false; // Direct execution for better performance
-  }
-
-  if (backend === 'webgpu') {
-    // Check WebGPU support properly
-    const webgpuSupported = 'gpu' in navigator;
-    console.log(`[Parakeet.js] WebGPU supported: ${webgpuSupported}`);
-    
-    if (webgpuSupported) {
-      try {
-        // In newer versions of ONNX Runtime Web, WebGPU initialization is automatic
-        // No need to call ort.env.webgpu.init() manually
-        console.log('[Parakeet.js] WebGPU will be initialized automatically when creating session');
-      } catch (error) {
-        console.warn('[Parakeet.js] WebGPU initialization failed:', error);
-        console.warn('[Parakeet.js] Falling back to WASM');
-        backend = 'wasm';
-      }
-    } else {
-      console.warn('[Parakeet.js] WebGPU not supported – falling back to WASM');
-      backend = 'wasm';
-    }
-  }
-
-  // Store the final backend choice for use in model selection
-  ort._selectedBackend = backend;
-
-  // Return the ort module for use in creating sessions and tensors
-  return ort;
+// Back-end initialisation helper for ONNX Runtime Web.
+// At runtime the caller can specify preferred backend ("webgpu", "wasm").
+// The function resolves once ONNX Runtime is ready and returns the `ort` module.
+
+/**
+ * Initialise ONNX Runtime Web and pick the execution provider.
+ * If WebGPU is requested but not supported, we transparently fall back to WASM.
+ * @param {Object} opts
+ * @param {('webgpu'|'wasm')} [opts.backend='webgpu'] Desired backend.
+ * @param {string} [opts.wasmPaths] Optional path prefix for WASM binaries.
+ * @returns {Promise<typeof import('onnxruntime-web').default>}
+ */
+export async function initOrt({ backend = 'webgpu', wasmPaths, numThreads } = {}) {
+  // Dynamic import to handle Vite bundling issues
+  let ort;
+  
+  try {
+    const ortModule = await import('onnxruntime-web');
+    ort = ortModule.default || ortModule;
+    
+    // Debug: Check the structure of ort
+    console.log('[Parakeet.js] ORT structure:', { 
+      hasDefault: !!ortModule.default, 
+      hasEnv: !!ort.env, 
+      hasWasm: !!ort.env?.wasm,
+      hasWebgpu: !!ort.env?.webgpu,
+      keys: Object.keys(ort).slice(0, 10) // Show first 10 keys
+    });
+    
+    // If still no env, try accessing it differently
+    if (!ort.env) {
+      console.log('[Parakeet.js] Trying alternative access patterns...');
+      console.log('[Parakeet.js] ortModule keys:', Object.keys(ortModule));
+      
+      // Sometimes the module structure is nested
+      if (ortModule.ort) {
+        ort = ortModule.ort;
+        console.log('[Parakeet.js] Found ort in ortModule.ort');
+      }
+    }
+  } catch (e) {
+    console.error('[Parakeet.js] Failed to import onnxruntime-web:', e);
+    throw new Error('Failed to load ONNX Runtime Web. Please check your network connection.');
+  }
+  
+  if (!ort || !ort.env) {
+    throw new Error('ONNX Runtime Web loaded but env is not available. This might be a bundling issue.');
+  }
+  
+  // Set up WASM paths first (needed for all backends)
+  if (!ort.env.wasm.wasmPaths) {
+    // Use the same version as in package.json
+    const ver = '1.22.0-dev.20250409-89f8206ba4';
+    ort.env.wasm.wasmPaths = `https://cdn.jsdelivr.net/npm/onnxruntime-web@${ver}/dist/`;
+  }
+
+  // Configure WASM for better performance
+  if (backend === 'wasm' || backend === 'webgpu') {
+    // Enable multi-threading if supported
+    if (typeof SharedArrayBuffer !== 'undefined') {
+      ort.env.wasm.numThreads = numThreads || navigator.hardwareConcurrency || 4;
+      ort.env.wasm.simd = true;
+      console.log(`[Parakeet.js] WASM configured with ${ort.env.wasm.numThreads} threads, SIMD enabled`);
+    } else {
+      console.warn('[Parakeet.js] SharedArrayBuffer not available - using single-threaded WASM');
+      ort.env.wasm.numThreads = 1;
+    }
+    
+    // Enable other WASM optimizations
+    ort.env.wasm.proxy = false; // Direct execution for better performance
+  }
+
+  if (backend === 'webgpu') {
+    // Check WebGPU support properly
+    const webgpuSupported = 'gpu' in navigator;
+    console.log(`[Parakeet.js] WebGPU supported: ${webgpuSupported}`);
+    
+    if (webgpuSupported) {
+      try {
+        // In newer versions of ONNX Runtime Web, WebGPU initialization is automatic
+        // No need to call ort.env.webgpu.init() manually
+        console.log('[Parakeet.js] WebGPU will be initialized automatically when creating session');
+      } catch (error) {
+        console.warn('[Parakeet.js] WebGPU initialization failed:', error);
+        console.warn('[Parakeet.js] Falling back to WASM');
+        backend = 'wasm';
+      }
+    } else {
+      console.warn('[Parakeet.js] WebGPU not supported – falling back to WASM');
+      backend = 'wasm';
+    }
+  }
+
+  // Store the final backend choice for use in model selection
+  ort._selectedBackend = backend;
+
+  // Expose ort globally so other modules (like SileroVAD) can use the same configured instance
+  if (typeof globalThis !== 'undefined') {
+    globalThis.ort = ort;
+  }
+  if (typeof self !== 'undefined') {
+    self.ort = ort;
+  }
+
+  // Return the ort module for use in creating sessions and tensors
+  return ort;
 }

package/src/hub.js

@@ -3,6 +3,8 @@
  * Downloads models from HF and caches them in browser storage.
  */
 
+import { MODELS, getModelConfig } from './models.js';
+
 const DB_NAME = 'parakeet-cache-db';
 const STORE_NAME = 'file-store';
 let dbPromise = null;
@@ -164,17 +166,24 @@ export async function getModelText(repoId, filename, options = {}) {
 
 /**
  * Convenience function to get all Parakeet model files for a given architecture.
- * @param {string} repoId HF repo (e.g., 'nvidia/parakeet-tdt-1.1b')
+ * @param {string} repoIdOrModelKey HF repo (e.g., 'nvidia/parakeet-tdt-1.1b') or model key (e.g., 'parakeet-tdt-0.6b-v3')
  * @param {Object} [options]
  * @param {('int8'|'fp32')} [options.encoderQuant='int8'] Encoder quantization
  * @param {('int8'|'fp32')} [options.decoderQuant='int8'] Decoder quantization
- * @param {('nemo80'|'nemo128')} [options.preprocessor='nemo128'] Preprocessor variant
+ * @param {('nemo80'|'nemo128')} [options.preprocessor] Preprocessor variant (auto-detected from model config if not specified)
  * @param {('webgpu'|'wasm')} [options.backend='webgpu'] Backend to use
  * @param {Function} [options.progress] Progress callback
- * @returns {Promise<{urls: object, filenames: object}>}
+ * @returns {Promise<{urls: object, filenames: object, modelConfig: object|null}>}
  */
-export async function getParakeetModel(repoId, options = {}) {
-  const { encoderQuant = 'int8', decoderQuant = 'int8', preprocessor = 'nemo128', backend = 'webgpu', progress } = options;
+export async function getParakeetModel(repoIdOrModelKey, options = {}) {
+  // Resolve model key to repo ID and get config
+  const modelConfig = getModelConfig(repoIdOrModelKey);
+  const repoId = modelConfig?.repoId || repoIdOrModelKey;
+  
+  // Use model config defaults if available
+  const defaultPreprocessor = modelConfig?.preprocessor || 'nemo128';
+  
+  const { encoderQuant = 'int8', decoderQuant = 'int8', preprocessor = defaultPreprocessor, backend = 'webgpu', progress } = options;
   
   // Decide quantisation per component
   let encoderQ = encoderQuant;
@@ -215,7 +224,8 @@ export async function getParakeetModel(repoId, options = {}) {
           encoder: encoderName,
           decoder: decoderName
       },
-      quantisation: { encoder: encoderQ, decoder: decoderQ }
+      quantisation: { encoder: encoderQ, decoder: decoderQ },
+      modelConfig: modelConfig || null,  // Include model config for downstream use
   };
   
   for (const { key, name } of filesToGet) {

package/src/index.js

@@ -1,29 +1,37 @@
-export { ParakeetModel } from './parakeet.js';
-export { getModelFile, getModelText, getParakeetModel } from './hub.js';
-
-/**
- * Convenience factory to load from a local path.
- *
- * Example:
- * import { fromUrls } from 'parakeet.js';
- * const model = await fromUrls({ ... });
- */
-export async function fromUrls(cfg) {
-  const { ParakeetModel } = await import('./parakeet.js');
-  return ParakeetModel.fromUrls(cfg);
-}
-
-/**
- * Convenience factory to load from HuggingFace Hub.
- *
- * Example:
- * import { fromHub } from 'parakeet.js';
- * const model = await fromHub('nvidia/parakeet-tdt-1.1b', { quantization: 'int8' });
- */
-export async function fromHub(repoId, options = {}) {
-  const { getParakeetModel } = await import('./hub.js');
-  const { ParakeetModel } = await import('./parakeet.js');
-  
-  const urls = await getParakeetModel(repoId, options);
-  return ParakeetModel.fromUrls({ ...urls, ...options });
+export { ParakeetModel } from './parakeet.js';
+export { getModelFile, getModelText, getParakeetModel } from './hub.js';
+export { MODELS, LANGUAGE_NAMES, DEFAULT_MODEL, getModelConfig, getModelKeyFromRepoId, supportsLanguage, listModels, getLanguageName } from './models.js';
+
+/**
+ * Convenience factory to load from a local path.
+ *
+ * Example:
+ * import { fromUrls } from 'parakeet.js';
+ * const model = await fromUrls({ ... });
+ */
+export async function fromUrls(cfg) {
+  const { ParakeetModel } = await import('./parakeet.js');
+  return ParakeetModel.fromUrls(cfg);
+}
+
+/**
+ * Convenience factory to load from HuggingFace Hub.
+ *
+ * Example:
+ * import { fromHub } from 'parakeet.js';
+ * const model = await fromHub('nvidia/parakeet-tdt-1.1b', { quantization: 'int8' });
+ * 
+ * // Or use a model key for known models:
+ * const model = await fromHub('parakeet-tdt-0.6b-v3', { quantization: 'int8' });
+ */
+export async function fromHub(repoIdOrModelKey, options = {}) {
+  const { getParakeetModel } = await import('./hub.js');
+  const { ParakeetModel } = await import('./parakeet.js');
+  const { MODELS } = await import('./models.js');
+  
+  // Resolve model key to repo ID if needed
+  const repoId = MODELS[repoIdOrModelKey]?.repoId || repoIdOrModelKey;
+  
+  const urls = await getParakeetModel(repoId, options);
+  return ParakeetModel.fromUrls({ ...urls, ...options });
 }