whispermix 1.5.2 → 1.6.4

package/README.md

@@ -75,6 +75,31 @@ whisperGroq.fromStream(audioStream)
 
 **Note:** Stream transcription is only available for API-based models (OpenAI and Groq). Local models (Whisper and Parakeet) require file input.
 
+### ⏱️ Word-Level Timestamps
+
+By default, WhisperMix returns only the transcribed text. To include the start and end time for each word, pass `wordTimestamps: true` per call:
+
+```javascript
+const result = await whisperGroq.fromFile('path/to/audio.mp3', {
+  wordTimestamps: true
+});
+
+console.log(result.text);
+console.log(result.words);
+// [{ word: 'Hello', start: 0.12, end: 0.48 }, ...]
+```
+
+You can also enable it on the instance:
+
+```javascript
+const whisper = new WhisperMix({
+  model: 'openai/whisper-1',
+  wordTimestamps: true
+});
+
+const result = await whisper.fromStream(audioStream);
+```
+
 ### 🐦 Parakeet local models
 
 Parakeet TDT v3 local models are downloaded once and cached in:
@@ -135,21 +160,25 @@ Creates a new WhisperMix instance.
 - `options.bottleneck`: (Optional) Configuration for Bottleneck rate limiting (API models only).
 - `options.chunkSize`: (Optional) The size in seconds of the chunks to split the audio into. Default is 890 seconds.
 - `options.language`: (Optional) Language for local Whisper model. Defaults to 'auto' for automatic detection.
-### `whisper.fromFile(filePath)`
+- `options.wordTimestamps`: (Optional) Return `{ text, words }` instead of plain text, where each word has `start` and `end` times in seconds.
+
+### `whisper.fromFile(filePath, options)`
 
 Transcribes audio from a file.
 
 - `filePath`: Path to the audio file.
+- `options.wordTimestamps`: (Optional) Return word-level timestamps for this call.
 
-Returns a Promise that resolves with the transcription text.
+Returns a Promise that resolves with the transcription text, or `{ text, words }` when `wordTimestamps` is enabled.
 
-### `whisper.fromStream(audioStream)`
+### `whisper.fromStream(audioStream, options)`
 
 Transcribes audio from a stream.
 
 - `audioStream`: A readable stream of the audio data.
+- `options.wordTimestamps`: (Optional) Return word-level timestamps for this call.
 
-Returns a Promise that resolves with the transcription text.
+Returns a Promise that resolves with the transcription text, or `{ text, words }` when `wordTimestamps` is enabled.
 
 ## 📄 License
 

package/index.js

@@ -49,6 +49,7 @@ const PARAKEET_LAYOUTS = {
         },
     },
 };
+const PARAKEET_DEFAULT_CHUNK_SIZE = 40;
 
 class WhisperMix {
     constructor(setup = {}) {
@@ -118,6 +119,9 @@ class WhisperMix {
         this.showProgress = this.showProgress || false;
         this.localBackend = this.config.backend || 'transformers';
         this.layout = this.config.layout;
+        if (setup.chunkSize === undefined && this.localBackend === 'onnx-asr-web') {
+            this.chunkSize = PARAKEET_DEFAULT_CHUNK_SIZE;
+        }
         this.transcriber = null;
         this._onnxAsrNodeModule = null;
         this._warnedParakeetLanguage = false;
@@ -125,7 +129,8 @@ class WhisperMix {
         this.limiter = new Bottleneck(this.bottleneck);
     }
 
-    async fromFile(filePath) {
+    async fromFile(filePath, options = {}) {
+        const transcriptionOptions = this._resolveTranscriptionOptions(options);
         const absolutePath = path.resolve(filePath);
         
         // Check if file exists
@@ -142,14 +147,10 @@ class WhisperMix {
 
             if (duration <= this.chunkSize) {
                 // Process as a single file
-                if (this.isLocal) {
-                    return this._transcribeLocalFile(absolutePath);
-                } else {
-                    return this.fromStream(fs.createReadStream(absolutePath));
-                }
+                return this._transcribeFilePath(absolutePath, transcriptionOptions);
             } else {
                 // Split audio and process chunks
-                let accumulatedTranscription = "";
+                const transcriptions = [];
                 const numChunks = Math.ceil(duration / this.chunkSize);
 
                 for (let i = 0; i < numChunks; i++) {
@@ -171,14 +172,8 @@ class WhisperMix {
                             .run();
                     });
 
-                    let transcription;
-                    if (this.isLocal) {
-                        transcription = await this._transcribeLocalFile(chunkPath);
-                    } else {
-                        const chunkStream = fs.createReadStream(chunkPath);
-                        transcription = await this.fromStream(chunkStream);
-                    }
-                    accumulatedTranscription += (transcription + " ").trimStart();
+                    const transcription = await this._transcribeFilePath(chunkPath, transcriptionOptions);
+                    transcriptions.push(this._offsetTranscription(transcription, startTime));
                     
                     // Clean up chunk immediately after processing
                     try {
@@ -187,7 +182,7 @@ class WhisperMix {
                         console.warn(`Could not delete chunk ${chunkPath}:`, unlinkErr);
                     }
                 }
-                return accumulatedTranscription.trim();
+                return this._mergeTranscriptions(transcriptions, transcriptionOptions, duration);
             }
         } finally {
             // Clean up the temporary directory
@@ -199,10 +194,18 @@ class WhisperMix {
         }
     }
 
-    async fromStream(audioStream) {
+    async _transcribeFilePath(filePath, options) {
+        if (this.isLocal) {
+            return this._transcribeLocalFile(filePath, options);
+        }
+        return this.fromStream(fs.createReadStream(filePath), options);
+    }
+
+    async fromStream(audioStream, options = {}) {
         if (this.isLocal) {
             throw new Error('fromStream is not supported for local Whisper model. Use fromFile instead.');
         }
+        const transcriptionOptions = this._resolveTranscriptionOptions(options);
 
         const chunks = [];
         for await (const chunk of audioStream) {
@@ -210,10 +213,10 @@ class WhisperMix {
         }
         const buffer = Buffer.concat(chunks);
 
-        return this.limiter.schedule(() => this._makeRequest(buffer));
+        return this.limiter.schedule(() => this._makeRequest(buffer, transcriptionOptions));
     }
 
-    async _makeRequest(buffer) {
+    async _makeRequest(buffer, options = {}) {
         try {
             const blob = new Blob([buffer], { type: 'audio/mpeg' });
             const formData = new FormData();
@@ -222,6 +225,10 @@ class WhisperMix {
             if (this.language) {
                 formData.append('language', this.language);
             }
+            if (options.wordTimestamps) {
+                formData.append('response_format', 'verbose_json');
+                formData.append('timestamp_granularities[]', 'word');
+            }
 
             const response = await fetch(this.apiUrl, {
                 method: 'POST',
@@ -237,13 +244,13 @@ class WhisperMix {
                 throw responseData;
             }
 
-            return responseData.text.trim();
+            return this._formatApiTranscription(responseData, options);
         } catch (error) {
             throw error?.message || error;
         }
     }
 
-    async _transcribeLocalFile(filePath) {
+    async _transcribeLocalFile(filePath, options = {}) {
         try {
             // Read the audio file as a buffer
             const buffer = fs.readFileSync(filePath);
@@ -282,11 +289,11 @@ class WhisperMix {
                     this._warnedParakeetLanguage = true;
                 }
                 const result = await transcriber.transcribeSamples(audioData, 16000);
-                const text = result?.text || result?.utterance_text;
+                const text = result?.text ?? result?.utterance_text;
                 if (typeof text !== 'string') {
                     throw new Error('Parakeet transcription returned no text output.');
                 }
-                return text.trim();
+                return this._formatLocalTranscription(text, result?.words, options);
             }
 
             // Pass the processed audio data
@@ -296,14 +303,162 @@ class WhisperMix {
             if (this.language !== undefined) {
                 transcriberOptions.language = this.language;
             }
+            if (options.wordTimestamps) {
+                transcriberOptions.return_timestamps = 'word';
+            }
             const result = await transcriber(audioData, transcriberOptions);
-            return result.text.trim();
+            const words = options.wordTimestamps ? this._wordsFromTransformersChunks(result.chunks) : undefined;
+            return this._formatLocalTranscription(result.text, words, options);
         } catch (error) {
             const cacheHint = this.localBackend === 'onnx-asr-web'
                 ? this._getParakeetCacheDir(this.layout)
                 : `${env.cacheDir}${this.modelName}/`;
-            throw new Error(`Local transcription failed: ${error.message}. If this happened after an interrupted download, remove the model cache at ${cacheHint} and try again.`);
+            const chunkHint = this.localBackend === 'onnx-asr-web' && /bad_alloc/i.test(error.message)
+                ? ` Parakeet local models should use short chunks; try chunkSize: ${PARAKEET_DEFAULT_CHUNK_SIZE} or less.`
+                : '';
+            throw new Error(`Local transcription failed: ${error.message}.${chunkHint} If this happened after an interrupted download, remove the model cache at ${cacheHint} and try again.`);
+        }
+    }
+
+    _resolveTranscriptionOptions(options = {}) {
+        return {
+            wordTimestamps: Boolean(options.wordTimestamps ?? this.wordTimestamps),
+        };
+    }
+
+    _formatApiTranscription(responseData, options) {
+        const text = this._formatText(responseData.text);
+        if (!options.wordTimestamps) {
+            return text;
+        }
+        return {
+            ...responseData,
+            text,
+            words: this._normalizeWords(responseData.words),
+        };
+    }
+
+    _formatLocalTranscription(text, words, options) {
+        const formattedText = this._formatText(text);
+        if (!options.wordTimestamps) {
+            return formattedText;
+        }
+        return {
+            text: formattedText,
+            words: this._normalizeWords(words),
+        };
+    }
+
+    _formatText(text) {
+        if (typeof text !== 'string') {
+            throw new Error('Transcription returned no text output.');
+        }
+        return text.trim();
+    }
+
+    _wordsFromTransformersChunks(chunks) {
+        if (!Array.isArray(chunks)) {
+            return undefined;
+        }
+        return chunks.map((chunk) => ({
+            word: chunk.text,
+            timestamp: chunk.timestamp,
+        }));
+    }
+
+    _normalizeWords(words) {
+        if (!Array.isArray(words)) {
+            throw new Error('Word timestamps were requested, but the selected backend did not return word timestamps.');
+        }
+
+        return words
+            .map((word) => {
+                if (!word || typeof word !== 'object') {
+                    throw new Error('Word timestamp response contains an invalid word entry.');
+                }
+
+                const text = typeof word.word === 'string' ? word.word : word.text;
+                const start = typeof word.start === 'number' ? word.start : word.timestamp?.[0];
+                const end = typeof word.end === 'number' ? word.end : word.timestamp?.[1];
+
+                if (typeof text !== 'string' || typeof start !== 'number' || typeof end !== 'number') {
+                    throw new Error('Word timestamp response contains an invalid word entry.');
+                }
+
+                return {
+                    word: text.trim(),
+                    start,
+                    end,
+                };
+            })
+            .filter((word) => word.word.length > 0);
+    }
+
+    _mergeTranscriptions(transcriptions, options, duration) {
+        const text = transcriptions
+            .map((transcription) => this._transcriptionText(transcription))
+            .filter((chunkText) => chunkText.length > 0)
+            .join(' ')
+            .trim();
+
+        if (!options.wordTimestamps) {
+            return text;
+        }
+
+        const words = transcriptions.flatMap((transcription) => {
+            if (!Array.isArray(transcription.words)) {
+                throw new Error('Word timestamps were requested, but a chunk returned no word timestamps.');
+            }
+            return transcription.words;
+        });
+        const segments = transcriptions.flatMap((transcription) => (
+            Array.isArray(transcription.segments) ? transcription.segments : []
+        ));
+        const result = { text, words, duration };
+
+        if (segments.length > 0) {
+            result.segments = segments;
+        }
+
+        return result;
+    }
+
+    _transcriptionText(transcription) {
+        if (typeof transcription === 'string') {
+            return transcription.trim();
+        }
+        return this._formatText(transcription.text);
+    }
+
+    _offsetTranscription(transcription, offsetSeconds) {
+        if (typeof transcription === 'string' || offsetSeconds === 0) {
+            return transcription;
+        }
+
+        return {
+            ...transcription,
+            words: Array.isArray(transcription.words)
+                ? this._offsetTimedItems(transcription.words, offsetSeconds)
+                : transcription.words,
+            segments: Array.isArray(transcription.segments)
+                ? this._offsetTimedItems(transcription.segments, offsetSeconds)
+                : transcription.segments,
+        };
+    }
+
+    _offsetTimedItems(items, offsetSeconds) {
+        return items.map((item) => ({
+            ...item,
+            start: this._offsetTime(item.start, offsetSeconds),
+            end: this._offsetTime(item.end, offsetSeconds),
+        }));
+    }
+
+    _offsetTime(value, offsetSeconds) {
+        if (typeof value !== 'number') {
+            return value;
         }
+        return Number((value + offsetSeconds).toFixed(3));
     }
 
     async _getLocalTranscriber() {

package/package.json

@@ -1,62 +1,61 @@
 {
-    "name": "whispermix",
-    "description": "🎙️ WhisperMix is a versatile module for transcribing audio using OpenAI’s Whisper or Groq’s Whisper v3 model.",
-    "version": "1.5.2",
-    "type": "module",
-    "keywords": [
-        "whisper",
-        "openai",
-        "groq",
-        "transcription",
-        "speech-to-text",
-        "audio",
-        "voice",
-        "ai",
-        "machine-learning",
-        "nlp",
-        "natural-language-processing",
-        "audio-processing",
-        "voice-recognition",
-        "speech-recognition",
-        "api-wrapper",
-        "whisper-large-v3",
-        "whisper-1",
-        "streaming",
-        "file-processing",
-        "multilingual",
-        "bottleneck",
-        "chunk",
-        "clasen",
-        "local",
-        "xenova",
-        "parakeet",
-        "nvidia",
-        "onnx"
-    ],
-    "repository": {
-        "type": "git",
-        "url": "git+https://github.com/clasen/WhisperMix.git"
-    },
-    "main": "index.js",
-    "scripts": {
-        "test": "echo \"Error: no test specified\" && exit 1"
-    },
-    "author": "Martin Clasen",
-    "license": "MIT",
-    "bugs": {
-        "url": "https://github.com/clasen/WhisperMix/issues"
-    },
-    "dependencies": {
-        "@huggingface/transformers": "^4.2.0",
-        "audio-decode": "^2.2.3",
-        "bottleneck": "^2.19.5",
-        "fluent-ffmpeg": "^2.1.3",
-        "get-audio-duration": "^4.0.1",
-        "onnx-asr-web": "^0.1.3"
-    },
-    "devDependencies": {
-        "node-addon-api": "^8.7.0",
-        "node-gyp": "^12.3.0"
-    },
-    "packageManager": "pnpm@11.1.2"
-}
+  "name": "whispermix",
+  "description": "🎙️ WhisperMix is a versatile module for transcribing audio using OpenAI’s Whisper or Groq’s Whisper v3 model.",
+  "version": "1.6.4",
+  "type": "module",
+  "keywords": [
+    "whisper",
+    "openai",
+    "groq",
+    "transcription",
+    "speech-to-text",
+    "audio",
+    "voice",
+    "ai",
+    "machine-learning",
+    "nlp",
+    "natural-language-processing",
+    "audio-processing",
+    "voice-recognition",
+    "speech-recognition",
+    "api-wrapper",
+    "whisper-large-v3",
+    "whisper-1",
+    "streaming",
+    "file-processing",
+    "multilingual",
+    "bottleneck",
+    "chunk",
+    "clasen",
+    "local",
+    "xenova",
+    "parakeet",
+    "nvidia",
+    "onnx"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/clasen/WhisperMix.git"
+  },
+  "main": "index.js",
+  "author": "Martin Clasen",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/clasen/WhisperMix/issues"
+  },
+  "dependencies": {
+    "@huggingface/transformers": "^4.2.0",
+    "audio-decode": "^2.2.3",
+    "bottleneck": "^2.19.5",
+    "fluent-ffmpeg": "^2.1.3",
+    "get-audio-duration": "^4.0.1",
+    "onnx-asr-web": "^0.1.3"
+  },
+  "devDependencies": {
+    "node-addon-api": "^8.7.0",
+    "node-gyp": "^12.3.0"
+  },
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  }
+}

package/skills/sfxmix/SKILL.md

@@ -0,0 +1,58 @@
+---
+name: whispermix
+description: "Transcribe audio to text with WhisperMix in Node.js (OpenAI, Groq, Whisper local or Parakeet local)."
+---
+
+# WhisperMix
+
+WhisperMix transcribes audio to text with one API and multiple backends.
+
+Use this skill when the user wants speech-to-text from an audio file (`.mp3`, `.wav`, `.m4a`, `.ogg`, `.flac`, `.webm`) in a Node.js project.
+
+Do not use for TTS, translation, or live microphone streaming.
+
+## Choose a model (quick)
+
+- **Cloud, best quality/speed:** `groq/whisper-large-v3` or `openai/whisper-1`
+- **Local, fastest startup:** `efederici/parakeet-tdt-0.6b-v3-int4`
+- **Local, better language control:** `xenova/whisper-large-v3`
+
+Important limits:
+- Local models are file-only (`fromFile`), no `fromStream`.
+- API models need `OPENAI_API_KEY` or `GROQ_API_KEY`.
+- `ffmpeg` is required for long files.
+
+## Basic usage
+
+Install:
+
+```bash
+npm install whispermix
+```
+
+File transcription:
+
+```javascript
+import WhisperMix from 'whispermix';
+
+const w = new WhisperMix({ model: 'groq/whisper-large-v3' });
+const text = await w.fromFile('audio.mp3');
+console.log(text);
+```
+
+Word timestamps:
+
+```javascript
+import WhisperMix from 'whispermix';
+
+const w = new WhisperMix({ model: 'openai/whisper-1' });
+const result = await w.fromFile('audio.mp3', { wordTimestamps: true });
+console.log(result.text);
+console.log(result.words);
+```
+
+## Troubleshooting
+
+- `OPENAI_API_KEY` / `GROQ_API_KEY is not set`: set env var or use a local model.
+- `fromStream not supported`: use `fromFile` with local models.
+- `Cannot find ffmpeg`: install `ffmpeg` and retry.

package/SKILL.md

@@ -1,88 +0,0 @@
----
-name: whispermix
-description: Transcribe audio to text using WhisperMix, a Node.js wrapper around OpenAI Whisper, Groq Whisper Large v3, local Whisper (xenova) and local Parakeet TDT v3 models. Use when the user asks to "transcribe audio", "speech to text", "convert audio/voice to text", mentions an audio file (.mp3, .wav, .m4a, .ogg, .flac, .webm) to turn into text, or names any of these models/providers: Whisper, OpenAI Whisper, Groq Whisper, Whisper Large v3, Parakeet, NVIDIA Parakeet, xenova/whisper, onnx-asr. Also use when the user wants to pick between a cloud API and a local on-device transcription model in a Node.js project.
----
-
-# WhisperMix
-
-Single Node.js entry point for audio transcription. Picks one of four backends behind the same API: OpenAI Whisper, Groq Whisper Large v3, local Whisper (xenova), or local Parakeet TDT v3. Handles long files by chunking and rate-limits API calls automatically.
-
-Do NOT use for TTS, live microphone streaming, or translation. WhisperMix is one-way: a complete audio file/stream → text.
-
-## Model selection (decide first)
-
-| Need | Pick |
-|---|---|
-| Offline, lowest latency | `istupakov/parakeet-tdt-0.6b-v3` |
-| Offline, smallest footprint (~410 MB) | `efederici/parakeet-tdt-0.6b-v3-int4` |
-| Highest accuracy, cloud | `groq/whisper-large-v3` (fast) or `openai/whisper-1` |
-| Offline + per-language control | `xenova/whisper-large-v3` or `xenova/whisper-base` |
-| Node stream input (not a file) | API only: `openai/whisper-1` or `groq/whisper-large-v3` |
-
-Constraints to surface before coding:
-- Local models (`xenova/*`, `*/parakeet-tdt-0.6b-v3*`) accept **files only**, not streams.
-- `language` option applies only to local Whisper. Parakeet is multilingual and ignores it.
-- API models need `OPENAI_API_KEY` or `GROQ_API_KEY` in the environment.
-- First Parakeet run downloads weights to `~/.cache/whispermix/parakeet/<modelKey>/` — warn the user.
-- Requires `ffmpeg` on `PATH` for long-audio chunking (>15 min split automatically).
-
-## API
-
-```bash
-npm install whispermix
-```
-
-ESM only. If the consumer is CommonJS, use `const WhisperMix = (await import('whispermix')).default;`.
-
-```javascript
-import WhisperMix from 'whispermix';
-
-const w = new WhisperMix({ model: '<modelKey>' });
-const text = await w.fromFile('path/to/audio.mp3');
-// API models only:
-const text2 = await w.fromStream(fs.createReadStream('path/to/audio.mp3'));
-```
-
-Constructor options:
-- `model` (required) — see selection table.
-- `language` — local Whisper only, e.g. `'spanish'`. Default `'auto'`.
-- `chunkSize` — seconds per chunk for long audio. Default `890` (~14m50s).
-- `bottleneck` — Bottleneck config for API models. Defaults: `minTime: 3000`, `maxConcurrent: 1`, `reservoir: 18`, `reservoirRefreshAmount: 18`, `reservoirRefreshInterval: 60000`.
-- `showProgress` — boolean, prints chunk/decoding progress.
-
-## Examples
-
-Cheapest local:
-
-```javascript
-import WhisperMix from 'whispermix';
-const w = new WhisperMix({ model: 'efederici/parakeet-tdt-0.6b-v3-int4', showProgress: true });
-console.log(await w.fromFile('meeting.wav'));
-```
-
-Groq with custom rate limit:
-
-```javascript
-import WhisperMix from 'whispermix';
-const w = new WhisperMix({
-    model: 'groq/whisper-large-v3',
-    bottleneck: { minTime: 4000, maxConcurrent: 1 },
-});
-console.log(await w.fromFile('podcast.mp3'));
-```
-
-Local Whisper, fixed language:
-
-```javascript
-import WhisperMix from 'whispermix';
-const w = new WhisperMix({ model: 'xenova/whisper-large-v3', language: 'spanish' });
-console.log(await w.fromFile('entrevista.m4a'));
-```
-
-## Troubleshooting
-
-- **`OPENAI_API_KEY`/`GROQ_API_KEY is not set`** — export the key, or switch to a local model.
-- **`fromStream` not supported** — local models are file-only. Use `fromFile`, or switch to an API model.
-- **`Cannot find ffmpeg`** — install it (`brew install ffmpeg` / `apt install ffmpeg`).
-- **First Parakeet call hangs** — weights downloading; enable `showProgress: true`.
-- **`ERR_REQUIRE_ESM`** — WhisperMix is ESM-only; use dynamic `import()` from CommonJS.