whisper.rn 0.5.0-rc.1 → 0.5.0-rc.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/README.md +119 -50
  2. package/lib/commonjs/AudioSessionIos.js +2 -1
  3. package/lib/commonjs/AudioSessionIos.js.map +1 -1
  4. package/lib/commonjs/index.js +1 -0
  5. package/lib/commonjs/index.js.map +1 -1
  6. package/lib/commonjs/jest-mock.js +126 -0
  7. package/lib/commonjs/jest-mock.js.map +1 -0
  8. package/lib/commonjs/realtime-transcription/RealtimeTranscriber.js +831 -0
  9. package/lib/commonjs/realtime-transcription/RealtimeTranscriber.js.map +1 -0
  10. package/lib/commonjs/realtime-transcription/SliceManager.js +233 -0
  11. package/lib/commonjs/realtime-transcription/SliceManager.js.map +1 -0
  12. package/lib/commonjs/realtime-transcription/adapters/AudioPcmStreamAdapter.js +133 -0
  13. package/lib/commonjs/realtime-transcription/adapters/AudioPcmStreamAdapter.js.map +1 -0
  14. package/lib/commonjs/realtime-transcription/adapters/JestAudioStreamAdapter.js +201 -0
  15. package/lib/commonjs/realtime-transcription/adapters/JestAudioStreamAdapter.js.map +1 -0
  16. package/lib/commonjs/realtime-transcription/adapters/SimulateFileAudioStreamAdapter.js +309 -0
  17. package/lib/commonjs/realtime-transcription/adapters/SimulateFileAudioStreamAdapter.js.map +1 -0
  18. package/lib/commonjs/realtime-transcription/index.js +27 -0
  19. package/lib/commonjs/realtime-transcription/index.js.map +1 -0
  20. package/lib/commonjs/realtime-transcription/types.js +114 -0
  21. package/lib/commonjs/realtime-transcription/types.js.map +1 -0
  22. package/lib/commonjs/utils/WavFileReader.js +158 -0
  23. package/lib/commonjs/utils/WavFileReader.js.map +1 -0
  24. package/lib/commonjs/utils/WavFileWriter.js +181 -0
  25. package/lib/commonjs/utils/WavFileWriter.js.map +1 -0
  26. package/lib/commonjs/utils/common.js +25 -0
  27. package/lib/commonjs/utils/common.js.map +1 -0
  28. package/lib/module/AudioSessionIos.js +2 -1
  29. package/lib/module/AudioSessionIos.js.map +1 -1
  30. package/lib/module/index.js +1 -0
  31. package/lib/module/index.js.map +1 -1
  32. package/lib/module/jest-mock.js +124 -0
  33. package/lib/module/jest-mock.js.map +1 -0
  34. package/lib/module/realtime-transcription/RealtimeTranscriber.js +825 -0
  35. package/lib/module/realtime-transcription/RealtimeTranscriber.js.map +1 -0
  36. package/lib/module/realtime-transcription/SliceManager.js +226 -0
  37. package/lib/module/realtime-transcription/SliceManager.js.map +1 -0
  38. package/lib/module/realtime-transcription/adapters/AudioPcmStreamAdapter.js +124 -0
  39. package/lib/module/realtime-transcription/adapters/AudioPcmStreamAdapter.js.map +1 -0
  40. package/lib/module/realtime-transcription/adapters/JestAudioStreamAdapter.js +194 -0
  41. package/lib/module/realtime-transcription/adapters/JestAudioStreamAdapter.js.map +1 -0
  42. package/lib/module/realtime-transcription/adapters/SimulateFileAudioStreamAdapter.js +302 -0
  43. package/lib/module/realtime-transcription/adapters/SimulateFileAudioStreamAdapter.js.map +1 -0
  44. package/lib/module/realtime-transcription/index.js +8 -0
  45. package/lib/module/realtime-transcription/index.js.map +1 -0
  46. package/lib/module/realtime-transcription/types.js +107 -0
  47. package/lib/module/realtime-transcription/types.js.map +1 -0
  48. package/lib/module/utils/WavFileReader.js +151 -0
  49. package/lib/module/utils/WavFileReader.js.map +1 -0
  50. package/lib/module/utils/WavFileWriter.js +174 -0
  51. package/lib/module/utils/WavFileWriter.js.map +1 -0
  52. package/lib/module/utils/common.js +18 -0
  53. package/lib/module/utils/common.js.map +1 -0
  54. package/lib/typescript/AudioSessionIos.d.ts +1 -1
  55. package/lib/typescript/AudioSessionIos.d.ts.map +1 -1
  56. package/lib/typescript/index.d.ts.map +1 -1
  57. package/lib/typescript/jest-mock.d.ts +2 -0
  58. package/lib/typescript/jest-mock.d.ts.map +1 -0
  59. package/lib/typescript/realtime-transcription/RealtimeTranscriber.d.ts +165 -0
  60. package/lib/typescript/realtime-transcription/RealtimeTranscriber.d.ts.map +1 -0
  61. package/lib/typescript/realtime-transcription/SliceManager.d.ts +72 -0
  62. package/lib/typescript/realtime-transcription/SliceManager.d.ts.map +1 -0
  63. package/lib/typescript/realtime-transcription/adapters/AudioPcmStreamAdapter.d.ts +22 -0
  64. package/lib/typescript/realtime-transcription/adapters/AudioPcmStreamAdapter.d.ts.map +1 -0
  65. package/lib/typescript/realtime-transcription/adapters/JestAudioStreamAdapter.d.ts +44 -0
  66. package/lib/typescript/realtime-transcription/adapters/JestAudioStreamAdapter.d.ts.map +1 -0
  67. package/lib/typescript/realtime-transcription/adapters/SimulateFileAudioStreamAdapter.d.ts +75 -0
  68. package/lib/typescript/realtime-transcription/adapters/SimulateFileAudioStreamAdapter.d.ts.map +1 -0
  69. package/lib/typescript/realtime-transcription/index.d.ts +6 -0
  70. package/lib/typescript/realtime-transcription/index.d.ts.map +1 -0
  71. package/lib/typescript/realtime-transcription/types.d.ts +216 -0
  72. package/lib/typescript/realtime-transcription/types.d.ts.map +1 -0
  73. package/lib/typescript/utils/WavFileReader.d.ts +61 -0
  74. package/lib/typescript/utils/WavFileReader.d.ts.map +1 -0
  75. package/lib/typescript/utils/WavFileWriter.d.ts +57 -0
  76. package/lib/typescript/utils/WavFileWriter.d.ts.map +1 -0
  77. package/lib/typescript/utils/common.d.ts +9 -0
  78. package/lib/typescript/utils/common.d.ts.map +1 -0
  79. package/package.json +18 -6
  80. package/src/AudioSessionIos.ts +3 -2
  81. package/src/index.ts +4 -0
  82. package/{jest/mock.js → src/jest-mock.ts} +2 -2
  83. package/src/realtime-transcription/RealtimeTranscriber.ts +983 -0
  84. package/src/realtime-transcription/SliceManager.ts +252 -0
  85. package/src/realtime-transcription/adapters/AudioPcmStreamAdapter.ts +143 -0
  86. package/src/realtime-transcription/adapters/JestAudioStreamAdapter.ts +251 -0
  87. package/src/realtime-transcription/adapters/SimulateFileAudioStreamAdapter.ts +378 -0
  88. package/src/realtime-transcription/index.ts +34 -0
  89. package/src/realtime-transcription/types.ts +277 -0
  90. package/src/utils/WavFileReader.ts +202 -0
  91. package/src/utils/WavFileWriter.ts +206 -0
  92. package/src/utils/common.ts +17 -0
@@ -0,0 +1,831 @@
1
+ "use strict";
2
+
3
+ Object.defineProperty(exports, "__esModule", {
4
+ value: true
5
+ });
6
+ exports.RealtimeTranscriber = void 0;
7
+ var _SliceManager = require("./SliceManager");
8
+ var _WavFileWriter = require("../utils/WavFileWriter");
9
+ var _types = require("./types");
10
+ /* eslint-disable class-methods-use-this */
11
+
12
+ /**
13
+ * RealtimeTranscriber provides real-time audio transcription with VAD support.
14
+ *
15
+ * Features:
16
+ * - Automatic slice management based on duration
17
+ * - VAD-based speech detection and auto-slicing
18
+ * - Configurable auto-slice mechanism that triggers on speech_end/silence events
19
+ * - Memory management for audio slices
20
+ * - Queue-based transcription processing
21
+ */
22
+ class RealtimeTranscriber {
23
+ callbacks = {};
24
+ isActive = false;
25
+ isTranscribing = false;
26
+ vadEnabled = false;
27
+ transcriptionQueue = [];
28
+ accumulatedData = new Uint8Array(0);
29
+ wavFileWriter = null;
30
+
31
+ // Simplified VAD state management
32
+ lastSpeechDetectedTime = 0;
33
+
34
+ // Track VAD state for proper event transitions
35
+ lastVadState = 'silence';
36
+
37
+ // Track last stats to emit only when changed
38
+ lastStatsSnapshot = null;
39
+
40
+ // Store transcription results by slice index
41
+ transcriptionResults = new Map();
42
+ constructor(dependencies) {
43
+ let options = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {};
44
+ let callbacks = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : {};
45
+ this.whisperContext = dependencies.whisperContext;
46
+ this.vadContext = dependencies.vadContext;
47
+ this.audioStream = dependencies.audioStream;
48
+ this.fs = dependencies.fs;
49
+ this.callbacks = callbacks;
50
+
51
+ // Set default options with proper types
52
+ this.options = {
53
+ audioSliceSec: options.audioSliceSec || 30,
54
+ audioMinSec: options.audioMinSec || 1,
55
+ maxSlicesInMemory: options.maxSlicesInMemory || 3,
56
+ vadOptions: options.vadOptions || _types.VAD_PRESETS.default,
57
+ vadPreset: options.vadPreset,
58
+ autoSliceOnSpeechEnd: options.autoSliceOnSpeechEnd || true,
59
+ autoSliceThreshold: options.autoSliceThreshold || 0.5,
60
+ transcribeOptions: options.transcribeOptions || {},
61
+ initialPrompt: options.initialPrompt,
62
+ promptPreviousSlices: options.promptPreviousSlices ?? true,
63
+ audioOutputPath: options.audioOutputPath,
64
+ logger: options.logger || (() => {})
65
+ };
66
+
67
+ // Apply VAD preset if specified
68
+ if (this.options.vadPreset && _types.VAD_PRESETS[this.options.vadPreset]) {
69
+ this.options.vadOptions = {
70
+ ..._types.VAD_PRESETS[this.options.vadPreset],
71
+ ...this.options.vadOptions
72
+ };
73
+ }
74
+
75
+ // Enable VAD if context is provided and not explicitly disabled
76
+ this.vadEnabled = !!this.vadContext;
77
+
78
+ // Initialize managers
79
+ this.sliceManager = new _SliceManager.SliceManager(this.options.audioSliceSec, this.options.maxSlicesInMemory);
80
+
81
+ // Set up audio stream callbacks
82
+ this.audioStream.onData(this.handleAudioData.bind(this));
83
+ this.audioStream.onError(this.handleError.bind(this));
84
+ this.audioStream.onStatusChange(this.handleAudioStatusChange.bind(this));
85
+ }
86
+
87
+ /**
88
+ * Start realtime transcription
89
+ */
90
+ async start() {
91
+ if (this.isActive) {
92
+ throw new Error('Realtime transcription is already active');
93
+ }
94
+ try {
95
+ var _this$callbacks$onSta, _this$callbacks, _this$options$audioSt4, _this$options$audioSt5, _this$options$audioSt6, _this$options$audioSt7, _this$options$audioSt8;
96
+ this.isActive = true;
97
+ (_this$callbacks$onSta = (_this$callbacks = this.callbacks).onStatusChange) === null || _this$callbacks$onSta === void 0 ? void 0 : _this$callbacks$onSta.call(_this$callbacks, true);
98
+
99
+ // Reset all state to ensure clean start
100
+ this.reset();
101
+
102
+ // Initialize WAV file writer if output path is specified
103
+ if (this.fs && this.options.audioOutputPath) {
104
+ var _this$options$audioSt, _this$options$audioSt2, _this$options$audioSt3;
105
+ this.wavFileWriter = new _WavFileWriter.WavFileWriter(this.fs, this.options.audioOutputPath, {
106
+ sampleRate: ((_this$options$audioSt = this.options.audioStreamConfig) === null || _this$options$audioSt === void 0 ? void 0 : _this$options$audioSt.sampleRate) || 16000,
107
+ channels: ((_this$options$audioSt2 = this.options.audioStreamConfig) === null || _this$options$audioSt2 === void 0 ? void 0 : _this$options$audioSt2.channels) || 1,
108
+ bitsPerSample: ((_this$options$audioSt3 = this.options.audioStreamConfig) === null || _this$options$audioSt3 === void 0 ? void 0 : _this$options$audioSt3.bitsPerSample) || 16
109
+ });
110
+ await this.wavFileWriter.initialize();
111
+ }
112
+
113
+ // Start audio recording
114
+ await this.audioStream.initialize({
115
+ sampleRate: ((_this$options$audioSt4 = this.options.audioStreamConfig) === null || _this$options$audioSt4 === void 0 ? void 0 : _this$options$audioSt4.sampleRate) || 16000,
116
+ channels: ((_this$options$audioSt5 = this.options.audioStreamConfig) === null || _this$options$audioSt5 === void 0 ? void 0 : _this$options$audioSt5.channels) || 1,
117
+ bitsPerSample: ((_this$options$audioSt6 = this.options.audioStreamConfig) === null || _this$options$audioSt6 === void 0 ? void 0 : _this$options$audioSt6.bitsPerSample) || 16,
118
+ audioSource: ((_this$options$audioSt7 = this.options.audioStreamConfig) === null || _this$options$audioSt7 === void 0 ? void 0 : _this$options$audioSt7.audioSource) || 6,
119
+ bufferSize: ((_this$options$audioSt8 = this.options.audioStreamConfig) === null || _this$options$audioSt8 === void 0 ? void 0 : _this$options$audioSt8.bufferSize) || 16 * 1024
120
+ });
121
+ await this.audioStream.start();
122
+
123
+ // Emit stats update for status change
124
+ this.emitStatsUpdate('status_change');
125
+ this.log('Realtime transcription started');
126
+ } catch (error) {
127
+ var _this$callbacks$onSta2, _this$callbacks2;
128
+ this.isActive = false;
129
+ (_this$callbacks$onSta2 = (_this$callbacks2 = this.callbacks).onStatusChange) === null || _this$callbacks$onSta2 === void 0 ? void 0 : _this$callbacks$onSta2.call(_this$callbacks2, false);
130
+ throw error;
131
+ }
132
+ }
133
+
134
+ /**
135
+ * Stop realtime transcription
136
+ */
137
+ async stop() {
138
+ if (!this.isActive) {
139
+ return;
140
+ }
141
+ try {
142
+ var _this$callbacks$onSta3, _this$callbacks3;
143
+ this.isActive = false;
144
+
145
+ // Stop audio recording
146
+ await this.audioStream.stop();
147
+
148
+ // Process any remaining accumulated data
149
+ if (this.accumulatedData.length > 0) {
150
+ this.processAccumulatedDataForSliceManagement();
151
+ }
152
+
153
+ // Process any remaining queued transcriptions
154
+ await this.processTranscriptionQueue();
155
+
156
+ // Finalize WAV file
157
+ if (this.wavFileWriter) {
158
+ await this.wavFileWriter.finalize();
159
+ this.wavFileWriter = null;
160
+ }
161
+
162
+ // Reset all state completely
163
+ this.reset();
164
+ (_this$callbacks$onSta3 = (_this$callbacks3 = this.callbacks).onStatusChange) === null || _this$callbacks$onSta3 === void 0 ? void 0 : _this$callbacks$onSta3.call(_this$callbacks3, false);
165
+
166
+ // Emit stats update for status change
167
+ this.emitStatsUpdate('status_change');
168
+ this.log('Realtime transcription stopped');
169
+ } catch (error) {
170
+ this.handleError(`Stop error: ${error}`);
171
+ }
172
+ }
173
+
174
+ /**
175
+ * Handle incoming audio data from audio stream
176
+ */
177
+ handleAudioData(streamData) {
178
+ if (!this.isActive) {
179
+ return;
180
+ }
181
+ try {
182
+ // Write to WAV file if enabled (convert to Uint8Array for WavFileWriter)
183
+ if (this.wavFileWriter) {
184
+ this.wavFileWriter.appendAudioData(streamData.data).catch(error => {
185
+ this.log(`Failed to write audio to WAV file: ${error}`);
186
+ });
187
+ }
188
+
189
+ // Always accumulate data for slice management
190
+ this.accumulateAudioData(streamData.data);
191
+ } catch (error) {
192
+ const errorMessage = error instanceof Error ? error.message : 'Audio processing error';
193
+ this.handleError(errorMessage);
194
+ }
195
+ }
196
+
197
+ /**
198
+ * Accumulate audio data for slice management
199
+ */
200
+ accumulateAudioData(newData) {
201
+ const combined = new Uint8Array(this.accumulatedData.length + newData.length);
202
+ combined.set(this.accumulatedData);
203
+ combined.set(new Uint8Array(newData), this.accumulatedData.length);
204
+ this.accumulatedData = combined;
205
+
206
+ // Process accumulated data when we have enough for slice management
207
+ const minBufferSamples = 16000 * 1; // 1 second for slice management
208
+ if (this.accumulatedData.length >= minBufferSamples) {
209
+ this.processAccumulatedDataForSliceManagement();
210
+ }
211
+ }
212
+
213
+ /**
214
+ * Process accumulated audio data through SliceManager
215
+ */
216
+ processAccumulatedDataForSliceManagement() {
217
+ if (this.accumulatedData.length === 0) {
218
+ return;
219
+ }
220
+
221
+ // Process through slice manager directly with Uint8Array
222
+ const result = this.sliceManager.addAudioData(this.accumulatedData);
223
+ if (result.slice) {
224
+ this.log(`Slice ${result.slice.index} ready (${result.slice.data.length} bytes)`);
225
+
226
+ // Process VAD for the slice if enabled
227
+ if (!this.isTranscribing && this.vadEnabled) {
228
+ this.processSliceVAD(result.slice).catch(error => {
229
+ this.handleError(`VAD processing error: ${error}`);
230
+ });
231
+ } else if (!this.isTranscribing) {
232
+ // If VAD is disabled, transcribe slices as they become ready
233
+ this.queueSliceForTranscription(result.slice).catch(error => {
234
+ this.handleError(`Failed to queue slice for transcription: ${error}`);
235
+ });
236
+ } else {
237
+ this.log(`Skipping slice ${result.slice.index} - already transcribing`);
238
+ }
239
+ this.emitStatsUpdate('memory_change');
240
+ }
241
+
242
+ // Clear accumulated data
243
+ this.accumulatedData = new Uint8Array(0);
244
+ }
245
+
246
+ /**
247
+ * Check if auto-slice should be triggered based on VAD event and timing
248
+ */
249
+ async checkAutoSlice(vadEvent, _slice) {
250
+ if (!this.options.autoSliceOnSpeechEnd || !this.vadEnabled) {
251
+ return;
252
+ }
253
+
254
+ // Only trigger on speech_end or silence events
255
+ const shouldTriggerAutoSlice = vadEvent.type === 'speech_end' || vadEvent.type === 'silence';
256
+ if (!shouldTriggerAutoSlice) {
257
+ return;
258
+ }
259
+
260
+ // Get current slice info from SliceManager
261
+ const currentSliceInfo = this.sliceManager.getCurrentSliceInfo();
262
+ const currentSlice = this.sliceManager.getSliceByIndex(currentSliceInfo.currentSliceIndex);
263
+ if (!currentSlice) {
264
+ return;
265
+ }
266
+
267
+ // Calculate current slice duration
268
+ const currentDuration = (Date.now() - currentSlice.startTime) / 1000; // Convert to seconds
269
+ const targetDuration = this.options.audioSliceSec;
270
+ const minDuration = this.options.audioMinSec;
271
+ const autoSliceThreshold = targetDuration * this.options.autoSliceThreshold;
272
+
273
+ // Check if conditions are met for auto-slice
274
+ const meetsMinDuration = currentDuration >= minDuration;
275
+ const meetsThreshold = currentDuration >= autoSliceThreshold;
276
+ if (meetsMinDuration && meetsThreshold) {
277
+ this.log(`Auto-slicing on ${vadEvent.type} at ${currentDuration.toFixed(1)}s ` + `(min: ${minDuration}s, threshold: ${autoSliceThreshold.toFixed(1)}s, target: ${targetDuration}s)`);
278
+
279
+ // Force next slice
280
+ await this.nextSlice();
281
+ } else {
282
+ this.log(`Auto-slice conditions not met on ${vadEvent.type}: ` + `duration=${currentDuration.toFixed(1)}s, min=${minDuration}s, threshold=${autoSliceThreshold.toFixed(1)}s ` + `(minOk=${meetsMinDuration}, thresholdOk=${meetsThreshold})`);
283
+ }
284
+ }
285
+
286
+ /**
287
+ * Process VAD for a completed slice
288
+ */
289
+ async processSliceVAD(slice) {
290
+ try {
291
+ var _this$callbacks$onVad, _this$callbacks4;
292
+ // Get audio data from the slice for VAD processing
293
+ const audioData = this.sliceManager.getAudioDataForTranscription(slice.index);
294
+ if (!audioData) {
295
+ this.log(`No audio data available for VAD processing of slice ${slice.index}`);
296
+ return;
297
+ }
298
+
299
+ // Convert base64 back to Uint8Array for VAD processing
300
+
301
+ // Detect speech in the slice
302
+ const vadEvent = await this.detectSpeech(audioData, slice.index);
303
+ vadEvent.timestamp = Date.now();
304
+
305
+ // Emit VAD event
306
+ (_this$callbacks$onVad = (_this$callbacks4 = this.callbacks).onVad) === null || _this$callbacks$onVad === void 0 ? void 0 : _this$callbacks$onVad.call(_this$callbacks4, vadEvent);
307
+
308
+ // Check if auto-slice should be triggered
309
+ await this.checkAutoSlice(vadEvent, slice);
310
+
311
+ // Check if speech was detected and if we should transcribe
312
+ const isSpeech = vadEvent.type === 'speech_start' || vadEvent.type === 'speech_continue';
313
+ const isSpeechEnd = vadEvent.type === 'speech_end';
314
+ if (isSpeech) {
315
+ const minDuration = this.options.audioMinSec;
316
+ // Check minimum duration requirement
317
+ const speechDuration = slice.data.length / 16000 / 2; // Convert bytes to seconds (16kHz, 16-bit)
318
+
319
+ if (speechDuration >= minDuration) {
320
+ this.log(`Speech detected in slice ${slice.index}, queueing for transcription`);
321
+ await this.queueSliceForTranscription(slice);
322
+ } else {
323
+ this.log(`Speech too short in slice ${slice.index} (${speechDuration.toFixed(2)}s < ${minDuration}s), skipping`);
324
+ }
325
+ } else if (isSpeechEnd) {
326
+ this.log(`Speech ended in slice ${slice.index}`);
327
+ // For speech_end events, we might want to queue the slice for transcription
328
+ // to capture the final part of the speech segment
329
+ const speechDuration = slice.data.length / 16000 / 2; // Convert bytes to seconds
330
+ const minDuration = this.options.audioMinSec;
331
+ if (speechDuration >= minDuration) {
332
+ this.log(`Speech end detected in slice ${slice.index}, queueing final segment for transcription`);
333
+ await this.queueSliceForTranscription(slice);
334
+ } else {
335
+ this.log(`Speech end segment too short in slice ${slice.index} (${speechDuration.toFixed(2)}s < ${minDuration}s), skipping`);
336
+ }
337
+ } else {
338
+ this.log(`No speech detected in slice ${slice.index}`);
339
+ }
340
+
341
+ // Emit stats update for VAD change
342
+ this.emitStatsUpdate('vad_change');
343
+ } catch (error) {
344
+ this.handleError(`VAD processing error for slice ${slice.index}: ${error}`);
345
+ }
346
+ }
347
+
348
+ /**
349
+ * Queue a slice for transcription
350
+ */
351
+ async queueSliceForTranscription(slice) {
352
+ try {
353
+ // Get audio data from the slice
354
+ const audioData = this.sliceManager.getAudioDataForTranscription(slice.index);
355
+ if (!audioData) {
356
+ this.log(`No audio data available for slice ${slice.index}`);
357
+ return;
358
+ }
359
+
360
+ // Add to transcription queue
361
+ this.transcriptionQueue.unshift({
362
+ sliceIndex: slice.index,
363
+ audioData
364
+ });
365
+ this.log(`Queued slice ${slice.index} for transcription (${slice.data.length} samples)`);
366
+ await this.processTranscriptionQueue();
367
+ } catch (error) {
368
+ this.handleError(`Failed to queue slice for transcription: ${error}`);
369
+ }
370
+ }
371
+
372
+ /**
373
+ * Detect speech using VAD context
374
+ */
375
+ async detectSpeech(audioData, sliceIndex) {
376
+ if (!this.vadContext) {
377
+ // When no VAD context is available, assume speech is always detected
378
+ // but still follow the state machine pattern
379
+ const currentTimestamp = Date.now();
380
+
381
+ // Assume speech is always detected when no VAD context
382
+ const vadEventType = this.lastVadState === 'silence' ? 'speech_start' : 'speech_continue';
383
+
384
+ // Update VAD state
385
+ this.lastVadState = 'speech';
386
+ const {
387
+ sampleRate = 16000
388
+ } = this.options.audioStreamConfig || {};
389
+ return {
390
+ type: vadEventType,
391
+ lastSpeechDetectedTime: 0,
392
+ timestamp: currentTimestamp,
393
+ confidence: 1.0,
394
+ duration: audioData.length / sampleRate / 2,
395
+ // Convert bytes to seconds
396
+ sliceIndex
397
+ };
398
+ }
399
+ try {
400
+ const audioBuffer = audioData.buffer;
401
+
402
+ // Use VAD context to detect speech segments
403
+ const vadSegments = await this.vadContext.detectSpeechData(audioBuffer, this.options.vadOptions);
404
+
405
+ // Calculate confidence based on speech segments
406
+ let confidence = 0.0;
407
+ let lastSpeechDetectedTime = 0;
408
+ if (vadSegments && vadSegments.length > 0) {
409
+ var _vadSegments;
410
+ // If there are speech segments, calculate average confidence
411
+ const totalTime = vadSegments.reduce((sum, segment) => sum + (segment.t1 - segment.t0), 0);
412
+ const audioDuration = audioData.length / 16000 / 2; // Convert bytes to seconds
413
+ confidence = totalTime > 0 ? Math.min(totalTime / audioDuration, 1.0) : 0.0;
414
+ lastSpeechDetectedTime = ((_vadSegments = vadSegments[vadSegments.length - 1]) === null || _vadSegments === void 0 ? void 0 : _vadSegments.t1) || -1;
415
+ }
416
+ const threshold = this.options.vadOptions.threshold || 0.5;
417
+ let isSpeech = confidence > threshold;
418
+ const currentTimestamp = Date.now();
419
+
420
+ // Determine VAD event type based on current and previous state
421
+ let vadEventType;
422
+ if (isSpeech) {
423
+ vadEventType = this.lastVadState === 'silence' ? 'speech_start' : 'speech_continue';
424
+ const minDuration = this.options.audioMinSec;
425
+ // Check if this is a new speech detection (different from last detected time)
426
+ if (lastSpeechDetectedTime === this.lastSpeechDetectedTime || (lastSpeechDetectedTime - this.lastSpeechDetectedTime) / 100 < minDuration) {
427
+ if (this.lastVadState === 'silence') vadEventType = 'silence';
428
+ if (this.lastVadState === 'speech') vadEventType = 'speech_end';
429
+ isSpeech = false;
430
+ confidence = 0.0;
431
+ }
432
+ this.lastSpeechDetectedTime = lastSpeechDetectedTime;
433
+ } else {
434
+ vadEventType = this.lastVadState === 'speech' ? 'speech_end' : 'silence';
435
+ }
436
+
437
+ // Update VAD state for next detection
438
+ this.lastVadState = isSpeech ? 'speech' : 'silence';
439
+ const {
440
+ sampleRate = 16000
441
+ } = this.options.audioStreamConfig || {};
442
+ return {
443
+ type: vadEventType,
444
+ lastSpeechDetectedTime,
445
+ timestamp: currentTimestamp,
446
+ confidence,
447
+ duration: audioData.length / sampleRate / 2,
448
+ // Convert bytes to seconds
449
+ sliceIndex,
450
+ currentThreshold: threshold
451
+ };
452
+ } catch (error) {
453
+ this.log(`VAD detection error: ${error}`);
454
+ // Re-throw the error so it can be handled by the caller
455
+ throw error;
456
+ }
457
+ }
458
+ isProcessingTranscriptionQueue = false;
459
+
460
+ /**
461
+ * Process the transcription queue
462
+ */
463
+ async processTranscriptionQueue() {
464
+ if (this.isProcessingTranscriptionQueue) return;
465
+ this.isProcessingTranscriptionQueue = true;
466
+ while (this.transcriptionQueue.length > 0) {
467
+ const item = this.transcriptionQueue.shift();
468
+ this.transcriptionQueue = []; // Old items are not needed anymore
469
+ if (item) {
470
+ // eslint-disable-next-line no-await-in-loop
471
+ await this.processTranscription(item).catch(error => {
472
+ this.handleError(`Transcription error: ${error}`);
473
+ });
474
+ }
475
+ }
476
+ this.isProcessingTranscriptionQueue = false;
477
+ }
478
+
479
+ /**
480
+ * Build prompt from initial prompt and previous slices
481
+ */
482
+ buildPrompt(currentSliceIndex) {
483
+ const promptParts = [];
484
+
485
+ // Add initial prompt if provided
486
+ if (this.options.initialPrompt) {
487
+ promptParts.push(this.options.initialPrompt);
488
+ }
489
+
490
+ // Add previous slice results if enabled
491
+ if (this.options.promptPreviousSlices) {
492
+ // Get transcription results from previous slices (up to the current slice)
493
+ const previousResults = Array.from(this.transcriptionResults.entries()).filter(_ref => {
494
+ let [sliceIndex] = _ref;
495
+ return sliceIndex < currentSliceIndex;
496
+ }).sort((_ref2, _ref3) => {
497
+ let [a] = _ref2;
498
+ let [b] = _ref3;
499
+ return a - b;
500
+ }) // Sort by slice index
501
+ .map(_ref4 => {
502
+ var _result$transcribeEve;
503
+ let [, result] = _ref4;
504
+ return (_result$transcribeEve = result.transcribeEvent.data) === null || _result$transcribeEve === void 0 ? void 0 : _result$transcribeEve.result;
505
+ }).filter(result => Boolean(result)); // Filter out empty results with type guard
506
+
507
+ if (previousResults.length > 0) {
508
+ promptParts.push(...previousResults);
509
+ }
510
+ }
511
+ return promptParts.join(' ') || undefined;
512
+ }
513
+
514
+ /**
515
+ * Process a single transcription
516
+ */
517
+ async processTranscription(item) {
518
+ if (!this.isActive) {
519
+ return;
520
+ }
521
+ this.isTranscribing = true;
522
+
523
+ // Emit stats update for status change
524
+ this.emitStatsUpdate('status_change');
525
+ const startTime = Date.now();
526
+ try {
527
+ var _this$callbacks$onTra, _this$callbacks5;
528
+ // Build prompt from initial prompt and previous slices
529
+ const prompt = this.buildPrompt(item.sliceIndex);
530
+ const audioBuffer = item.audioData.buffer;
531
+ const {
532
+ promise
533
+ } = this.whisperContext.transcribeData(audioBuffer, {
534
+ ...this.options.transcribeOptions,
535
+ prompt,
536
+ // Include the constructed prompt
537
+ onProgress: undefined // Disable progress for realtime
538
+ });
539
+
540
+ const result = await promise;
541
+ const endTime = Date.now();
542
+
543
+ // Create transcribe event
544
+ const {
545
+ sampleRate = 16000
546
+ } = this.options.audioStreamConfig || {};
547
+ const transcribeEvent = {
548
+ type: 'transcribe',
549
+ sliceIndex: item.sliceIndex,
550
+ data: result,
551
+ isCapturing: this.audioStream.isRecording(),
552
+ processTime: endTime - startTime,
553
+ recordingTime: item.audioData.length / (sampleRate / 1000) / 2,
554
+ // ms,
555
+ memoryUsage: this.sliceManager.getMemoryUsage()
556
+ };
557
+
558
+ // Emit transcribe event
559
+ (_this$callbacks$onTra = (_this$callbacks5 = this.callbacks).onTranscribe) === null || _this$callbacks$onTra === void 0 ? void 0 : _this$callbacks$onTra.call(_this$callbacks5, transcribeEvent);
560
+
561
+ // Save transcription results
562
+ const slice = this.sliceManager.getSliceByIndex(item.sliceIndex);
563
+ if (slice) {
564
+ this.transcriptionResults.set(item.sliceIndex, {
565
+ slice: {
566
+ // Don't keep data in the slice
567
+ index: slice.index,
568
+ sampleCount: slice.sampleCount,
569
+ startTime: slice.startTime,
570
+ endTime: slice.endTime,
571
+ isProcessed: slice.isProcessed,
572
+ isReleased: slice.isReleased
573
+ },
574
+ transcribeEvent
575
+ });
576
+ }
577
+
578
+ // Emit stats update for memory/slice changes
579
+ this.emitStatsUpdate('memory_change');
580
+ this.log(`Transcribed speech segment ${item.sliceIndex}: "${result.result}"`);
581
+ } catch (error) {
582
+ var _this$callbacks$onTra2, _this$callbacks6;
583
+ // Emit error event to transcribe callback
584
+ const errorEvent = {
585
+ type: 'error',
586
+ sliceIndex: item.sliceIndex,
587
+ data: undefined,
588
+ isCapturing: this.audioStream.isRecording(),
589
+ processTime: Date.now() - startTime,
590
+ recordingTime: 0,
591
+ memoryUsage: this.sliceManager.getMemoryUsage()
592
+ };
593
+ (_this$callbacks$onTra2 = (_this$callbacks6 = this.callbacks).onTranscribe) === null || _this$callbacks$onTra2 === void 0 ? void 0 : _this$callbacks$onTra2.call(_this$callbacks6, errorEvent);
594
+ this.handleError(`Transcription failed for speech segment ${item.sliceIndex}: ${error}`);
595
+ } finally {
596
+ // Check if we should continue processing queue
597
+ if (this.transcriptionQueue.length > 0) {
598
+ await this.processTranscriptionQueue();
599
+ } else {
600
+ this.isTranscribing = false;
601
+ }
602
+ }
603
+ }
604
+
605
+ /**
606
+ * Handle audio status changes
607
+ */
608
+ handleAudioStatusChange(isRecording) {
609
+ this.log(`Audio recording: ${isRecording ? 'started' : 'stopped'}`);
610
+ }
611
+
612
+ /**
613
+ * Handle errors from components
614
+ */
615
+ handleError(error) {
616
+ var _this$callbacks$onErr, _this$callbacks7;
617
+ this.log(`Error: ${error}`);
618
+ (_this$callbacks$onErr = (_this$callbacks7 = this.callbacks).onError) === null || _this$callbacks$onErr === void 0 ? void 0 : _this$callbacks$onErr.call(_this$callbacks7, error);
619
+ }
620
+
621
+ /**
622
+ * Update callbacks
623
+ */
624
+ updateCallbacks(callbacks) {
625
+ this.callbacks = {
626
+ ...this.callbacks,
627
+ ...callbacks
628
+ };
629
+ }
630
+
631
+ /**
632
+ * Update VAD options dynamically
633
+ */
634
+ updateVadOptions(options) {
635
+ this.options.vadOptions = {
636
+ ...this.options.vadOptions,
637
+ ...options
638
+ };
639
+ }
640
+
641
+ /**
642
+ * Update auto-slice options dynamically
643
+ */
644
+ updateAutoSliceOptions(options) {
645
+ if (options.autoSliceOnSpeechEnd !== undefined) {
646
+ this.options.autoSliceOnSpeechEnd = options.autoSliceOnSpeechEnd;
647
+ }
648
+ if (options.autoSliceThreshold !== undefined) {
649
+ this.options.autoSliceThreshold = options.autoSliceThreshold;
650
+ }
651
+ this.log(`Auto-slice options updated: enabled=${this.options.autoSliceOnSpeechEnd}, threshold=${this.options.autoSliceThreshold}`);
652
+ }
653
+
654
+ /**
655
+ * Get current statistics
656
+ */
657
+ getStatistics() {
658
+ return {
659
+ isActive: this.isActive,
660
+ isTranscribing: this.isTranscribing,
661
+ vadEnabled: this.vadEnabled,
662
+ audioStats: {
663
+ isRecording: this.audioStream.isRecording(),
664
+ accumulatedSamples: this.accumulatedData.length
665
+ },
666
+ vadStats: this.vadEnabled ? {
667
+ enabled: true,
668
+ contextAvailable: !!this.vadContext,
669
+ lastSpeechDetectedTime: this.lastSpeechDetectedTime
670
+ } : null,
671
+ sliceStats: this.sliceManager.getCurrentSliceInfo(),
672
+ autoSliceConfig: {
673
+ enabled: this.options.autoSliceOnSpeechEnd,
674
+ threshold: this.options.autoSliceThreshold,
675
+ targetDuration: this.options.audioSliceSec,
676
+ minDuration: this.options.audioMinSec
677
+ }
678
+ };
679
+ }
680
+
681
+ /**
682
+ * Get all transcription results
683
+ */
684
+ getTranscriptionResults() {
685
+ return Array.from(this.transcriptionResults.values());
686
+ }
687
+
688
+ /**
689
+ * Force move to the next slice, finalizing the current one regardless of capacity
690
+ */
691
+ async nextSlice() {
692
+ var _this$callbacks$onTra3, _this$callbacks8;
693
+ if (!this.isActive) {
694
+ this.log('Cannot force next slice - transcriber is not active');
695
+ return;
696
+ }
697
+
698
+ // Emit start event to indicate slice processing has started
699
+ const startEvent = {
700
+ type: 'start',
701
+ sliceIndex: -1,
702
+ // Use -1 to indicate forced slice
703
+ data: undefined,
704
+ isCapturing: this.audioStream.isRecording(),
705
+ processTime: 0,
706
+ recordingTime: 0,
707
+ memoryUsage: this.sliceManager.getMemoryUsage()
708
+ };
709
+ (_this$callbacks$onTra3 = (_this$callbacks8 = this.callbacks).onTranscribe) === null || _this$callbacks$onTra3 === void 0 ? void 0 : _this$callbacks$onTra3.call(_this$callbacks8, startEvent);
710
+
711
+ // Check if there are pending transcriptions or currently transcribing
712
+ if (this.isTranscribing || this.transcriptionQueue.length > 0) {
713
+ this.log('Waiting for pending transcriptions to complete before forcing next slice...');
714
+
715
+ // Wait for current transcription queue to be processed
716
+ await this.processTranscriptionQueue();
717
+ }
718
+ const result = this.sliceManager.forceNextSlice();
719
+ if (result.slice) {
720
+ this.log(`Forced slice ${result.slice.index} ready (${result.slice.data.length} bytes)`);
721
+
722
+ // Process VAD for the slice if enabled
723
+ if (!this.isTranscribing && this.vadEnabled) {
724
+ this.processSliceVAD(result.slice).catch(error => {
725
+ this.handleError(`VAD processing error: ${error}`);
726
+ });
727
+ } else if (!this.isTranscribing) {
728
+ // If VAD is disabled, transcribe slices as they become ready
729
+ this.queueSliceForTranscription(result.slice).catch(error => {
730
+ this.handleError(`Failed to queue slice for transcription: ${error}`);
731
+ });
732
+ } else {
733
+ this.log(`Skipping slice ${result.slice.index} - already transcribing`);
734
+ }
735
+ this.emitStatsUpdate('memory_change');
736
+ } else {
737
+ this.log('Forced next slice but no slice data to process');
738
+ }
739
+ }
740
+
741
+ /**
742
+ * Reset all components
743
+ */
744
+ reset() {
745
+ this.sliceManager.reset();
746
+ this.transcriptionQueue = [];
747
+ this.isTranscribing = false;
748
+ this.accumulatedData = new Uint8Array(0);
749
+
750
+ // Reset simplified VAD state
751
+ this.lastSpeechDetectedTime = -1;
752
+ this.lastVadState = 'silence';
753
+
754
+ // Reset stats snapshot for clean start
755
+ this.lastStatsSnapshot = null;
756
+
757
+ // Cancel WAV file writing if in progress
758
+ if (this.wavFileWriter) {
759
+ this.wavFileWriter.cancel().catch(error => {
760
+ this.log(`Failed to cancel WAV file writing: ${error}`);
761
+ });
762
+ this.wavFileWriter = null;
763
+ }
764
+
765
+ // Clear transcription results
766
+ this.transcriptionResults.clear();
767
+ }
768
+
769
+ /**
770
+ * Release all resources
771
+ */
772
+ async release() {
773
+ var _this$wavFileWriter;
774
+ if (this.isActive) {
775
+ await this.stop();
776
+ }
777
+ await this.audioStream.release();
778
+ await ((_this$wavFileWriter = this.wavFileWriter) === null || _this$wavFileWriter === void 0 ? void 0 : _this$wavFileWriter.finalize());
779
+ this.vadContext = undefined;
780
+ }
781
+
782
+ /**
783
+ * Emit stats update event if stats have changed significantly
784
+ */
785
+ emitStatsUpdate(eventType) {
786
+ const currentStats = this.getStatistics();
787
+
788
+ // Check if stats have changed significantly
789
+ if (!this.lastStatsSnapshot || RealtimeTranscriber.shouldEmitStatsUpdate(currentStats, this.lastStatsSnapshot)) {
790
+ var _this$callbacks$onSta4, _this$callbacks9;
791
+ const statsEvent = {
792
+ timestamp: Date.now(),
793
+ type: eventType,
794
+ data: currentStats
795
+ };
796
+ (_this$callbacks$onSta4 = (_this$callbacks9 = this.callbacks).onStatsUpdate) === null || _this$callbacks$onSta4 === void 0 ? void 0 : _this$callbacks$onSta4.call(_this$callbacks9, statsEvent);
797
+ this.lastStatsSnapshot = {
798
+ ...currentStats
799
+ };
800
+ }
801
+ }
802
+
803
+ /**
804
+ * Determine if stats update should be emitted
805
+ */
806
+ static shouldEmitStatsUpdate(current, previous) {
807
+ var _current$sliceStats, _current$sliceStats$m, _previous$sliceStats, _previous$sliceStats$;
808
+ // Always emit on status changes
809
+ if (current.isActive !== previous.isActive || current.isTranscribing !== previous.isTranscribing) {
810
+ return true;
811
+ }
812
+
813
+ // Emit on significant memory changes (>10% or >5MB)
814
+ const currentMemory = ((_current$sliceStats = current.sliceStats) === null || _current$sliceStats === void 0 ? void 0 : (_current$sliceStats$m = _current$sliceStats.memoryUsage) === null || _current$sliceStats$m === void 0 ? void 0 : _current$sliceStats$m.estimatedMB) || 0;
815
+ const previousMemory = ((_previous$sliceStats = previous.sliceStats) === null || _previous$sliceStats === void 0 ? void 0 : (_previous$sliceStats$ = _previous$sliceStats.memoryUsage) === null || _previous$sliceStats$ === void 0 ? void 0 : _previous$sliceStats$.estimatedMB) || 0;
816
+ const memoryDiff = Math.abs(currentMemory - previousMemory);
817
+ if (memoryDiff > 5 || previousMemory > 0 && memoryDiff / previousMemory > 0.1) {
818
+ return true;
819
+ }
820
+ return false;
821
+ }
822
+
823
+ /**
824
+ * Logger function
825
+ */
826
+ log(message) {
827
+ this.options.logger(`[RealtimeTranscriber] ${message}`);
828
+ }
829
+ }
830
+ exports.RealtimeTranscriber = RealtimeTranscriber;
831
+ //# sourceMappingURL=RealtimeTranscriber.js.map