RealTimeSTT 0.3.101__tar.gz → 0.3.102__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {realtimestt-0.3.101/realtimestt.egg-info → realtimestt-0.3.102}/PKG-INFO +1 -1
- {realtimestt-0.3.101 → realtimestt-0.3.102}/RealtimeSTT/audio_recorder.py +134 -63
- {realtimestt-0.3.101 → realtimestt-0.3.102}/RealtimeSTT/audio_recorder_client.py +71 -6
- realtimestt-0.3.102/RealtimeSTT/safepipe.py +245 -0
- {realtimestt-0.3.101 → realtimestt-0.3.102}/RealtimeSTT_server/stt_server.py +75 -12
- {realtimestt-0.3.101 → realtimestt-0.3.102/realtimestt.egg-info}/PKG-INFO +1 -1
- {realtimestt-0.3.101 → realtimestt-0.3.102}/realtimestt.egg-info/SOURCES.txt +1 -0
- {realtimestt-0.3.101 → realtimestt-0.3.102}/setup.py +1 -1
- {realtimestt-0.3.101 → realtimestt-0.3.102}/LICENSE +0 -0
- {realtimestt-0.3.101 → realtimestt-0.3.102}/MANIFEST.in +0 -0
- {realtimestt-0.3.101 → realtimestt-0.3.102}/README.md +0 -0
- {realtimestt-0.3.101 → realtimestt-0.3.102}/RealtimeSTT/__init__.py +0 -0
- {realtimestt-0.3.101 → realtimestt-0.3.102}/RealtimeSTT/audio_input.py +0 -0
- {realtimestt-0.3.101 → realtimestt-0.3.102}/RealtimeSTT/warmup_audio.wav +0 -0
- {realtimestt-0.3.101 → realtimestt-0.3.102}/RealtimeSTT_server/__init__.py +0 -0
- {realtimestt-0.3.101 → realtimestt-0.3.102}/RealtimeSTT_server/install_packages.py +0 -0
- {realtimestt-0.3.101 → realtimestt-0.3.102}/RealtimeSTT_server/stt_cli_client.py +0 -0
- {realtimestt-0.3.101 → realtimestt-0.3.102}/realtimestt.egg-info/dependency_links.txt +0 -0
- {realtimestt-0.3.101 → realtimestt-0.3.102}/realtimestt.egg-info/entry_points.txt +0 -0
- {realtimestt-0.3.101 → realtimestt-0.3.102}/realtimestt.egg-info/requires.txt +0 -0
- {realtimestt-0.3.101 → realtimestt-0.3.102}/realtimestt.egg-info/top_level.txt +0 -0
- {realtimestt-0.3.101 → realtimestt-0.3.102}/requirements.txt +0 -0
- {realtimestt-0.3.101 → realtimestt-0.3.102}/setup.cfg +0 -0
|
@@ -34,6 +34,7 @@ from scipy.signal import resample
|
|
|
34
34
|
import signal as system_signal
|
|
35
35
|
from ctypes import c_bool
|
|
36
36
|
from scipy import signal
|
|
37
|
+
from .safepipe import SafePipe
|
|
37
38
|
import soundfile as sf
|
|
38
39
|
import faster_whisper
|
|
39
40
|
import openwakeword
|
|
@@ -93,7 +94,7 @@ if platform.system() != 'Darwin':
|
|
|
93
94
|
class TranscriptionWorker:
|
|
94
95
|
def __init__(self, conn, stdout_pipe, model_path, download_root, compute_type, gpu_device_index, device,
|
|
95
96
|
ready_event, shutdown_event, interrupt_stop_event, beam_size, initial_prompt, suppress_tokens,
|
|
96
|
-
batch_size, faster_whisper_vad_filter):
|
|
97
|
+
batch_size, faster_whisper_vad_filter, normalize_audio):
|
|
97
98
|
self.conn = conn
|
|
98
99
|
self.stdout_pipe = stdout_pipe
|
|
99
100
|
self.model_path = model_path
|
|
@@ -109,6 +110,7 @@ class TranscriptionWorker:
|
|
|
109
110
|
self.suppress_tokens = suppress_tokens
|
|
110
111
|
self.batch_size = batch_size
|
|
111
112
|
self.faster_whisper_vad_filter = faster_whisper_vad_filter
|
|
113
|
+
self.normalize_audio = normalize_audio
|
|
112
114
|
self.queue = queue.Queue()
|
|
113
115
|
|
|
114
116
|
def custom_print(self, *args, **kwargs):
|
|
@@ -120,13 +122,16 @@ class TranscriptionWorker:
|
|
|
120
122
|
|
|
121
123
|
def poll_connection(self):
|
|
122
124
|
while not self.shutdown_event.is_set():
|
|
123
|
-
|
|
124
|
-
|
|
125
|
+
try:
|
|
126
|
+
# Use a longer timeout to reduce polling frequency
|
|
127
|
+
if self.conn.poll(0.01): # Increased from 0.01 to 0.5 seconds
|
|
125
128
|
data = self.conn.recv()
|
|
126
129
|
self.queue.put(data)
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
+
else:
|
|
131
|
+
# Sleep only if no data, but use a shorter sleep
|
|
132
|
+
time.sleep(TIME_SLEEP)
|
|
133
|
+
except Exception as e:
|
|
134
|
+
logging.error(f"Error receiving data from connection: {e}", exc_info=True)
|
|
130
135
|
time.sleep(TIME_SLEEP)
|
|
131
136
|
|
|
132
137
|
def run(self):
|
|
@@ -170,15 +175,32 @@ class TranscriptionWorker:
|
|
|
170
175
|
try:
|
|
171
176
|
while not self.shutdown_event.is_set():
|
|
172
177
|
try:
|
|
173
|
-
audio, language = self.queue.get(timeout=0.1)
|
|
178
|
+
audio, language, use_prompt = self.queue.get(timeout=0.1)
|
|
174
179
|
try:
|
|
175
180
|
logging.debug(f"Transcribing audio with language {language}")
|
|
181
|
+
start_t = time.time()
|
|
182
|
+
|
|
183
|
+
# normalize audio to -0.95 dBFS
|
|
184
|
+
if audio is not None and audio .size > 0:
|
|
185
|
+
if self.normalize_audio:
|
|
186
|
+
peak = np.max(np.abs(audio))
|
|
187
|
+
if peak > 0:
|
|
188
|
+
audio = (audio / peak) * 0.95
|
|
189
|
+
else:
|
|
190
|
+
logging.error("Received None audio for transcription")
|
|
191
|
+
self.conn.send(('error', "Received None audio for transcription"))
|
|
192
|
+
continue
|
|
193
|
+
|
|
194
|
+
prompt = None
|
|
195
|
+
if use_prompt:
|
|
196
|
+
prompt = self.initial_prompt if self.initial_prompt else None
|
|
197
|
+
|
|
176
198
|
if self.batch_size > 0:
|
|
177
199
|
segments, info = model.transcribe(
|
|
178
200
|
audio,
|
|
179
201
|
language=language if language else None,
|
|
180
202
|
beam_size=self.beam_size,
|
|
181
|
-
initial_prompt=
|
|
203
|
+
initial_prompt=prompt,
|
|
182
204
|
suppress_tokens=self.suppress_tokens,
|
|
183
205
|
batch_size=self.batch_size,
|
|
184
206
|
vad_filter=self.faster_whisper_vad_filter
|
|
@@ -188,13 +210,13 @@ class TranscriptionWorker:
|
|
|
188
210
|
audio,
|
|
189
211
|
language=language if language else None,
|
|
190
212
|
beam_size=self.beam_size,
|
|
191
|
-
initial_prompt=
|
|
213
|
+
initial_prompt=prompt,
|
|
192
214
|
suppress_tokens=self.suppress_tokens,
|
|
193
215
|
vad_filter=self.faster_whisper_vad_filter
|
|
194
216
|
)
|
|
195
|
-
|
|
217
|
+
elapsed = time.time() - start_t
|
|
196
218
|
transcription = " ".join(seg.text for seg in segments).strip()
|
|
197
|
-
logging.debug(f"Final text detected with main model: {transcription}")
|
|
219
|
+
logging.debug(f"Final text detected with main model: {transcription} in {elapsed:.4f}s")
|
|
198
220
|
self.conn.send(('success', (transcription, info)))
|
|
199
221
|
except Exception as e:
|
|
200
222
|
logging.error(f"General error in transcription: {e}", exc_info=True)
|
|
@@ -277,6 +299,8 @@ class AudioToTextRecorder:
|
|
|
277
299
|
on_vad_stop=None,
|
|
278
300
|
on_vad_detect_start=None,
|
|
279
301
|
on_vad_detect_stop=None,
|
|
302
|
+
on_turn_detection_start=None,
|
|
303
|
+
on_turn_detection_stop=None,
|
|
280
304
|
|
|
281
305
|
# Wake word parameters
|
|
282
306
|
wakeword_backend: str = "",
|
|
@@ -309,6 +333,7 @@ class AudioToTextRecorder:
|
|
|
309
333
|
no_log_file: bool = False,
|
|
310
334
|
use_extended_logging: bool = False,
|
|
311
335
|
faster_whisper_vad_filter: bool = True,
|
|
336
|
+
normalize_audio: bool = False,
|
|
312
337
|
):
|
|
313
338
|
"""
|
|
314
339
|
Initializes an audio recorder and transcription
|
|
@@ -436,6 +461,14 @@ class AudioToTextRecorder:
|
|
|
436
461
|
called when the system stops listening for voice activity. This is
|
|
437
462
|
not called when VAD actually stops (use on_vad_stop for this), but
|
|
438
463
|
when the system stops listening for it.
|
|
464
|
+
- on_turn_detection_start (callable, default=None): Callback function
|
|
465
|
+
to be called when the system starts to listen for a turn of speech.
|
|
466
|
+
- on_turn_detection_stop (callable, default=None): Callback function to
|
|
467
|
+
be called when the system stops listening for a turn of speech.
|
|
468
|
+
- wakeword_backend (str, default=""): Specifies the backend library to
|
|
469
|
+
use for wake word detection. Supported options include 'pvporcupine'
|
|
470
|
+
for using the Porcupine wake word engine or 'oww' for using the
|
|
471
|
+
OpenWakeWord engine.
|
|
439
472
|
- wakeword_backend (str, default="pvporcupine"): Specifies the backend
|
|
440
473
|
library to use for wake word detection. Supported options include
|
|
441
474
|
'pvporcupine' for using the Porcupine wake word engine or 'oww' for
|
|
@@ -530,6 +563,9 @@ class AudioToTextRecorder:
|
|
|
530
563
|
the system will additionally use the VAD filter from the faster_whisper library
|
|
531
564
|
for voice activity detection. This filter is more robust against
|
|
532
565
|
background noise but requires additional GPU resources.
|
|
566
|
+
- normalize_audio (bool, default=False): If set to True, the system will
|
|
567
|
+
normalize the audio to a specific range before processing. This can
|
|
568
|
+
help improve the quality of the transcription.
|
|
533
569
|
|
|
534
570
|
Raises:
|
|
535
571
|
Exception: Errors related to initializing transcription
|
|
@@ -564,6 +600,8 @@ class AudioToTextRecorder:
|
|
|
564
600
|
self.on_vad_stop = on_vad_stop
|
|
565
601
|
self.on_vad_detect_start = on_vad_detect_start
|
|
566
602
|
self.on_vad_detect_stop = on_vad_detect_stop
|
|
603
|
+
self.on_turn_detection_start = on_turn_detection_start
|
|
604
|
+
self.on_turn_detection_stop = on_turn_detection_stop
|
|
567
605
|
self.on_wakeword_detection_start = on_wakeword_detection_start
|
|
568
606
|
self.on_wakeword_detection_end = on_wakeword_detection_end
|
|
569
607
|
self.on_recorded_chunk = on_recorded_chunk
|
|
@@ -641,6 +679,7 @@ class AudioToTextRecorder:
|
|
|
641
679
|
self.early_transcription_on_silence = early_transcription_on_silence
|
|
642
680
|
self.use_extended_logging = use_extended_logging
|
|
643
681
|
self.faster_whisper_vad_filter = faster_whisper_vad_filter
|
|
682
|
+
self.normalize_audio = normalize_audio
|
|
644
683
|
self.awaiting_speech_end = False
|
|
645
684
|
|
|
646
685
|
# ----------------------------------------------------------------------------
|
|
@@ -687,8 +726,9 @@ class AudioToTextRecorder:
|
|
|
687
726
|
self.interrupt_stop_event = mp.Event()
|
|
688
727
|
self.was_interrupted = mp.Event()
|
|
689
728
|
self.main_transcription_ready_event = mp.Event()
|
|
690
|
-
|
|
691
|
-
self.
|
|
729
|
+
|
|
730
|
+
self.parent_transcription_pipe, child_transcription_pipe = SafePipe()
|
|
731
|
+
self.parent_stdout_pipe, child_stdout_pipe = SafePipe()
|
|
692
732
|
|
|
693
733
|
# Set device for model
|
|
694
734
|
self.device = "cuda" if self.device == "cuda" and torch.cuda.is_available() else "cpu"
|
|
@@ -784,7 +824,7 @@ class AudioToTextRecorder:
|
|
|
784
824
|
for _ in range(len(self.wake_words_list))
|
|
785
825
|
]
|
|
786
826
|
|
|
787
|
-
if self.wakeword_backend in {'pvp', 'pvporcupine'}:
|
|
827
|
+
if wake_words and self.wakeword_backend in {'pvp', 'pvporcupine'}:
|
|
788
828
|
|
|
789
829
|
try:
|
|
790
830
|
self.porcupine = pvporcupine.create(
|
|
@@ -797,7 +837,8 @@ class AudioToTextRecorder:
|
|
|
797
837
|
except Exception as e:
|
|
798
838
|
logger.exception(
|
|
799
839
|
"Error initializing porcupine "
|
|
800
|
-
f"wake word detection engine: {e}"
|
|
840
|
+
f"wake word detection engine: {e}. "
|
|
841
|
+
f"Wakewords: {self.wake_words_list}."
|
|
801
842
|
)
|
|
802
843
|
raise
|
|
803
844
|
|
|
@@ -805,7 +846,7 @@ class AudioToTextRecorder:
|
|
|
805
846
|
"Porcupine wake word detection engine initialized successfully"
|
|
806
847
|
)
|
|
807
848
|
|
|
808
|
-
elif self.wakeword_backend in {'oww', 'openwakeword', 'openwakewords'}:
|
|
849
|
+
elif wake_words and self.wakeword_backend in {'oww', 'openwakeword', 'openwakewords'}:
|
|
809
850
|
|
|
810
851
|
openwakeword.utils.download_models()
|
|
811
852
|
|
|
@@ -848,7 +889,7 @@ class AudioToTextRecorder:
|
|
|
848
889
|
)
|
|
849
890
|
|
|
850
891
|
else:
|
|
851
|
-
logger.exception(f"Wakeword engine {self.wakeword_backend} unknown/unsupported. Please specify one of: pvporcupine, openwakeword.")
|
|
892
|
+
logger.exception(f"Wakeword engine {self.wakeword_backend} unknown/unsupported or wake_words not specified. Please specify one of: pvporcupine, openwakeword.")
|
|
852
893
|
|
|
853
894
|
|
|
854
895
|
# Setup voice activity detection model WebRTC
|
|
@@ -972,6 +1013,10 @@ class AudioToTextRecorder:
|
|
|
972
1013
|
worker = TranscriptionWorker(*args, **kwargs)
|
|
973
1014
|
worker.run()
|
|
974
1015
|
|
|
1016
|
+
def _run_callback(self, cb, *args, **kwargs):
|
|
1017
|
+
if cb:
|
|
1018
|
+
threading.Thread(target=cb, args=args, kwargs=kwargs, daemon=True).start()
|
|
1019
|
+
|
|
975
1020
|
@staticmethod
|
|
976
1021
|
def _audio_data_worker(
|
|
977
1022
|
audio_queue,
|
|
@@ -1441,14 +1486,22 @@ class AudioToTextRecorder:
|
|
|
1441
1486
|
raise # Re-raise the exception after cleanup
|
|
1442
1487
|
|
|
1443
1488
|
|
|
1444
|
-
def perform_final_transcription(self, audio_bytes=None):
|
|
1489
|
+
def perform_final_transcription(self, audio_bytes=None, use_prompt=True):
|
|
1445
1490
|
start_time = 0
|
|
1446
1491
|
with self.transcription_lock:
|
|
1492
|
+
if audio_bytes is None:
|
|
1493
|
+
audio_bytes = copy.deepcopy(self.audio)
|
|
1494
|
+
|
|
1495
|
+
if audio_bytes is None or len(audio_bytes) == 0:
|
|
1496
|
+
print("No audio data available for transcription")
|
|
1497
|
+
#logger.info("No audio data available for transcription")
|
|
1498
|
+
return ""
|
|
1499
|
+
|
|
1447
1500
|
try:
|
|
1448
1501
|
if self.transcribe_count == 0:
|
|
1449
1502
|
logger.debug("Adding transcription request, no early transcription started")
|
|
1450
1503
|
start_time = time.time() # Start timing
|
|
1451
|
-
self.parent_transcription_pipe.send((audio_bytes, self.language))
|
|
1504
|
+
self.parent_transcription_pipe.send((audio_bytes, self.language, use_prompt))
|
|
1452
1505
|
self.transcribe_count += 1
|
|
1453
1506
|
|
|
1454
1507
|
while self.transcribe_count > 0:
|
|
@@ -1651,7 +1704,7 @@ class AudioToTextRecorder:
|
|
|
1651
1704
|
self.start_recording_event.set()
|
|
1652
1705
|
|
|
1653
1706
|
if self.on_recording_start:
|
|
1654
|
-
self.on_recording_start
|
|
1707
|
+
self._run_callback(self.on_recording_start)
|
|
1655
1708
|
|
|
1656
1709
|
return self
|
|
1657
1710
|
|
|
@@ -1695,7 +1748,7 @@ class AudioToTextRecorder:
|
|
|
1695
1748
|
self.last_recording_stop_time = self.recording_stop_time
|
|
1696
1749
|
|
|
1697
1750
|
if self.on_recording_stop:
|
|
1698
|
-
self.on_recording_stop
|
|
1751
|
+
self._run_callback(self.on_recording_stop)
|
|
1699
1752
|
|
|
1700
1753
|
return self
|
|
1701
1754
|
|
|
@@ -1870,7 +1923,7 @@ class AudioToTextRecorder:
|
|
|
1870
1923
|
if self.on_recorded_chunk:
|
|
1871
1924
|
if self.use_extended_logging:
|
|
1872
1925
|
logger.debug('Debug: Calling on_recorded_chunk')
|
|
1873
|
-
self.on_recorded_chunk
|
|
1926
|
+
self._run_callback(self.on_recorded_chunk, data)
|
|
1874
1927
|
|
|
1875
1928
|
if self.use_extended_logging:
|
|
1876
1929
|
logger.debug('Debug: Checking if handle_buffer_overflow is True')
|
|
@@ -1940,7 +1993,7 @@ class AudioToTextRecorder:
|
|
|
1940
1993
|
if self.on_wakeword_timeout:
|
|
1941
1994
|
if self.use_extended_logging:
|
|
1942
1995
|
logger.debug('Debug: Calling on_wakeword_timeout')
|
|
1943
|
-
self.on_wakeword_timeout
|
|
1996
|
+
self._run_callback(self.on_wakeword_timeout)
|
|
1944
1997
|
delay_was_passed = wake_word_activation_delay_passed
|
|
1945
1998
|
|
|
1946
1999
|
if self.use_extended_logging:
|
|
@@ -1993,7 +2046,7 @@ class AudioToTextRecorder:
|
|
|
1993
2046
|
if self.on_wakeword_detected:
|
|
1994
2047
|
if self.use_extended_logging:
|
|
1995
2048
|
logger.debug('Debug: Calling on_wakeword_detected')
|
|
1996
|
-
self.on_wakeword_detected
|
|
2049
|
+
self._run_callback(self.on_wakeword_detected)
|
|
1997
2050
|
|
|
1998
2051
|
if self.use_extended_logging:
|
|
1999
2052
|
logger.debug('Debug: Checking voice activity conditions')
|
|
@@ -2010,7 +2063,7 @@ class AudioToTextRecorder:
|
|
|
2010
2063
|
if self._is_voice_active():
|
|
2011
2064
|
|
|
2012
2065
|
if self.on_vad_start:
|
|
2013
|
-
self.on_vad_start
|
|
2066
|
+
self._run_callback(self.on_vad_start)
|
|
2014
2067
|
|
|
2015
2068
|
if self.use_extended_logging:
|
|
2016
2069
|
logger.debug('Debug: Voice activity detected')
|
|
@@ -2040,7 +2093,13 @@ class AudioToTextRecorder:
|
|
|
2040
2093
|
|
|
2041
2094
|
if self.use_extended_logging:
|
|
2042
2095
|
logger.debug('Debug: Resetting speech_end_silence_start')
|
|
2043
|
-
|
|
2096
|
+
|
|
2097
|
+
if self.speech_end_silence_start != 0:
|
|
2098
|
+
self.speech_end_silence_start = 0
|
|
2099
|
+
if self.on_turn_detection_stop:
|
|
2100
|
+
if self.use_extended_logging:
|
|
2101
|
+
logger.debug('Debug: Calling on_turn_detection_stop')
|
|
2102
|
+
self._run_callback(self.on_turn_detection_stop)
|
|
2044
2103
|
|
|
2045
2104
|
else:
|
|
2046
2105
|
if self.use_extended_logging:
|
|
@@ -2097,6 +2156,11 @@ class AudioToTextRecorder:
|
|
|
2097
2156
|
|
|
2098
2157
|
self.speech_end_silence_start = time.time()
|
|
2099
2158
|
self.awaiting_speech_end = True
|
|
2159
|
+
if self.on_turn_detection_start:
|
|
2160
|
+
if self.use_extended_logging:
|
|
2161
|
+
logger.debug('Debug: Calling on_turn_detection_start')
|
|
2162
|
+
|
|
2163
|
+
self._run_callback(self.on_turn_detection_start)
|
|
2100
2164
|
|
|
2101
2165
|
if self.use_extended_logging:
|
|
2102
2166
|
logger.debug('Debug: Checking early transcription conditions')
|
|
@@ -2111,7 +2175,7 @@ class AudioToTextRecorder:
|
|
|
2111
2175
|
|
|
2112
2176
|
if self.use_extended_logging:
|
|
2113
2177
|
logger.debug("Debug: early transcription request pipe send")
|
|
2114
|
-
self.parent_transcription_pipe.send((audio, self.language))
|
|
2178
|
+
self.parent_transcription_pipe.send((audio, self.language, True))
|
|
2115
2179
|
if self.use_extended_logging:
|
|
2116
2180
|
logger.debug("Debug: early transcription request pipe send return")
|
|
2117
2181
|
self.allowed_to_early_transcribe = False
|
|
@@ -2123,7 +2187,14 @@ class AudioToTextRecorder:
|
|
|
2123
2187
|
if self.speech_end_silence_start:
|
|
2124
2188
|
if self.use_extended_logging:
|
|
2125
2189
|
logger.info("Resetting self.speech_end_silence_start")
|
|
2126
|
-
|
|
2190
|
+
|
|
2191
|
+
if self.speech_end_silence_start != 0:
|
|
2192
|
+
self.speech_end_silence_start = 0
|
|
2193
|
+
if self.on_turn_detection_stop:
|
|
2194
|
+
if self.use_extended_logging:
|
|
2195
|
+
logger.debug('Debug: Calling on_turn_detection_stop')
|
|
2196
|
+
self._run_callback(self.on_turn_detection_stop)
|
|
2197
|
+
|
|
2127
2198
|
self.allowed_to_early_transcribe = True
|
|
2128
2199
|
|
|
2129
2200
|
if self.use_extended_logging:
|
|
@@ -2134,7 +2205,7 @@ class AudioToTextRecorder:
|
|
|
2134
2205
|
self.post_speech_silence_duration:
|
|
2135
2206
|
|
|
2136
2207
|
if self.on_vad_stop:
|
|
2137
|
-
self.on_vad_stop
|
|
2208
|
+
self._run_callback(self.on_vad_stop)
|
|
2138
2209
|
|
|
2139
2210
|
if self.use_extended_logging:
|
|
2140
2211
|
logger.debug('Debug: Formatting silence start time')
|
|
@@ -2155,9 +2226,12 @@ class AudioToTextRecorder:
|
|
|
2155
2226
|
self.frames.append(data)
|
|
2156
2227
|
self.stop()
|
|
2157
2228
|
if not self.is_recording:
|
|
2158
|
-
if self.
|
|
2159
|
-
|
|
2160
|
-
|
|
2229
|
+
if self.speech_end_silence_start != 0:
|
|
2230
|
+
self.speech_end_silence_start = 0
|
|
2231
|
+
if self.on_turn_detection_stop:
|
|
2232
|
+
if self.use_extended_logging:
|
|
2233
|
+
logger.debug('Debug: Calling on_turn_detection_stop')
|
|
2234
|
+
self._run_callback(self.on_turn_detection_stop)
|
|
2161
2235
|
|
|
2162
2236
|
if self.use_extended_logging:
|
|
2163
2237
|
logger.debug('Debug: Handling non-wake word scenario')
|
|
@@ -2192,7 +2266,7 @@ class AudioToTextRecorder:
|
|
|
2192
2266
|
if self.wakeword_detected and self.on_wakeword_timeout:
|
|
2193
2267
|
if self.use_extended_logging:
|
|
2194
2268
|
logger.debug('Debug: Calling on_wakeword_timeout')
|
|
2195
|
-
self.on_wakeword_timeout
|
|
2269
|
+
self._run_callback(self.on_wakeword_timeout)
|
|
2196
2270
|
self.wakeword_detected = False
|
|
2197
2271
|
|
|
2198
2272
|
if self.use_extended_logging:
|
|
@@ -2283,7 +2357,7 @@ class AudioToTextRecorder:
|
|
|
2283
2357
|
if self.use_main_model_for_realtime:
|
|
2284
2358
|
with self.transcription_lock:
|
|
2285
2359
|
try:
|
|
2286
|
-
self.parent_transcription_pipe.send((audio_array, self.language))
|
|
2360
|
+
self.parent_transcription_pipe.send((audio_array, self.language, True))
|
|
2287
2361
|
if self.parent_transcription_pipe.poll(timeout=5): # Wait for 5 seconds
|
|
2288
2362
|
logger.debug("Receive from realtime worker after transcription request to main model")
|
|
2289
2363
|
status, result = self.parent_transcription_pipe.recv()
|
|
@@ -2304,6 +2378,13 @@ class AudioToTextRecorder:
|
|
|
2304
2378
|
continue
|
|
2305
2379
|
else:
|
|
2306
2380
|
# Perform transcription and assemble the text
|
|
2381
|
+
if self.normalize_audio:
|
|
2382
|
+
# normalize audio to -0.95 dBFS
|
|
2383
|
+
if audio_array is not None and audio_array.size > 0:
|
|
2384
|
+
peak = np.max(np.abs(audio_array))
|
|
2385
|
+
if peak > 0:
|
|
2386
|
+
audio_array = (audio_array / peak) * 0.95
|
|
2387
|
+
|
|
2307
2388
|
if self.realtime_batch_size > 0:
|
|
2308
2389
|
segments, info = self.realtime_model_type.transcribe(
|
|
2309
2390
|
audio_array,
|
|
@@ -2372,20 +2453,17 @@ class AudioToTextRecorder:
|
|
|
2372
2453
|
)
|
|
2373
2454
|
|
|
2374
2455
|
if matching_pos < 0:
|
|
2375
|
-
|
|
2376
|
-
|
|
2377
|
-
|
|
2378
|
-
|
|
2379
|
-
|
|
2380
|
-
|
|
2381
|
-
|
|
2382
|
-
|
|
2383
|
-
|
|
2384
|
-
|
|
2385
|
-
|
|
2386
|
-
True
|
|
2387
|
-
)
|
|
2388
|
-
)
|
|
2456
|
+
# pick which text to send
|
|
2457
|
+
text_to_send = (
|
|
2458
|
+
self.realtime_stabilized_safetext
|
|
2459
|
+
if self.realtime_stabilized_safetext
|
|
2460
|
+
else self.realtime_transcription_text
|
|
2461
|
+
)
|
|
2462
|
+
# preprocess once
|
|
2463
|
+
processed = self._preprocess_output(text_to_send, True)
|
|
2464
|
+
# invoke on its own thread
|
|
2465
|
+
self._run_callback(self._on_realtime_transcription_stabilized, processed)
|
|
2466
|
+
|
|
2389
2467
|
else:
|
|
2390
2468
|
# We found parts of the stabilized text
|
|
2391
2469
|
# in the transcripted text
|
|
@@ -2398,17 +2476,10 @@ class AudioToTextRecorder:
|
|
|
2398
2476
|
# AND at the same time delivers fresh detected
|
|
2399
2477
|
# parts on the first run without the need for
|
|
2400
2478
|
# two transcriptions
|
|
2401
|
-
self._on_realtime_transcription_stabilized(
|
|
2402
|
-
self._preprocess_output(output_text, True)
|
|
2403
|
-
)
|
|
2479
|
+
self._run_callback(self._on_realtime_transcription_stabilized, self._preprocess_output(output_text, True))
|
|
2404
2480
|
|
|
2405
2481
|
# Invoke the callback with the transcribed text
|
|
2406
|
-
self._on_realtime_transcription_update(
|
|
2407
|
-
self._preprocess_output(
|
|
2408
|
-
self.realtime_transcription_text,
|
|
2409
|
-
True
|
|
2410
|
-
)
|
|
2411
|
-
)
|
|
2482
|
+
self._run_callback(self._on_realtime_transcription_update, self._preprocess_output(self.realtime_transcription_text,True))
|
|
2412
2483
|
|
|
2413
2484
|
# If not recording, sleep briefly before checking again
|
|
2414
2485
|
else:
|
|
@@ -2572,21 +2643,21 @@ class AudioToTextRecorder:
|
|
|
2572
2643
|
# Execute callbacks based on transitioning FROM a particular state
|
|
2573
2644
|
if old_state == "listening":
|
|
2574
2645
|
if self.on_vad_detect_stop:
|
|
2575
|
-
self.on_vad_detect_stop
|
|
2646
|
+
self._run_callback(self.on_vad_detect_stop)
|
|
2576
2647
|
elif old_state == "wakeword":
|
|
2577
2648
|
if self.on_wakeword_detection_end:
|
|
2578
|
-
self.on_wakeword_detection_end
|
|
2649
|
+
self._run_callback(self.on_wakeword_detection_end)
|
|
2579
2650
|
|
|
2580
2651
|
# Execute callbacks based on transitioning TO a particular state
|
|
2581
2652
|
if new_state == "listening":
|
|
2582
2653
|
if self.on_vad_detect_start:
|
|
2583
|
-
self.on_vad_detect_start
|
|
2654
|
+
self._run_callback(self.on_vad_detect_start)
|
|
2584
2655
|
self._set_spinner("speak now")
|
|
2585
2656
|
if self.spinner and self.halo:
|
|
2586
2657
|
self.halo._interval = 250
|
|
2587
2658
|
elif new_state == "wakeword":
|
|
2588
2659
|
if self.on_wakeword_detection_start:
|
|
2589
|
-
self.on_wakeword_detection_start
|
|
2660
|
+
self._run_callback(self.on_wakeword_detection_start)
|
|
2590
2661
|
self._set_spinner(f"say {self.wake_words}")
|
|
2591
2662
|
if self.spinner and self.halo:
|
|
2592
2663
|
self.halo._interval = 500
|
|
@@ -2709,7 +2780,7 @@ class AudioToTextRecorder:
|
|
|
2709
2780
|
"""
|
|
2710
2781
|
if self.on_realtime_transcription_stabilized:
|
|
2711
2782
|
if self.is_recording:
|
|
2712
|
-
self.on_realtime_transcription_stabilized
|
|
2783
|
+
self._run_callback(self.on_realtime_transcription_stabilized, text)
|
|
2713
2784
|
|
|
2714
2785
|
def _on_realtime_transcription_update(self, text):
|
|
2715
2786
|
"""
|
|
@@ -2727,7 +2798,7 @@ class AudioToTextRecorder:
|
|
|
2727
2798
|
"""
|
|
2728
2799
|
if self.on_realtime_transcription_update:
|
|
2729
2800
|
if self.is_recording:
|
|
2730
|
-
self.on_realtime_transcription_update
|
|
2801
|
+
self._run_callback(self.on_realtime_transcription_update, text)
|
|
2731
2802
|
|
|
2732
2803
|
def __enter__(self):
|
|
2733
2804
|
"""
|
|
@@ -6,13 +6,13 @@ from urllib.parse import urlparse
|
|
|
6
6
|
from datetime import datetime
|
|
7
7
|
from websocket import WebSocketApp
|
|
8
8
|
from websocket import ABNF
|
|
9
|
+
import numpy as np
|
|
9
10
|
import subprocess
|
|
10
11
|
import threading
|
|
11
12
|
import platform
|
|
12
13
|
import logging
|
|
13
|
-
import socket
|
|
14
14
|
import struct
|
|
15
|
-
import
|
|
15
|
+
import base64
|
|
16
16
|
import wave
|
|
17
17
|
import json
|
|
18
18
|
import time
|
|
@@ -60,6 +60,23 @@ class bcolors:
|
|
|
60
60
|
BOLD = '\033[1m'
|
|
61
61
|
UNDERLINE = '\033[4m'
|
|
62
62
|
|
|
63
|
+
def format_timestamp_ns(timestamp_ns: int) -> str:
|
|
64
|
+
# Split into whole seconds and the nanosecond remainder
|
|
65
|
+
seconds = timestamp_ns // 1_000_000_000
|
|
66
|
+
remainder_ns = timestamp_ns % 1_000_000_000
|
|
67
|
+
|
|
68
|
+
# Convert seconds part into a datetime object (local time)
|
|
69
|
+
dt = datetime.fromtimestamp(seconds)
|
|
70
|
+
|
|
71
|
+
# Format the main time as HH:MM:SS
|
|
72
|
+
time_str = dt.strftime("%H:%M:%S")
|
|
73
|
+
|
|
74
|
+
# For instance, if you want milliseconds, divide the remainder by 1e6 and format as 3-digit
|
|
75
|
+
milliseconds = remainder_ns // 1_000_000
|
|
76
|
+
formatted_timestamp = f"{time_str}.{milliseconds:03d}"
|
|
77
|
+
|
|
78
|
+
return formatted_timestamp
|
|
79
|
+
|
|
63
80
|
class AudioToTextRecorderClient:
|
|
64
81
|
"""
|
|
65
82
|
A class responsible for capturing audio from the microphone, detecting
|
|
@@ -112,8 +129,12 @@ class AudioToTextRecorderClient:
|
|
|
112
129
|
pre_recording_buffer_duration: float = (
|
|
113
130
|
INIT_PRE_RECORDING_BUFFER_DURATION
|
|
114
131
|
),
|
|
132
|
+
on_vad_start=None,
|
|
133
|
+
on_vad_stop=None,
|
|
115
134
|
on_vad_detect_start=None,
|
|
116
135
|
on_vad_detect_stop=None,
|
|
136
|
+
on_turn_detection_start=None,
|
|
137
|
+
on_turn_detection_stop=None,
|
|
117
138
|
|
|
118
139
|
# Wake word parameters
|
|
119
140
|
wakeword_backend: str = "pvporcupine",
|
|
@@ -151,6 +172,7 @@ class AudioToTextRecorderClient:
|
|
|
151
172
|
data_url: str = DEFAULT_DATA_URL,
|
|
152
173
|
autostart_server: bool = True,
|
|
153
174
|
output_wav_file: str = None,
|
|
175
|
+
faster_whisper_vad_filter: bool = False,
|
|
154
176
|
):
|
|
155
177
|
|
|
156
178
|
# Set instance variables from constructor parameters
|
|
@@ -190,8 +212,13 @@ class AudioToTextRecorderClient:
|
|
|
190
212
|
self.min_length_of_recording = min_length_of_recording
|
|
191
213
|
self.min_gap_between_recordings = min_gap_between_recordings
|
|
192
214
|
self.pre_recording_buffer_duration = pre_recording_buffer_duration
|
|
215
|
+
|
|
216
|
+
self.on_vad_start = on_vad_start
|
|
217
|
+
self.on_vad_stop = on_vad_stop
|
|
193
218
|
self.on_vad_detect_start = on_vad_detect_start
|
|
194
219
|
self.on_vad_detect_stop = on_vad_detect_stop
|
|
220
|
+
self.on_turn_detection_start = on_turn_detection_start
|
|
221
|
+
self.on_turn_detection_stop = on_turn_detection_stop
|
|
195
222
|
|
|
196
223
|
# Wake word parameters
|
|
197
224
|
self.wakeword_backend = wakeword_backend
|
|
@@ -221,6 +248,7 @@ class AudioToTextRecorderClient:
|
|
|
221
248
|
self.allowed_latency_limit = allowed_latency_limit
|
|
222
249
|
self.no_log_file = no_log_file
|
|
223
250
|
self.use_extended_logging = use_extended_logging
|
|
251
|
+
self.faster_whisper_vad_filter = faster_whisper_vad_filter
|
|
224
252
|
|
|
225
253
|
# Server URLs
|
|
226
254
|
self.control_url = control_url
|
|
@@ -315,12 +343,24 @@ class AudioToTextRecorderClient:
|
|
|
315
343
|
print(f"Error in AudioToTextRecorderClient.text(): {e}")
|
|
316
344
|
return ""
|
|
317
345
|
|
|
318
|
-
def feed_audio(self, chunk, original_sample_rate=16000):
|
|
346
|
+
def feed_audio(self, chunk, audio_meta_data, original_sample_rate=16000):
|
|
347
|
+
# Start with the base metadata
|
|
319
348
|
metadata = {"sampleRate": original_sample_rate}
|
|
349
|
+
|
|
350
|
+
# Merge additional metadata if provided
|
|
351
|
+
if audio_meta_data:
|
|
352
|
+
server_sent_to_stt_ns = time.time_ns()
|
|
353
|
+
audio_meta_data["server_sent_to_stt"] = server_sent_to_stt_ns
|
|
354
|
+
metadata["server_sent_to_stt_formatted"] = format_timestamp_ns(server_sent_to_stt_ns)
|
|
355
|
+
|
|
356
|
+
metadata.update(audio_meta_data)
|
|
357
|
+
|
|
358
|
+
# Convert metadata to JSON and prepare the message
|
|
320
359
|
metadata_json = json.dumps(metadata)
|
|
321
360
|
metadata_length = len(metadata_json)
|
|
322
361
|
message = struct.pack('<I', metadata_length) + metadata_json.encode('utf-8') + chunk
|
|
323
362
|
|
|
363
|
+
# Send the message if the connection is running
|
|
324
364
|
if self.is_running:
|
|
325
365
|
self.data_ws.send(message, opcode=ABNF.OPCODE_BINARY)
|
|
326
366
|
|
|
@@ -339,6 +379,9 @@ class AudioToTextRecorderClient:
|
|
|
339
379
|
def clear_audio_queue(self):
|
|
340
380
|
self.call_method("clear_audio_queue")
|
|
341
381
|
|
|
382
|
+
def perform_final_transcription(self):
|
|
383
|
+
self.call_method("perform_final_transcription")
|
|
384
|
+
|
|
342
385
|
def stop(self):
|
|
343
386
|
self.call_method("stop")
|
|
344
387
|
|
|
@@ -424,8 +467,8 @@ class AudioToTextRecorderClient:
|
|
|
424
467
|
# args += ['--allowed_latency_limit', str(self.allowed_latency_limit)]
|
|
425
468
|
# if self.no_log_file:
|
|
426
469
|
# args.append('--no_log_file') # flag, no need for True
|
|
427
|
-
|
|
428
|
-
|
|
470
|
+
if self.debug_mode:
|
|
471
|
+
args.append('--debug') # flag, no need for True/False
|
|
429
472
|
|
|
430
473
|
if self.language:
|
|
431
474
|
args += ['--language', self.language]
|
|
@@ -668,14 +711,36 @@ class AudioToTextRecorderClient:
|
|
|
668
711
|
if self.on_recording_stop:
|
|
669
712
|
self.on_recording_stop()
|
|
670
713
|
elif data.get('type') == 'transcription_start':
|
|
714
|
+
audio_bytes_base64 = data.get('audio_bytes_base64')
|
|
715
|
+
decoded_bytes = base64.b64decode(audio_bytes_base64)
|
|
716
|
+
|
|
717
|
+
# Reconstruct the np.int16 array from the decoded bytes
|
|
718
|
+
audio_array = np.frombuffer(decoded_bytes, dtype=np.int16)
|
|
719
|
+
|
|
720
|
+
# If the original data was normalized, convert to np.float32 and normalize
|
|
721
|
+
INT16_MAX_ABS_VALUE = 32768.0
|
|
722
|
+
normalized_audio = audio_array.astype(np.float32) / INT16_MAX_ABS_VALUE
|
|
723
|
+
|
|
671
724
|
if self.on_transcription_start:
|
|
672
|
-
self.on_transcription_start()
|
|
725
|
+
self.on_transcription_start(normalized_audio)
|
|
673
726
|
elif data.get('type') == 'vad_detect_start':
|
|
674
727
|
if self.on_vad_detect_start:
|
|
675
728
|
self.on_vad_detect_start()
|
|
676
729
|
elif data.get('type') == 'vad_detect_stop':
|
|
677
730
|
if self.on_vad_detect_stop:
|
|
678
731
|
self.on_vad_detect_stop()
|
|
732
|
+
elif data.get('type') == 'vad_start':
|
|
733
|
+
if self.on_vad_start:
|
|
734
|
+
self.on_vad_start()
|
|
735
|
+
elif data.get('type') == 'vad_stop':
|
|
736
|
+
if self.on_vad_stop:
|
|
737
|
+
self.on_vad_stop()
|
|
738
|
+
elif data.get('type') == 'start_turn_detection':
|
|
739
|
+
if self.on_turn_detection_start:
|
|
740
|
+
self.on_turn_detection_start()
|
|
741
|
+
elif data.get('type') == 'stop_turn_detection':
|
|
742
|
+
if self.on_turn_detection_stop:
|
|
743
|
+
self.on_turn_detection_stop()
|
|
679
744
|
elif data.get('type') == 'wakeword_detected':
|
|
680
745
|
if self.on_wakeword_detected:
|
|
681
746
|
self.on_wakeword_detected()
|
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
import multiprocessing as mp
|
|
3
|
+
import queue
|
|
4
|
+
import threading
|
|
5
|
+
import time
|
|
6
|
+
import logging
|
|
7
|
+
|
|
8
|
+
# Configure logging. Adjust level and formatting as needed.
|
|
9
|
+
# logging.basicConfig(level=logging.DEBUG,
|
|
10
|
+
# format='[%(asctime)s] %(levelname)s:%(name)s: %(message)s')
|
|
11
|
+
logger = logging.getLogger(__name__)
|
|
12
|
+
|
|
13
|
+
try:
|
|
14
|
+
# Only set the start method if it hasn't been set already.
|
|
15
|
+
if sys.platform.startswith('linux') or sys.platform == 'darwin': # For Linux or macOS
|
|
16
|
+
mp.set_start_method("spawn")
|
|
17
|
+
elif mp.get_start_method(allow_none=True) is None:
|
|
18
|
+
mp.set_start_method("spawn")
|
|
19
|
+
except RuntimeError as e:
|
|
20
|
+
logger.debug("Start method has already been set. Details: %s", e)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class ParentPipe:
|
|
24
|
+
"""
|
|
25
|
+
A thread-safe wrapper around the 'parent end' of a multiprocessing pipe.
|
|
26
|
+
All actual pipe operations happen in a dedicated worker thread, so it's safe
|
|
27
|
+
for multiple threads to call send(), recv(), or poll() on the same ParentPipe
|
|
28
|
+
without interfering.
|
|
29
|
+
"""
|
|
30
|
+
def __init__(self, parent_synthesize_pipe):
|
|
31
|
+
self.name = "ParentPipe"
|
|
32
|
+
self._pipe = parent_synthesize_pipe # The raw pipe.
|
|
33
|
+
self._closed = False # A flag to mark if close() has been called.
|
|
34
|
+
|
|
35
|
+
# The request queue for sending operations to the worker.
|
|
36
|
+
self._request_queue = queue.Queue()
|
|
37
|
+
|
|
38
|
+
# This event signals the worker thread to stop.
|
|
39
|
+
self._stop_event = threading.Event()
|
|
40
|
+
|
|
41
|
+
# Worker thread that executes actual .send(), .recv(), .poll() calls.
|
|
42
|
+
self._worker_thread = threading.Thread(
|
|
43
|
+
target=self._pipe_worker,
|
|
44
|
+
name=f"{self.name}_Worker",
|
|
45
|
+
daemon=True
|
|
46
|
+
)
|
|
47
|
+
self._worker_thread.start()
|
|
48
|
+
|
|
49
|
+
def _pipe_worker(self):
|
|
50
|
+
while not self._stop_event.is_set():
|
|
51
|
+
try:
|
|
52
|
+
request = self._request_queue.get(timeout=0.1)
|
|
53
|
+
except queue.Empty:
|
|
54
|
+
continue
|
|
55
|
+
|
|
56
|
+
if request["type"] == "CLOSE":
|
|
57
|
+
# Exit worker loop on CLOSE request.
|
|
58
|
+
break
|
|
59
|
+
|
|
60
|
+
try:
|
|
61
|
+
if request["type"] == "SEND":
|
|
62
|
+
data = request["data"]
|
|
63
|
+
logger.debug("[%s] Worker: sending => %s", self.name, data)
|
|
64
|
+
self._pipe.send(data)
|
|
65
|
+
request["result_queue"].put(None)
|
|
66
|
+
|
|
67
|
+
elif request["type"] == "RECV":
|
|
68
|
+
logger.debug("[%s] Worker: receiving...", self.name)
|
|
69
|
+
data = self._pipe.recv()
|
|
70
|
+
request["result_queue"].put(data)
|
|
71
|
+
|
|
72
|
+
elif request["type"] == "POLL":
|
|
73
|
+
timeout = request.get("timeout", 0.0)
|
|
74
|
+
logger.debug("[%s] Worker: poll() with timeout: %s", self.name, timeout)
|
|
75
|
+
result = self._pipe.poll(timeout)
|
|
76
|
+
request["result_queue"].put(result)
|
|
77
|
+
|
|
78
|
+
except (EOFError, BrokenPipeError, OSError) as e:
|
|
79
|
+
# When the other end has closed or an error occurs,
|
|
80
|
+
# log and notify the waiting thread.
|
|
81
|
+
logger.debug("[%s] Worker: pipe closed or error occurred (%s). Shutting down.", self.name, e)
|
|
82
|
+
request["result_queue"].put(None)
|
|
83
|
+
break
|
|
84
|
+
|
|
85
|
+
except Exception as e:
|
|
86
|
+
logger.exception("[%s] Worker: unexpected error.", self.name)
|
|
87
|
+
request["result_queue"].put(e)
|
|
88
|
+
break
|
|
89
|
+
|
|
90
|
+
logger.debug("[%s] Worker: stopping.", self.name)
|
|
91
|
+
try:
|
|
92
|
+
self._pipe.close()
|
|
93
|
+
except Exception as e:
|
|
94
|
+
logger.debug("[%s] Worker: error during pipe close: %s", self.name, e)
|
|
95
|
+
|
|
96
|
+
def send(self, data):
|
|
97
|
+
"""
|
|
98
|
+
Synchronously asks the worker thread to perform .send().
|
|
99
|
+
"""
|
|
100
|
+
if self._closed:
|
|
101
|
+
logger.debug("[%s] send() called but pipe is already closed", self.name)
|
|
102
|
+
return
|
|
103
|
+
logger.debug("[%s] send() requested with: %s", self.name, data)
|
|
104
|
+
result_queue = queue.Queue()
|
|
105
|
+
request = {
|
|
106
|
+
"type": "SEND",
|
|
107
|
+
"data": data,
|
|
108
|
+
"result_queue": result_queue
|
|
109
|
+
}
|
|
110
|
+
self._request_queue.put(request)
|
|
111
|
+
result_queue.get() # Wait until sending completes.
|
|
112
|
+
logger.debug("[%s] send() completed", self.name)
|
|
113
|
+
|
|
114
|
+
def recv(self):
|
|
115
|
+
"""
|
|
116
|
+
Synchronously asks the worker to perform .recv() and returns the data.
|
|
117
|
+
"""
|
|
118
|
+
if self._closed:
|
|
119
|
+
logger.debug("[%s] recv() called but pipe is already closed", self.name)
|
|
120
|
+
return None
|
|
121
|
+
logger.debug("[%s] recv() requested", self.name)
|
|
122
|
+
result_queue = queue.Queue()
|
|
123
|
+
request = {
|
|
124
|
+
"type": "RECV",
|
|
125
|
+
"result_queue": result_queue
|
|
126
|
+
}
|
|
127
|
+
self._request_queue.put(request)
|
|
128
|
+
data = result_queue.get()
|
|
129
|
+
|
|
130
|
+
# Log a preview for huge byte blobs.
|
|
131
|
+
if isinstance(data, tuple) and len(data) == 2 and isinstance(data[1], bytes):
|
|
132
|
+
data_preview = (data[0], f"<{len(data[1])} bytes>")
|
|
133
|
+
else:
|
|
134
|
+
data_preview = data
|
|
135
|
+
logger.debug("[%s] recv() returning => %s", self.name, data_preview)
|
|
136
|
+
return data
|
|
137
|
+
|
|
138
|
+
def poll(self, timeout=0.0):
|
|
139
|
+
"""
|
|
140
|
+
Synchronously checks whether data is available.
|
|
141
|
+
Returns True if data is ready, or False otherwise.
|
|
142
|
+
"""
|
|
143
|
+
if self._closed:
|
|
144
|
+
return False
|
|
145
|
+
logger.debug("[%s] poll() requested with timeout: %s", self.name, timeout)
|
|
146
|
+
result_queue = queue.Queue()
|
|
147
|
+
request = {
|
|
148
|
+
"type": "POLL",
|
|
149
|
+
"timeout": timeout,
|
|
150
|
+
"result_queue": result_queue
|
|
151
|
+
}
|
|
152
|
+
self._request_queue.put(request)
|
|
153
|
+
try:
|
|
154
|
+
# Use a slightly longer timeout to give the worker a chance.
|
|
155
|
+
result = result_queue.get(timeout=timeout + 0.1)
|
|
156
|
+
except queue.Empty:
|
|
157
|
+
result = False
|
|
158
|
+
logger.debug("[%s] poll() returning => %s", self.name, result)
|
|
159
|
+
return result
|
|
160
|
+
|
|
161
|
+
def close(self):
|
|
162
|
+
"""
|
|
163
|
+
Closes the pipe and stops the worker thread. The _closed flag makes
|
|
164
|
+
sure no further operations are attempted.
|
|
165
|
+
"""
|
|
166
|
+
if self._closed:
|
|
167
|
+
return
|
|
168
|
+
logger.debug("[%s] close() called", self.name)
|
|
169
|
+
self._closed = True
|
|
170
|
+
stop_request = {"type": "CLOSE", "result_queue": queue.Queue()}
|
|
171
|
+
self._request_queue.put(stop_request)
|
|
172
|
+
self._stop_event.set()
|
|
173
|
+
self._worker_thread.join()
|
|
174
|
+
logger.debug("[%s] closed", self.name)
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def SafePipe(debug=False):
|
|
178
|
+
"""
|
|
179
|
+
Returns a pair: (thread-safe parent pipe, raw child pipe).
|
|
180
|
+
"""
|
|
181
|
+
parent_synthesize_pipe, child_synthesize_pipe = mp.Pipe()
|
|
182
|
+
parent_pipe = ParentPipe(parent_synthesize_pipe)
|
|
183
|
+
return parent_pipe, child_synthesize_pipe
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def child_process_code(child_end):
|
|
187
|
+
"""
|
|
188
|
+
Example child process code that receives messages, logs them,
|
|
189
|
+
sends acknowledgements, and then closes.
|
|
190
|
+
"""
|
|
191
|
+
for i in range(3):
|
|
192
|
+
msg = child_end.recv()
|
|
193
|
+
logger.debug("[Child] got: %s", msg)
|
|
194
|
+
child_end.send(f"ACK: {msg}")
|
|
195
|
+
child_end.close()
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
if __name__ == "__main__":
|
|
199
|
+
parent_pipe, child_pipe = SafePipe()
|
|
200
|
+
|
|
201
|
+
# Create child process with the child_process_code function.
|
|
202
|
+
p = mp.Process(target=child_process_code, args=(child_pipe,))
|
|
203
|
+
p.start()
|
|
204
|
+
|
|
205
|
+
# Event to signal sender threads to stop if needed.
|
|
206
|
+
stop_polling_event = threading.Event()
|
|
207
|
+
|
|
208
|
+
def sender_thread(n):
|
|
209
|
+
try:
|
|
210
|
+
parent_pipe.send(f"hello_from_thread_{n}")
|
|
211
|
+
except Exception as e:
|
|
212
|
+
logger.debug("[sender_thread_%s] send exception: %s", n, e)
|
|
213
|
+
return
|
|
214
|
+
|
|
215
|
+
# Use a poll loop with error handling.
|
|
216
|
+
for _ in range(10):
|
|
217
|
+
try:
|
|
218
|
+
if parent_pipe.poll(0.1):
|
|
219
|
+
reply = parent_pipe.recv()
|
|
220
|
+
logger.debug("[sender_thread_%s] got: %s", n, reply)
|
|
221
|
+
break
|
|
222
|
+
else:
|
|
223
|
+
logger.debug("[sender_thread_%s] no data yet...", n)
|
|
224
|
+
except (OSError, EOFError, BrokenPipeError) as e:
|
|
225
|
+
logger.debug("[sender_thread_%s] poll/recv exception: %s. Exiting thread.", n, e)
|
|
226
|
+
break
|
|
227
|
+
|
|
228
|
+
# Allow exit if a shutdown is signaled.
|
|
229
|
+
if stop_polling_event.is_set():
|
|
230
|
+
logger.debug("[sender_thread_%s] stop event set. Exiting thread.", n)
|
|
231
|
+
break
|
|
232
|
+
|
|
233
|
+
threads = []
|
|
234
|
+
for i in range(3):
|
|
235
|
+
t = threading.Thread(target=sender_thread, args=(i,))
|
|
236
|
+
t.start()
|
|
237
|
+
threads.append(t)
|
|
238
|
+
|
|
239
|
+
for t in threads:
|
|
240
|
+
t.join()
|
|
241
|
+
|
|
242
|
+
# Signal shutdown to any polling threads, then close the pipe.
|
|
243
|
+
stop_polling_event.set()
|
|
244
|
+
parent_pipe.close()
|
|
245
|
+
p.join()
|
|
@@ -64,6 +64,7 @@ stt-server [OPTIONS]
|
|
|
64
64
|
- `--handle_buffer_overflow`: Handle buffer overflow during transcription.
|
|
65
65
|
- `--suppress_tokens`: Suppress tokens during transcription.
|
|
66
66
|
- `--allowed_latency_limit`: Allowed latency limit for real-time transcription.
|
|
67
|
+
- `--faster_whisper_vad_filter`: Enable VAD filter for Faster Whisper; default False.
|
|
67
68
|
|
|
68
69
|
|
|
69
70
|
### WebSocket Interface:
|
|
@@ -81,6 +82,7 @@ from datetime import datetime
|
|
|
81
82
|
import logging
|
|
82
83
|
import asyncio
|
|
83
84
|
import pyaudio
|
|
85
|
+
import base64
|
|
84
86
|
import sys
|
|
85
87
|
|
|
86
88
|
|
|
@@ -185,6 +187,9 @@ allowed_parameters = [
|
|
|
185
187
|
'recording_stop_time',
|
|
186
188
|
'last_transcription_bytes',
|
|
187
189
|
'last_transcription_bytes_b64',
|
|
190
|
+
'speech_end_silence_start',
|
|
191
|
+
'is_recording',
|
|
192
|
+
'use_wake_words',
|
|
188
193
|
]
|
|
189
194
|
|
|
190
195
|
# Queues and connections for control and data
|
|
@@ -222,6 +227,23 @@ def debug_print(message):
|
|
|
222
227
|
thread_name = threading.current_thread().name
|
|
223
228
|
print(f"{Fore.CYAN}[DEBUG][{timestamp}][{thread_name}] {message}{Style.RESET_ALL}", file=sys.stderr)
|
|
224
229
|
|
|
230
|
+
def format_timestamp_ns(timestamp_ns: int) -> str:
|
|
231
|
+
# Split into whole seconds and the nanosecond remainder
|
|
232
|
+
seconds = timestamp_ns // 1_000_000_000
|
|
233
|
+
remainder_ns = timestamp_ns % 1_000_000_000
|
|
234
|
+
|
|
235
|
+
# Convert seconds part into a datetime object (local time)
|
|
236
|
+
dt = datetime.fromtimestamp(seconds)
|
|
237
|
+
|
|
238
|
+
# Format the main time as HH:MM:SS
|
|
239
|
+
time_str = dt.strftime("%H:%M:%S")
|
|
240
|
+
|
|
241
|
+
# For instance, if you want milliseconds, divide the remainder by 1e6 and format as 3-digit
|
|
242
|
+
milliseconds = remainder_ns // 1_000_000
|
|
243
|
+
formatted_timestamp = f"{time_str}.{milliseconds:03d}"
|
|
244
|
+
|
|
245
|
+
return formatted_timestamp
|
|
246
|
+
|
|
225
247
|
def text_detected(text, loop):
|
|
226
248
|
global prev_text
|
|
227
249
|
|
|
@@ -332,11 +354,28 @@ def on_wakeword_detection_end(loop):
|
|
|
332
354
|
asyncio.run_coroutine_threadsafe(audio_queue.put(message), loop)
|
|
333
355
|
|
|
334
356
|
def on_transcription_start(_audio_bytes, loop):
|
|
357
|
+
bytes_b64 = base64.b64encode(_audio_bytes.tobytes()).decode('utf-8')
|
|
358
|
+
message = json.dumps({
|
|
359
|
+
'type': 'transcription_start',
|
|
360
|
+
'audio_bytes_base64': bytes_b64
|
|
361
|
+
})
|
|
362
|
+
asyncio.run_coroutine_threadsafe(audio_queue.put(message), loop)
|
|
363
|
+
|
|
364
|
+
def on_turn_detection_start(loop):
|
|
365
|
+
print("&&& stt_server on_turn_detection_start")
|
|
366
|
+
message = json.dumps({
|
|
367
|
+
'type': 'start_turn_detection'
|
|
368
|
+
})
|
|
369
|
+
asyncio.run_coroutine_threadsafe(audio_queue.put(message), loop)
|
|
370
|
+
|
|
371
|
+
def on_turn_detection_stop(loop):
|
|
372
|
+
print("&&& stt_server on_turn_detection_stop")
|
|
335
373
|
message = json.dumps({
|
|
336
|
-
'type': '
|
|
374
|
+
'type': 'stop_turn_detection'
|
|
337
375
|
})
|
|
338
376
|
asyncio.run_coroutine_threadsafe(audio_queue.put(message), loop)
|
|
339
377
|
|
|
378
|
+
|
|
340
379
|
# def on_realtime_transcription_update(text, loop):
|
|
341
380
|
# # Send real-time transcription updates to the client
|
|
342
381
|
# text = preprocess_text(text)
|
|
@@ -385,6 +424,8 @@ def parse_arguments():
|
|
|
385
424
|
|
|
386
425
|
parser.add_argument('-D', '--debug', action='store_true', help='Enable debug logging for detailed server operations')
|
|
387
426
|
|
|
427
|
+
parser.add_argument('--debug_websockets', action='store_true', help='Enable debug logging for detailed server websocket operations')
|
|
428
|
+
|
|
388
429
|
parser.add_argument('-W', '--write', metavar='FILE', help='Save received audio to a WAV file')
|
|
389
430
|
|
|
390
431
|
parser.add_argument('-b', '--batch', '--batch_size', type=int, default=16, help='Batch size for inference. This parameter controls the number of audio chunks processed in parallel during transcription. Default is 16.')
|
|
@@ -492,6 +533,9 @@ def parse_arguments():
|
|
|
492
533
|
parser.add_argument('--allowed_latency_limit', type=int, default=100,
|
|
493
534
|
help='Maximal amount of chunks that can be unprocessed in queue before discarding chunks.. Default is 100.')
|
|
494
535
|
|
|
536
|
+
parser.add_argument('--faster_whisper_vad_filter', action='store_true',
|
|
537
|
+
help='Enable VAD filter for Faster Whisper. Default is False.')
|
|
538
|
+
|
|
495
539
|
parser.add_argument('--logchunks', action='store_true', help='Enable logging of incoming audio chunks (periods)')
|
|
496
540
|
|
|
497
541
|
# Parse arguments
|
|
@@ -503,12 +547,17 @@ def parse_arguments():
|
|
|
503
547
|
log_incoming_chunks = args.logchunks
|
|
504
548
|
dynamic_silence_timing = args.silence_timing
|
|
505
549
|
|
|
506
|
-
if debug_logging:
|
|
507
|
-
loglevel = logging.DEBUG
|
|
508
|
-
logging.basicConfig(level=loglevel, format='[%(asctime)s] %(levelname)s - %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
|
|
509
|
-
else:
|
|
510
|
-
loglevel = logging.WARNING
|
|
511
550
|
|
|
551
|
+
ws_logger = logging.getLogger('websockets')
|
|
552
|
+
if args.debug_websockets:
|
|
553
|
+
# If app debug is on, let websockets be verbose too
|
|
554
|
+
ws_logger.setLevel(logging.DEBUG)
|
|
555
|
+
# Ensure it uses the handler configured by basicConfig
|
|
556
|
+
ws_logger.propagate = False # Prevent duplicate messages if it also propagates to root
|
|
557
|
+
else:
|
|
558
|
+
# If app debug is off, silence websockets below WARNING
|
|
559
|
+
ws_logger.setLevel(logging.WARNING)
|
|
560
|
+
ws_logger.propagate = True # Allow WARNING/ERROR messages to reach root logger's handler
|
|
512
561
|
|
|
513
562
|
# Replace escaped newlines with actual newlines in initial_prompt
|
|
514
563
|
if args.initial_prompt:
|
|
@@ -675,7 +724,7 @@ async def data_handler(websocket):
|
|
|
675
724
|
while True:
|
|
676
725
|
message = await websocket.recv()
|
|
677
726
|
if isinstance(message, bytes):
|
|
678
|
-
if
|
|
727
|
+
if extended_logging:
|
|
679
728
|
debug_print(f"Received audio chunk (size: {len(message)} bytes)")
|
|
680
729
|
elif log_incoming_chunks:
|
|
681
730
|
print(".", end='', flush=True)
|
|
@@ -685,7 +734,14 @@ async def data_handler(websocket):
|
|
|
685
734
|
metadata = json.loads(metadata_json)
|
|
686
735
|
sample_rate = metadata['sampleRate']
|
|
687
736
|
|
|
688
|
-
|
|
737
|
+
if 'server_sent_to_stt' in metadata:
|
|
738
|
+
stt_received_ns = time.time_ns()
|
|
739
|
+
metadata["stt_received"] = stt_received_ns
|
|
740
|
+
metadata["stt_received_formatted"] = format_timestamp_ns(stt_received_ns)
|
|
741
|
+
print(f"Server received audio chunk of length {len(message)} bytes, metadata: {metadata}")
|
|
742
|
+
|
|
743
|
+
if extended_logging:
|
|
744
|
+
debug_print(f"Processing audio chunk with sample rate {sample_rate}")
|
|
689
745
|
chunk = message[4+metadata_length:]
|
|
690
746
|
|
|
691
747
|
if writechunks:
|
|
@@ -697,10 +753,13 @@ async def data_handler(websocket):
|
|
|
697
753
|
|
|
698
754
|
wav_file.writeframes(chunk)
|
|
699
755
|
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
756
|
+
if sample_rate != 16000:
|
|
757
|
+
resampled_chunk = decode_and_resample(chunk, sample_rate, 16000)
|
|
758
|
+
if extended_logging:
|
|
759
|
+
debug_print(f"Resampled chunk size: {len(resampled_chunk)} bytes")
|
|
760
|
+
recorder.feed_audio(resampled_chunk)
|
|
761
|
+
else:
|
|
762
|
+
recorder.feed_audio(chunk)
|
|
704
763
|
else:
|
|
705
764
|
print(f"{bcolors.WARNING}Received non-binary message on data connection{bcolors.ENDC}")
|
|
706
765
|
except websockets.exceptions.ConnectionClosed as e:
|
|
@@ -780,6 +839,9 @@ async def main_async():
|
|
|
780
839
|
'on_wakeword_detection_start': make_callback(loop, on_wakeword_detection_start),
|
|
781
840
|
'on_wakeword_detection_end': make_callback(loop, on_wakeword_detection_end),
|
|
782
841
|
'on_transcription_start': make_callback(loop, on_transcription_start),
|
|
842
|
+
'on_turn_detection_start': make_callback(loop, on_turn_detection_start),
|
|
843
|
+
'on_turn_detection_stop': make_callback(loop, on_turn_detection_stop),
|
|
844
|
+
|
|
783
845
|
# 'on_recorded_chunk': make_callback(loop, on_recorded_chunk),
|
|
784
846
|
'no_log_file': True, # Disable logging to file
|
|
785
847
|
'use_extended_logging': args.use_extended_logging,
|
|
@@ -790,6 +852,7 @@ async def main_async():
|
|
|
790
852
|
'handle_buffer_overflow': args.handle_buffer_overflow,
|
|
791
853
|
'suppress_tokens': args.suppress_tokens,
|
|
792
854
|
'allowed_latency_limit': args.allowed_latency_limit,
|
|
855
|
+
'faster_whisper_vad_filter': args.faster_whisper_vad_filter,
|
|
793
856
|
}
|
|
794
857
|
|
|
795
858
|
try:
|
|
@@ -14,7 +14,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
|
|
|
14
14
|
|
|
15
15
|
setuptools.setup(
|
|
16
16
|
name="realtimestt",
|
|
17
|
-
version="0.3.
|
|
17
|
+
version="0.3.102",
|
|
18
18
|
author="Kolja Beigel",
|
|
19
19
|
author_email="kolja.beigel@web.de",
|
|
20
20
|
description="A fast Voice Activity Detection and Transcription System",
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|