whisperlivekit 0.2.26__tar.gz → 0.2.28__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {whisperlivekit-0.2.26/whisperlivekit.egg-info → whisperlivekit-0.2.28}/PKG-INFO +21 -7
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/README.md +6 -1
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/pyproject.toml +62 -8
- whisperlivekit-0.2.28/tests/test_asr_coalescing.py +33 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/tests/test_asr_coalescing_boundaries.py +3 -0
- whisperlivekit-0.2.28/tests/test_asr_nemotron_mlx.py +297 -0
- whisperlivekit-0.2.26/tests/test_backend_deep_bugs.py → whisperlivekit-0.2.28/tests/test_backends.py +122 -446
- whisperlivekit-0.2.28/tests/test_benchmark.py +158 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/tests/test_canary_backend.py +0 -19
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/tests/test_deepgram_compat.py +12 -2
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/tests/test_funasr_backend.py +0 -6
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/tests/test_pause_segmentation.py +11 -25
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/tests/test_pipeline.py +1 -2
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/tests/test_retention.py +0 -9
- whisperlivekit-0.2.28/tests/test_server.py +673 -0
- whisperlivekit-0.2.28/tests/test_session_context.py +267 -0
- whisperlivekit-0.2.28/tests/test_session_limits.py +312 -0
- whisperlivekit-0.2.28/tests/test_silent_backend_guard.py +292 -0
- whisperlivekit-0.2.28/tests/test_simul_calibration_files.py +163 -0
- whisperlivekit-0.2.28/tests/test_sortformer_event_loop.py +75 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/tests/test_sortformer_max_speakers.py +0 -10
- whisperlivekit-0.2.28/tests/test_streaming_lifecycle.py +311 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/tests/test_translation_alignatt.py +44 -23
- whisperlivekit-0.2.28/tests/test_translation_mlx.py +324 -0
- whisperlivekit-0.2.28/whisperlivekit/__init__.py +27 -0
- whisperlivekit-0.2.28/whisperlivekit/asr_nemotron_mlx.py +217 -0
- whisperlivekit-0.2.28/whisperlivekit/audio_input.py +98 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/audio_processor.py +125 -212
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/backend_support.py +13 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/basic_server.py +259 -113
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/benchmark/compat.py +25 -5
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/benchmark/datasets.py +13 -0
- whisperlivekit-0.2.28/whisperlivekit/benchmark/metrics.py +271 -0
- whisperlivekit-0.2.28/whisperlivekit/benchmark/report.py +49 -0
- whisperlivekit-0.2.28/whisperlivekit/benchmark/resources.py +53 -0
- whisperlivekit-0.2.28/whisperlivekit/benchmark/runner.py +164 -0
- whisperlivekit-0.2.28/whisperlivekit/calibrations/translation_heads_mlx-community_Hy-MT2-1_8B-8bit_en-zh.json +4262 -0
- whisperlivekit-0.2.28/whisperlivekit/calibrations/translation_heads_mlx-community_Hy-MT2-1_8B-8bit_zh-en.json +824 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/cli.py +32 -10
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/config.py +42 -2
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/core.py +71 -17
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/deepgram_compat.py +6 -1
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/diarization/sortformer_backend.py +7 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/diff_protocol.py +1 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/ffmpeg_manager.py +100 -65
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/local_agreement/backends.py +19 -6
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/metrics_collector.py +13 -11
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/parse_args.py +75 -7
- whisperlivekit-0.2.28/whisperlivekit/processing_queue.py +146 -0
- whisperlivekit-0.2.28/whisperlivekit/session_asr_proxy.py +159 -0
- whisperlivekit-0.2.28/whisperlivekit/session_limits.py +40 -0
- whisperlivekit-0.2.28/whisperlivekit/simul_mt_calibration.py +68 -0
- whisperlivekit-0.2.28/whisperlivekit/simul_mt_capture.py +216 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/simul_whisper/backend.py +5 -1
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/simul_whisper/decoder_state.py +2 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/simul_whisper/mlx/decoder_state.py +2 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/simul_whisper/mlx/simul_whisper.py +4 -1
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/simul_whisper/simul_whisper.py +4 -1
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/test_client.py +11 -3
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/test_harness.py +49 -34
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/timed_objects.py +12 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/translation.py +11 -5
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/translation_alignatt.py +65 -32
- whisperlivekit-0.2.28/whisperlivekit/translation_hunyuan_mlx.py +13 -0
- whisperlivekit-0.2.28/whisperlivekit/translation_mlx_llm_mt.py +317 -0
- whisperlivekit-0.2.28/whisperlivekit/translation_mlx_llm_mt_simul.py +214 -0
- whisperlivekit-0.2.28/whisperlivekit/translation_processor.py +66 -0
- whisperlivekit-0.2.28/whisperlivekit/translation_profiles.py +164 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/web/live_transcription.js +28 -13
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/whisper/__init__.py +5 -5
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28/whisperlivekit.egg-info}/PKG-INFO +21 -7
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit.egg-info/SOURCES.txt +24 -2
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit.egg-info/requires.txt +22 -5
- whisperlivekit-0.2.26/tests/test_asr_coalescing.py +0 -55
- whisperlivekit-0.2.26/tests/test_silent_backend_guard.py +0 -109
- whisperlivekit-0.2.26/whisperlivekit/__init__.py +0 -20
- whisperlivekit-0.2.26/whisperlivekit/benchmark/metrics.py +0 -273
- whisperlivekit-0.2.26/whisperlivekit/benchmark/report.py +0 -160
- whisperlivekit-0.2.26/whisperlivekit/benchmark/runner.py +0 -180
- whisperlivekit-0.2.26/whisperlivekit/session_asr_proxy.py +0 -41
- whisperlivekit-0.2.26/whisperlivekit/whisper/val.py +0 -201
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/LICENSE +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/MANIFEST.in +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/setup.cfg +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/tests/fixtures/sortformer_2spk/6930-75918-0000.flac +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/tests/fixtures/sortformer_2spk/7902-96591-0000.flac +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/tests/fixtures/sortformer_2spk/LICENSE.txt +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/tests/fixtures/sortformer_2spk/README.md +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/tests/test_asr_coalescing_pipeline.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/tests/test_funasr_backend_integration.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/tests/test_qwen3_backend_shims.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/tests/test_sortformer_real_fixture.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/tests/test_speaker_boundaries.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/api_auth.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/benchmark/__init__.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/canary_backend.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/diarization/__init__.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/diarization/diart_backend.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/diarization/utils.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/funasr_backend.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/local_agreement/__init__.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/local_agreement/online_asr.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/local_agreement/whisper_online.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/metrics.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/model_mapping.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/model_paths.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/qwen3_streaming/__init__.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/qwen3_streaming/_shim.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/qwen3_streaming/asr.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/qwen3_streaming/causal.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/qwen3_streaming/features.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/qwen3_streaming/model.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/qwen3_streaming/model_config.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/qwen3_streaming/online.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/qwen3_streaming/stable_commit.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/qwen3_streaming/streamer.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/qwen3_vllm_asr.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/qwen3_vllm_metal_asr.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/silero_vad_iterator.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/silero_vad_models/__init__.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/silero_vad_models/silero_vad.jit +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/silero_vad_models/silero_vad.onnx +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/silero_vad_models/silero_vad_16k_op15.onnx +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/silero_vad_models/silero_vad_half.onnx +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/simul_whisper/__init__.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/simul_whisper/align_att_base.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/simul_whisper/beam.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/simul_whisper/config.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/simul_whisper/eow_detection.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/simul_whisper/mlx/__init__.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/simul_whisper/mlx/decoders.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/simul_whisper/mlx_encoder.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/simul_whisper/token_buffer.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/test_data.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/thread_safety.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/tokens_alignment.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/voxtral_hf_streaming.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/voxtral_mlx/__init__.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/voxtral_mlx/loader.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/voxtral_mlx/model.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/voxtral_mlx/spectrogram.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/voxtral_mlx_asr.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/warmup.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/web/__init__.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/web/live_transcription.css +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/web/live_transcription.html +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/web/pcm_worklet.js +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/web/recorder_worker.js +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/web/src/dark_mode.svg +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/web/src/language.svg +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/web/src/light_mode.svg +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/web/src/settings.svg +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/web/src/silence.svg +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/web/src/speaker.svg +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/web/src/system_mode.svg +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/web/src/translate.svg +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/web/web_interface.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/whisper/__main__.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/whisper/assets/__init__.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/whisper/assets/gpt2.tiktoken +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/whisper/assets/mel_filters.npz +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/whisper/assets/multilingual.tiktoken +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/whisper/audio.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/whisper/decoding.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/whisper/model.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/whisper/normalizers/__init__.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/whisper/normalizers/basic.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/whisper/normalizers/english.json +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/whisper/normalizers/english.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/whisper/timing.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/whisper/tokenizer.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/whisper/transcribe.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/whisper/triton_ops.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/whisper/utils.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit/whisper/version.py +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit.egg-info/dependency_links.txt +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit.egg-info/entry_points.txt +0 -0
- {whisperlivekit-0.2.26 → whisperlivekit-0.2.28}/whisperlivekit.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: whisperlivekit
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.28
|
|
4
4
|
Summary: Real-time speech-to-text models
|
|
5
5
|
Author: Quentin Fuxa
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -22,7 +22,7 @@ Requires-Dist: uvicorn
|
|
|
22
22
|
Requires-Dist: websockets
|
|
23
23
|
Requires-Dist: huggingface-hub>=0.25.0
|
|
24
24
|
Requires-Dist: faster-whisper>=1.2.0
|
|
25
|
-
Requires-Dist: torch>=2.
|
|
25
|
+
Requires-Dist: torch>=2.6.0
|
|
26
26
|
Requires-Dist: torchaudio>=2.0.0
|
|
27
27
|
Requires-Dist: tqdm
|
|
28
28
|
Requires-Dist: tiktoken
|
|
@@ -33,7 +33,9 @@ Requires-Dist: pytest>=7.0; extra == "test"
|
|
|
33
33
|
Requires-Dist: pytest-asyncio>=0.21; extra == "test"
|
|
34
34
|
Requires-Dist: datasets>=2.14; extra == "test"
|
|
35
35
|
Requires-Dist: librosa; extra == "test"
|
|
36
|
-
Requires-Dist: deepgram-sdk==7.
|
|
36
|
+
Requires-Dist: deepgram-sdk==7.8.1; extra == "test"
|
|
37
|
+
Requires-Dist: psutil>=5.9; extra == "test"
|
|
38
|
+
Requires-Dist: matplotlib>=3.8; extra == "test"
|
|
37
39
|
Provides-Extra: translation
|
|
38
40
|
Requires-Dist: nllw; extra == "translation"
|
|
39
41
|
Provides-Extra: sentence-tokenizer
|
|
@@ -48,12 +50,19 @@ Provides-Extra: voxtral-mlx
|
|
|
48
50
|
Requires-Dist: mlx>=0.11.0; (sys_platform == "darwin" and platform_machine == "arm64") and extra == "voxtral-mlx"
|
|
49
51
|
Requires-Dist: mlx-whisper>=0.4.0; (sys_platform == "darwin" and platform_machine == "arm64") and extra == "voxtral-mlx"
|
|
50
52
|
Requires-Dist: mistral-common[audio]; extra == "voxtral-mlx"
|
|
53
|
+
Provides-Extra: mlx-llm-mt
|
|
54
|
+
Requires-Dist: mlx<0.32,>=0.31.2; (sys_platform == "darwin" and platform_machine == "arm64") and extra == "mlx-llm-mt"
|
|
55
|
+
Requires-Dist: mlx-lm<0.32,>=0.31.3; (sys_platform == "darwin" and platform_machine == "arm64") and extra == "mlx-llm-mt"
|
|
56
|
+
Provides-Extra: nemotron-mlx-asr
|
|
57
|
+
Requires-Dist: mlx>=0.11.0; (sys_platform == "darwin" and platform_machine == "arm64") and extra == "nemotron-mlx-asr"
|
|
58
|
+
Requires-Dist: mlx-audio<0.6,>=0.5.1; (sys_platform == "darwin" and platform_machine == "arm64") and extra == "nemotron-mlx-asr"
|
|
51
59
|
Provides-Extra: voxtral-hf
|
|
52
60
|
Requires-Dist: transformers>=5.2.0; python_version >= "3.10" and extra == "voxtral-hf"
|
|
53
61
|
Requires-Dist: mistral-common[audio]; extra == "voxtral-hf"
|
|
54
62
|
Requires-Dist: accelerate>=0.12; extra == "voxtral-hf"
|
|
55
63
|
Provides-Extra: qwen3-vllm-metal
|
|
56
|
-
Requires-Dist: qwen3-asr-causal
|
|
64
|
+
Requires-Dist: qwen3-asr-causal>=0.1.0; extra == "qwen3-vllm-metal"
|
|
65
|
+
Requires-Dist: mlx>=0.11.0; (sys_platform == "darwin" and platform_machine == "arm64") and extra == "qwen3-vllm-metal"
|
|
57
66
|
Provides-Extra: qwen3-vllm
|
|
58
67
|
Requires-Dist: qwen3-asr-causal[vllm]>=0.1.0; extra == "qwen3-vllm"
|
|
59
68
|
Provides-Extra: qwen3-streaming
|
|
@@ -61,10 +70,10 @@ Requires-Dist: qwen3-asr-causal[streaming]>=0.1.0; extra == "qwen3-streaming"
|
|
|
61
70
|
Provides-Extra: listen
|
|
62
71
|
Requires-Dist: sounddevice>=0.4.6; extra == "listen"
|
|
63
72
|
Provides-Extra: cpu
|
|
64
|
-
Requires-Dist: torch>=2.
|
|
73
|
+
Requires-Dist: torch>=2.6.0; extra == "cpu"
|
|
65
74
|
Requires-Dist: torchaudio>=2.0.0; extra == "cpu"
|
|
66
75
|
Provides-Extra: cu129
|
|
67
|
-
Requires-Dist: torch>=2.
|
|
76
|
+
Requires-Dist: torch>=2.6.0; extra == "cu129"
|
|
68
77
|
Requires-Dist: torchaudio>=2.0.0; extra == "cu129"
|
|
69
78
|
Requires-Dist: triton>=2.0.0; (platform_machine == "x86_64" and (sys_platform == "linux" or sys_platform == "linux2")) and extra == "cu129"
|
|
70
79
|
Provides-Extra: diarization-sortformer
|
|
@@ -104,6 +113,10 @@ Dynamic: license-file
|
|
|
104
113
|
</p>
|
|
105
114
|
|
|
106
115
|
|
|
116
|
+
<p align="center">
|
|
117
|
+
<a href="https://trendshift.io/repositories/14685?utm_source=repository-badge&utm_medium=badge&utm_campaign=badge-repository-14685" target="_blank" rel="noopener noreferrer"><img src="https://trendshift.io/api/badge/repositories/14685" alt="QuentinFuxa%2FWhisperLiveKit | Trendshift" width="250" height="55"/></a>
|
|
118
|
+
</p>
|
|
119
|
+
|
|
107
120
|
### Powered by Leading Research:
|
|
108
121
|
|
|
109
122
|
- Simul-[Whisper](https://arxiv.org/pdf/2406.10052)/[Streaming](https://arxiv.org/abs/2506.17077) (SOTA 2025) - Ultra-low latency transcription using [AlignAtt policy](https://arxiv.org/pdf/2305.11408).
|
|
@@ -179,6 +192,7 @@ Per-session WebSocket query parameters:
|
|
|
179
192
|
|---|---|---|
|
|
180
193
|
| `language` | `?language=fr` | transcription language for this session (one shared engine serves mixed-language sessions) |
|
|
181
194
|
| `target_language` | `?target_language=de` | translation target for this session (server must run with `--target-language`) |
|
|
195
|
+
| `context` | `?context=WhisperLiveKit%2C+Qwen3-ASR` | terminology, names, or phrase-list text used to condition this session; supported by Whisper-family and SimulStreaming backends |
|
|
182
196
|
| `mode` | `?mode=diff` | incremental snapshot/diff protocol instead of resending the full state (experimental, for integrators building their own client, see `diff_protocol.py`); the bundled web UI uses `full` |
|
|
183
197
|
| `token` | `?token=...` | API token when the server runs with `--api-token` (also accepted as an `Authorization: Bearer` header) |
|
|
184
198
|
|
|
@@ -206,7 +220,7 @@ For a native SwiftUI macOS client, see [macos/WhisperLiveKitMac](macos/WhisperLi
|
|
|
206
220
|
| **Voxtral (HF backend)** | `uv sync --extra voxtral-hf` | `pip install -e ".[voxtral-hf]"` |
|
|
207
221
|
| **Qwen3-ASR vLLM (CUDA)** | `uv sync --extra qwen3-vllm` | `pip install -e ".[qwen3-vllm]"` |
|
|
208
222
|
| **Qwen3-ASR streaming (HF, CUDA/MPS/CPU)** | `uv sync --extra qwen3-streaming` | `pip install -e ".[qwen3-streaming]"` |
|
|
209
|
-
| **Qwen3-ASR vLLM Metal (Apple Silicon)** | Install
|
|
223
|
+
| **Qwen3-ASR vLLM Metal (Apple Silicon)** | Install vllm-metal with its official installer, activate `~/.venv-vllm-metal`, then `uv pip install -e ".[qwen3-vllm-metal]"` | Install vllm-metal with its official installer, activate `~/.venv-vllm-metal`, then `pip install -e ".[qwen3-vllm-metal]"` |
|
|
210
224
|
| **Speaker diarization (Sortformer / NeMo 3)** | `uv sync --extra diarization-sortformer` | `pip install -e ".[diarization-sortformer]"` |
|
|
211
225
|
| *[Not recommended]* Speaker diarization with Diart (Python 3.11 or 3.12) | `uv sync --extra diarization-diart` | `pip install -e ".[diarization-diart]"` |
|
|
212
226
|
| **Canary-1b-v2 (NeMo, CUDA/CPU)** | `uv sync --extra canary` | `pip install -e ".[canary]"` |
|
|
@@ -20,6 +20,10 @@
|
|
|
20
20
|
</p>
|
|
21
21
|
|
|
22
22
|
|
|
23
|
+
<p align="center">
|
|
24
|
+
<a href="https://trendshift.io/repositories/14685?utm_source=repository-badge&utm_medium=badge&utm_campaign=badge-repository-14685" target="_blank" rel="noopener noreferrer"><img src="https://trendshift.io/api/badge/repositories/14685" alt="QuentinFuxa%2FWhisperLiveKit | Trendshift" width="250" height="55"/></a>
|
|
25
|
+
</p>
|
|
26
|
+
|
|
23
27
|
### Powered by Leading Research:
|
|
24
28
|
|
|
25
29
|
- Simul-[Whisper](https://arxiv.org/pdf/2406.10052)/[Streaming](https://arxiv.org/abs/2506.17077) (SOTA 2025) - Ultra-low latency transcription using [AlignAtt policy](https://arxiv.org/pdf/2305.11408).
|
|
@@ -95,6 +99,7 @@ Per-session WebSocket query parameters:
|
|
|
95
99
|
|---|---|---|
|
|
96
100
|
| `language` | `?language=fr` | transcription language for this session (one shared engine serves mixed-language sessions) |
|
|
97
101
|
| `target_language` | `?target_language=de` | translation target for this session (server must run with `--target-language`) |
|
|
102
|
+
| `context` | `?context=WhisperLiveKit%2C+Qwen3-ASR` | terminology, names, or phrase-list text used to condition this session; supported by Whisper-family and SimulStreaming backends |
|
|
98
103
|
| `mode` | `?mode=diff` | incremental snapshot/diff protocol instead of resending the full state (experimental, for integrators building their own client, see `diff_protocol.py`); the bundled web UI uses `full` |
|
|
99
104
|
| `token` | `?token=...` | API token when the server runs with `--api-token` (also accepted as an `Authorization: Bearer` header) |
|
|
100
105
|
|
|
@@ -122,7 +127,7 @@ For a native SwiftUI macOS client, see [macos/WhisperLiveKitMac](macos/WhisperLi
|
|
|
122
127
|
| **Voxtral (HF backend)** | `uv sync --extra voxtral-hf` | `pip install -e ".[voxtral-hf]"` |
|
|
123
128
|
| **Qwen3-ASR vLLM (CUDA)** | `uv sync --extra qwen3-vllm` | `pip install -e ".[qwen3-vllm]"` |
|
|
124
129
|
| **Qwen3-ASR streaming (HF, CUDA/MPS/CPU)** | `uv sync --extra qwen3-streaming` | `pip install -e ".[qwen3-streaming]"` |
|
|
125
|
-
| **Qwen3-ASR vLLM Metal (Apple Silicon)** | Install
|
|
130
|
+
| **Qwen3-ASR vLLM Metal (Apple Silicon)** | Install vllm-metal with its official installer, activate `~/.venv-vllm-metal`, then `uv pip install -e ".[qwen3-vllm-metal]"` | Install vllm-metal with its official installer, activate `~/.venv-vllm-metal`, then `pip install -e ".[qwen3-vllm-metal]"` |
|
|
126
131
|
| **Speaker diarization (Sortformer / NeMo 3)** | `uv sync --extra diarization-sortformer` | `pip install -e ".[diarization-sortformer]"` |
|
|
127
132
|
| *[Not recommended]* Speaker diarization with Diart (Python 3.11 or 3.12) | `uv sync --extra diarization-diart` | `pip install -e ".[diarization-diart]"` |
|
|
128
133
|
| **Canary-1b-v2 (NeMo, CUDA/CPU)** | `uv sync --extra canary` | `pip install -e ".[canary]"` |
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "whisperlivekit"
|
|
7
|
-
version = "0.2.
|
|
7
|
+
version = "0.2.28"
|
|
8
8
|
description = "Real-time speech-to-text models"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
authors = [{ name = "Quentin Fuxa" }]
|
|
@@ -28,7 +28,7 @@ dependencies = [
|
|
|
28
28
|
"websockets",
|
|
29
29
|
"huggingface-hub>=0.25.0",
|
|
30
30
|
"faster-whisper>=1.2.0",
|
|
31
|
-
"torch>=2.
|
|
31
|
+
"torch>=2.6.0",
|
|
32
32
|
"torchaudio>=2.0.0",
|
|
33
33
|
"tqdm",
|
|
34
34
|
"tiktoken",
|
|
@@ -42,7 +42,9 @@ test = [
|
|
|
42
42
|
"pytest-asyncio>=0.21",
|
|
43
43
|
"datasets>=2.14",
|
|
44
44
|
"librosa",
|
|
45
|
-
"deepgram-sdk==7.
|
|
45
|
+
"deepgram-sdk==7.8.1",
|
|
46
|
+
"psutil>=5.9",
|
|
47
|
+
"matplotlib>=3.8",
|
|
46
48
|
]
|
|
47
49
|
translation = ["nllw"]
|
|
48
50
|
sentence_tokenizer = ["mosestokenizer", "wtpsplit"]
|
|
@@ -56,13 +58,30 @@ voxtral-mlx = [
|
|
|
56
58
|
'mlx-whisper>=0.4.0; sys_platform == "darwin" and platform_machine == "arm64"',
|
|
57
59
|
"mistral-common[audio]",
|
|
58
60
|
]
|
|
61
|
+
# In-process MLX translation backend via mlx-lm (Hunyuan-MT as first config;
|
|
62
|
+
# TranslateGemma etc. via the config registry). Requires mlx-lm>=0.31.3 which
|
|
63
|
+
# pulls transformers>=5; conflicts with qwen3-asr-causal[streaming] which pins
|
|
64
|
+
# transformers==4.57.6. Declare the conflict below; install only one.
|
|
65
|
+
mlx-llm-mt = [
|
|
66
|
+
'mlx>=0.31.2,<0.32; sys_platform == "darwin" and platform_machine == "arm64"',
|
|
67
|
+
'mlx-lm>=0.31.3,<0.32; sys_platform == "darwin" and platform_machine == "arm64"',
|
|
68
|
+
]
|
|
69
|
+
nemotron-mlx-asr = [
|
|
70
|
+
'mlx>=0.11.0; sys_platform == "darwin" and platform_machine == "arm64"',
|
|
71
|
+
'mlx-audio>=0.5.1,<0.6; sys_platform == "darwin" and platform_machine == "arm64"',
|
|
72
|
+
]
|
|
59
73
|
voxtral-hf = [
|
|
60
74
|
"transformers>=5.2.0; python_version >= '3.10'",
|
|
61
75
|
"mistral-common[audio]",
|
|
62
76
|
"accelerate>=0.12",
|
|
63
77
|
]
|
|
78
|
+
# vllm-metal and its CPU vLLM runtime are distributed outside PyPI; install them
|
|
79
|
+
# with the official vllm-metal installer, then install this extra into that
|
|
80
|
+
# environment. Keeping the GitHub release wheel out of the lock lets Dependabot
|
|
81
|
+
# resolve uv updates.
|
|
64
82
|
qwen3-vllm-metal = [
|
|
65
|
-
"qwen3-asr-causal
|
|
83
|
+
"qwen3-asr-causal>=0.1.0",
|
|
84
|
+
'mlx>=0.11.0; sys_platform == "darwin" and platform_machine == "arm64"',
|
|
66
85
|
]
|
|
67
86
|
qwen3-vllm = [
|
|
68
87
|
"qwen3-asr-causal[vllm]>=0.1.0",
|
|
@@ -71,9 +90,9 @@ qwen3-streaming = [
|
|
|
71
90
|
"qwen3-asr-causal[streaming]>=0.1.0",
|
|
72
91
|
]
|
|
73
92
|
listen = ["sounddevice>=0.4.6"]
|
|
74
|
-
cpu = ["torch>=2.
|
|
93
|
+
cpu = ["torch>=2.6.0", "torchaudio>=2.0.0"]
|
|
75
94
|
cu129 = [
|
|
76
|
-
"torch>=2.
|
|
95
|
+
"torch>=2.6.0",
|
|
77
96
|
"torchaudio>=2.0.0",
|
|
78
97
|
'triton>=2.0.0; platform_machine == "x86_64" and (sys_platform == "linux" or sys_platform == "linux2")',
|
|
79
98
|
]
|
|
@@ -98,6 +117,18 @@ diarization-diart = [
|
|
|
98
117
|
dev = ["rich>=14.3.3", "ruff==0.16.*"]
|
|
99
118
|
|
|
100
119
|
[tool.uv]
|
|
120
|
+
# Resolve Diart's Python <3.13 dependency graph separately from Python 3.13.
|
|
121
|
+
# Only lock the platforms Torch publishes wheels for. uv 0.13 (used by
|
|
122
|
+
# Dependabot and the Docker images) rejects a fork that has no Torch wheel, and
|
|
123
|
+
# dependency markers otherwise create emscripten and s390x forks.
|
|
124
|
+
environments = [
|
|
125
|
+
"python_version < '3.13' and sys_platform == 'linux' and (platform_machine == 'x86_64' or platform_machine == 'aarch64')",
|
|
126
|
+
"python_version < '3.13' and sys_platform == 'darwin' and platform_machine == 'arm64'",
|
|
127
|
+
"python_version < '3.13' and sys_platform == 'win32' and platform_machine == 'AMD64'",
|
|
128
|
+
"python_version >= '3.13' and sys_platform == 'linux' and (platform_machine == 'x86_64' or platform_machine == 'aarch64')",
|
|
129
|
+
"python_version >= '3.13' and sys_platform == 'darwin' and platform_machine == 'arm64'",
|
|
130
|
+
"python_version >= '3.13' and sys_platform == 'win32' and platform_machine == 'AMD64'",
|
|
131
|
+
]
|
|
101
132
|
conflicts = [
|
|
102
133
|
[
|
|
103
134
|
{ extra = "cpu" },
|
|
@@ -155,11 +186,34 @@ conflicts = [
|
|
|
155
186
|
{ extra = "canary" },
|
|
156
187
|
{ extra = "qwen3-vllm-metal" },
|
|
157
188
|
],
|
|
189
|
+
[
|
|
190
|
+
{ extra = "mlx-llm-mt" },
|
|
191
|
+
{ extra = "qwen3-streaming" },
|
|
192
|
+
],
|
|
193
|
+
[
|
|
194
|
+
{ extra = "mlx-llm-mt" },
|
|
195
|
+
{ extra = "qwen3-vllm" },
|
|
196
|
+
],
|
|
197
|
+
[
|
|
198
|
+
{ extra = "mlx-llm-mt" },
|
|
199
|
+
{ extra = "qwen3-vllm-metal" },
|
|
200
|
+
],
|
|
201
|
+
[
|
|
202
|
+
{ extra = "nemotron-mlx-asr" },
|
|
203
|
+
{ extra = "qwen3-streaming" },
|
|
204
|
+
],
|
|
205
|
+
[
|
|
206
|
+
{ extra = "nemotron-mlx-asr" },
|
|
207
|
+
{ extra = "qwen3-vllm" },
|
|
208
|
+
],
|
|
209
|
+
[
|
|
210
|
+
{ extra = "nemotron-mlx-asr" },
|
|
211
|
+
{ extra = "qwen3-vllm-metal" },
|
|
212
|
+
],
|
|
158
213
|
]
|
|
159
214
|
|
|
160
215
|
[tool.uv.sources]
|
|
161
216
|
qwen3-asr-causal = { path = "third_party/qwen3-asr-causal", editable = true }
|
|
162
|
-
vllm-metal = { url = "https://github.com/vllm-project/vllm-metal/releases/download/v0.2.0-20260519-054638/vllm_metal-0.2.0-cp312-cp312-macosx_11_0_arm64.whl" }
|
|
163
217
|
torch = [
|
|
164
218
|
{ index = "pytorch-cpu", extra = "cpu", marker = "platform_system != 'Darwin'" },
|
|
165
219
|
{ index = "pytorch-cpu", extra = "diarization-diart", marker = "platform_system != 'Darwin'" },
|
|
@@ -220,7 +274,7 @@ packages = [
|
|
|
220
274
|
]
|
|
221
275
|
|
|
222
276
|
[tool.setuptools.package-data]
|
|
223
|
-
whisperlivekit = ["web/*.html", "web/*.css", "web/*.js", "web/src/*.svg"]
|
|
277
|
+
whisperlivekit = ["web/*.html", "web/*.css", "web/*.js", "web/src/*.svg", "calibrations/*.json"]
|
|
224
278
|
"whisperlivekit.whisper.assets" = ["*.tiktoken", "*.npz"]
|
|
225
279
|
"whisperlivekit.whisper.normalizers" = ["*.json"]
|
|
226
280
|
"whisperlivekit.silero_vad_models" = ["*.jit", "*.onnx"]
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
"""ASR call coalescing: the deferral gate and its opt-in defaults."""
|
|
2
|
+
|
|
3
|
+
import math
|
|
4
|
+
|
|
5
|
+
from whisperlivekit.audio_processor import (
|
|
6
|
+
resolve_coalesce_min_s,
|
|
7
|
+
should_defer_inference,
|
|
8
|
+
)
|
|
9
|
+
from whisperlivekit.config import WhisperLiveKitConfig
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def test_disabled_or_invalid_windows_never_defer_audio(caplog):
|
|
13
|
+
windows = [WhisperLiveKitConfig().asr_coalesce_min_s, None, 0, -1,
|
|
14
|
+
math.nan, math.inf, -math.inf]
|
|
15
|
+
for window in windows:
|
|
16
|
+
caplog.clear()
|
|
17
|
+
with caplog.at_level("WARNING"):
|
|
18
|
+
threshold = resolve_coalesce_min_s(window)
|
|
19
|
+
assert not should_defer_inference(0, .5, threshold), window
|
|
20
|
+
assert not should_defer_inference(1000, .5, threshold), window
|
|
21
|
+
if window is not None and (window < 0 or not math.isfinite(window)):
|
|
22
|
+
assert "coalescing disabled" in caplog.text, window
|
|
23
|
+
else:
|
|
24
|
+
assert not caplog.records, window
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def test_deferral_gate_honors_the_configured_window():
|
|
28
|
+
threshold = resolve_coalesce_min_s(0.75)
|
|
29
|
+
assert should_defer_inference(0.0, 0.5, threshold) is True
|
|
30
|
+
assert should_defer_inference(0.5, 0.5, threshold) is False
|
|
31
|
+
assert should_defer_inference(0.0, 0.75, threshold) is False
|
|
32
|
+
assert should_defer_inference(0.0, 3.0, threshold) is False
|
|
33
|
+
assert should_defer_inference(0.0, 0.04, 0.0) is False
|
|
@@ -121,6 +121,9 @@ def _assert_asr_metrics(processor, *, calls, tokens):
|
|
|
121
121
|
"""Every successful backend invocation has one latency and unique tokens."""
|
|
122
122
|
assert processor.metrics.n_transcription_calls == calls
|
|
123
123
|
assert len(processor.metrics.transcription_durations) == calls
|
|
124
|
+
assert processor.metrics.total_processing_time_s == pytest.approx(
|
|
125
|
+
sum(processor.metrics.transcription_durations)
|
|
126
|
+
)
|
|
124
127
|
assert all(duration >= 0.0 for duration in processor.metrics.transcription_durations)
|
|
125
128
|
assert processor.metrics.n_tokens_produced == tokens
|
|
126
129
|
|
|
@@ -0,0 +1,297 @@
|
|
|
1
|
+
"""Exercise PCM buffering, RNNT commits and session boundaries on the MLX CPU device."""
|
|
2
|
+
import threading
|
|
3
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
4
|
+
from types import SimpleNamespace
|
|
5
|
+
|
|
6
|
+
import numpy as np
|
|
7
|
+
import pytest
|
|
8
|
+
|
|
9
|
+
mx = pytest.importorskip('mlx.core')
|
|
10
|
+
pytest.importorskip('mlx_audio')
|
|
11
|
+
|
|
12
|
+
from whisperlivekit.asr_nemotron_mlx import NemotronMLXOnlineProcessor # noqa: E402
|
|
13
|
+
from whisperlivekit.core import online_factory # noqa: E402
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@pytest.fixture
|
|
17
|
+
def shared(monkeypatch):
|
|
18
|
+
from mlx_audio.stt.models.nemotron_asr import audio, streaming
|
|
19
|
+
|
|
20
|
+
previous_device = mx.default_device()
|
|
21
|
+
mx.set_default_device(mx.cpu)
|
|
22
|
+
|
|
23
|
+
class Mel:
|
|
24
|
+
def __init__(self, config):
|
|
25
|
+
self.samples = np.empty(0)
|
|
26
|
+
|
|
27
|
+
def push(self, samples, *, final):
|
|
28
|
+
self.samples = np.concatenate([self.samples, np.array(samples)])
|
|
29
|
+
frames = len(self.samples) // 1280
|
|
30
|
+
if final and len(self.samples) % 1280:
|
|
31
|
+
frames += 1
|
|
32
|
+
emitted = [self.samples[i*1280] for i in range(frames)]
|
|
33
|
+
self.samples = self.samples[frames*1280:]
|
|
34
|
+
return mx.array(emitted).reshape(1, frames, 1)
|
|
35
|
+
|
|
36
|
+
class Encoder:
|
|
37
|
+
def __init__(self, encoder, **kwargs):
|
|
38
|
+
pass
|
|
39
|
+
|
|
40
|
+
def push(self, mel, *, final):
|
|
41
|
+
return [mel] if mel.shape[1] else []
|
|
42
|
+
|
|
43
|
+
def materialize(self, *arrays):
|
|
44
|
+
mx.eval(*arrays)
|
|
45
|
+
|
|
46
|
+
class Model:
|
|
47
|
+
thread_ids = set()
|
|
48
|
+
encoder = None
|
|
49
|
+
encoder_config = SimpleNamespace(subsampling_factor=8)
|
|
50
|
+
preprocessor_config = SimpleNamespace(hop_length=160)
|
|
51
|
+
prompt_dictionary = {'en-US': 0, 'zh-CN': 1, 'fr-FR': 2}
|
|
52
|
+
vocabulary = ['<unk>', '▁Bonjour', '▁monde', '.', '你好', '世界']
|
|
53
|
+
blank_id = 6
|
|
54
|
+
max_symbols = 10
|
|
55
|
+
|
|
56
|
+
def decoder(self, current, hidden):
|
|
57
|
+
self.thread_ids.add(threading.get_ident())
|
|
58
|
+
last = -1 if current is None else int(current.item())
|
|
59
|
+
count = 1 if hidden is None else int(hidden[0].item()) + 1
|
|
60
|
+
return mx.array([[[last]]]), (mx.array([count]), mx.array([0]))
|
|
61
|
+
|
|
62
|
+
def joint(self, feature, decoded):
|
|
63
|
+
token = int(feature.item())
|
|
64
|
+
prediction = self.blank_id if int(decoded.item()) == token else token
|
|
65
|
+
return mx.array([int(i == prediction) for i in range(self.blank_id + 1)])
|
|
66
|
+
|
|
67
|
+
def apply_prompt(self, encoded, language):
|
|
68
|
+
return encoded
|
|
69
|
+
|
|
70
|
+
monkeypatch.setattr(audio, 'StreamingLogMelSpectrogram', Mel)
|
|
71
|
+
monkeypatch.setattr(streaming, 'ConformerStreamingState', Encoder)
|
|
72
|
+
yield SimpleNamespace(model=Model(), original_language='fr-FR', att_context=[56, 6],
|
|
73
|
+
backend_choice='nemotron-mlx-asr')
|
|
74
|
+
mx.set_default_device(previous_device)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def test_stream_drains_partial_frame_at_pause_and_eof_without_repeating_words(shared):
|
|
78
|
+
online = NemotronMLXOnlineProcessor(shared)
|
|
79
|
+
online.insert_audio_chunk(np.ones(1280), 0.08)
|
|
80
|
+
assert online._encoder is None # No decoding on the PCM insertion path.
|
|
81
|
+
first, _ = online.process_iter()
|
|
82
|
+
online.insert_audio_chunk(np.full(640, 2), 0.12)
|
|
83
|
+
assert online.process_iter()[0] == []
|
|
84
|
+
tail, _ = online.start_silence()
|
|
85
|
+
assert online.finish()[0] == []
|
|
86
|
+
online.end_silence(1.0, 0.12)
|
|
87
|
+
online.insert_audio_chunk(np.full(800, 3), 1.17)
|
|
88
|
+
last, _ = online.finish()
|
|
89
|
+
assert ''.join(t.text for t in first + tail + last) == ' Bonjour monde.'
|
|
90
|
+
np.testing.assert_allclose([(t.start, t.end) for t in first + tail + last],
|
|
91
|
+
[(0, 0.08), (0.08, 0.12), (1.12, 1.17)])
|
|
92
|
+
assert online.get_buffer().text == ''
|
|
93
|
+
assert online.finish()[0] == []
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def test_token_starts_are_populated_and_monotonic_before_finish(shared):
|
|
97
|
+
# The transducer times come out of the greedy decode, so they exist while the
|
|
98
|
+
# utterance is still open.
|
|
99
|
+
shared.model.vocabulary = ['<unk>', '▁un', '▁deux', '▁trois', '▁quatre']
|
|
100
|
+
shared.model.blank_id = 5
|
|
101
|
+
online = NemotronMLXOnlineProcessor(shared)
|
|
102
|
+
online.insert_audio_chunk(
|
|
103
|
+
np.concatenate([np.full(1280, value) for value in (1, 2, 3, 4)]), 0.32)
|
|
104
|
+
tokens, _ = online.process_iter()
|
|
105
|
+
assert online._encoder is not None # Mid-utterance: finish() never ran.
|
|
106
|
+
assert [token.text for token in tokens] == [' un', ' deux', ' trois', ' quatre']
|
|
107
|
+
starts = [token.start for token in tokens]
|
|
108
|
+
ends = [token.end for token in tokens]
|
|
109
|
+
assert all(start is not None for start in starts + ends)
|
|
110
|
+
assert starts == sorted(starts), 'token starts must not go backwards'
|
|
111
|
+
assert len(set(starts)) == len(starts), 'each frame must carry its own start time'
|
|
112
|
+
assert ends == sorted(ends)
|
|
113
|
+
np.testing.assert_allclose(starts, [0.0, 0.08, 0.16, 0.24])
|
|
114
|
+
np.testing.assert_allclose(ends, [0.08, 0.16, 0.24, 0.32])
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def test_sentence_boundary_keeps_one_space(shared):
|
|
118
|
+
# Reproduces the real token sequence: the model closes an utterance with a
|
|
119
|
+
# standalone space token and opens the next one with a space-prefixed word.
|
|
120
|
+
# Joined verbatim that reads 'We. We'.
|
|
121
|
+
shared.model.vocabulary = ['<unk>', '▁', '▁We', '.']
|
|
122
|
+
shared.model.blank_id = 4
|
|
123
|
+
online = NemotronMLXOnlineProcessor(shared)
|
|
124
|
+
online.insert_audio_chunk(
|
|
125
|
+
np.concatenate([np.full(1280, 2), np.full(1280, 3), np.full(1280, 1)]), 0.24)
|
|
126
|
+
first, _ = online.finish()
|
|
127
|
+
online.insert_audio_chunk(np.concatenate([np.full(1280, 2), np.full(1280, 3)]), 0.40)
|
|
128
|
+
second, _ = online.finish()
|
|
129
|
+
assert ''.join(t.text for t in first) == ' We.'
|
|
130
|
+
assert ''.join(t.text for t in second) == ' We.'
|
|
131
|
+
assert ''.join(t.text for t in first + second) == ' We. We.'
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def test_space_ending_line_survives_a_tokenless_finish(shared):
|
|
135
|
+
# A trailing-silence chunk finishes with no tokens, so the line already
|
|
136
|
+
# committed still ends in a space. Reading the trailing-space flag off that
|
|
137
|
+
# empty chunk drops it, the next utterance keeps its leading space, and the
|
|
138
|
+
# joined text reads 'We. We'.
|
|
139
|
+
shared.model.vocabulary = ['<unk>', '▁', '▁We', '.']
|
|
140
|
+
shared.model.blank_id = 4
|
|
141
|
+
online = NemotronMLXOnlineProcessor(shared)
|
|
142
|
+
online.insert_audio_chunk(
|
|
143
|
+
np.concatenate([np.full(1280, 2), np.full(1280, 3), np.full(1280, 1)]), 0.24)
|
|
144
|
+
head, _ = online.process_iter()
|
|
145
|
+
assert ''.join(t.text for t in head) == ' We. '
|
|
146
|
+
assert online.finish()[0] == [] # Trailing-silence chunk commits no token.
|
|
147
|
+
online.insert_audio_chunk(
|
|
148
|
+
np.concatenate([np.full(1280, 1), np.full(1280, 2), np.full(1280, 3)]), 0.48)
|
|
149
|
+
tail, _ = online.finish()
|
|
150
|
+
assert [t.text for t in tail] == ['We', '.']
|
|
151
|
+
joined = ''.join(t.text for t in head + tail)
|
|
152
|
+
assert joined == ' We. We.'
|
|
153
|
+
assert ' ' not in joined
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def test_space_only_final_chunk_does_not_glue_the_next_utterance(shared):
|
|
157
|
+
# A final chunk whose only committed token is the closing space returns nothing
|
|
158
|
+
# after the trim. That space was never handed to the caller, so the next
|
|
159
|
+
# utterance must keep its leading space; clearing the flag instead glues the two
|
|
160
|
+
# lines into 'WeWe.'.
|
|
161
|
+
shared.model.vocabulary = ['<unk>', '▁', 'We', '.']
|
|
162
|
+
shared.model.blank_id = 4
|
|
163
|
+
scripted = []
|
|
164
|
+
|
|
165
|
+
def joint(feature, decoded):
|
|
166
|
+
prediction = scripted.pop(0)
|
|
167
|
+
return mx.array([int(i == prediction) for i in range(shared.model.blank_id + 1)])
|
|
168
|
+
|
|
169
|
+
shared.model.joint = joint
|
|
170
|
+
online = NemotronMLXOnlineProcessor(shared)
|
|
171
|
+
scripted.extend(value for piece in [2] for value in (piece, shared.model.blank_id))
|
|
172
|
+
online.insert_audio_chunk(np.zeros(1280, dtype=np.float32), 0.16)
|
|
173
|
+
head, _ = online.process_iter()
|
|
174
|
+
scripted.extend(value for piece in [1] for value in (piece, shared.model.blank_id))
|
|
175
|
+
online.insert_audio_chunk(np.zeros(1280, dtype=np.float32), 0.32)
|
|
176
|
+
tail, _ = online.finish()
|
|
177
|
+
assert [t.text for t in head] == ['We']
|
|
178
|
+
assert tail == [] # Only the closing space token arrived, and it was trimmed.
|
|
179
|
+
scripted.extend(value for piece in [1, 2, 3] for value in (piece, shared.model.blank_id))
|
|
180
|
+
online.insert_audio_chunk(np.zeros(1280 * 3, dtype=np.float32), 0.80)
|
|
181
|
+
second, _ = online.finish()
|
|
182
|
+
joined = ''.join(t.text for t in head + tail + second)
|
|
183
|
+
assert joined == 'We We.'
|
|
184
|
+
assert 'WeWe' not in joined
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def test_utterance_start_collapses_repeated_space_tokens(shared):
|
|
188
|
+
# The next utterance's first decode can emit two space-only pieces before its
|
|
189
|
+
# first word (`first_tokens=[' ', ' ', 'I']`), a seam inside one call that the
|
|
190
|
+
# previous utterance's trailing trim cannot reach: joined verbatim that reads
|
|
191
|
+
# 'We. We'. The separator space must survive as exactly one.
|
|
192
|
+
shared.model.vocabulary = ['<unk>', '▁', 'We', '.']
|
|
193
|
+
shared.model.blank_id = 4
|
|
194
|
+
scripted = []
|
|
195
|
+
|
|
196
|
+
def joint(feature, decoded):
|
|
197
|
+
prediction = scripted.pop(0)
|
|
198
|
+
return mx.array([int(i == prediction) for i in range(shared.model.blank_id + 1)])
|
|
199
|
+
|
|
200
|
+
shared.model.joint = joint
|
|
201
|
+
online = NemotronMLXOnlineProcessor(shared)
|
|
202
|
+
utterances = []
|
|
203
|
+
for pieces in ([2, 3], [1, 1, 2, 3], [1, 1, 2, 3]):
|
|
204
|
+
# One mel frame per scripted piece, plus a blank frame to advance past it.
|
|
205
|
+
scripted.extend(value for piece in pieces for value in (piece, shared.model.blank_id))
|
|
206
|
+
online.insert_audio_chunk(np.zeros(1280 * len(pieces), dtype=np.float32), 0.16 * len(pieces))
|
|
207
|
+
utterances.append(online.finish()[0])
|
|
208
|
+
first, second, third = utterances
|
|
209
|
+
assert [t.text for t in first] == ['We', '.']
|
|
210
|
+
assert [t.text for t in second] == [' ', 'We', '.']
|
|
211
|
+
assert [t.text for t in third] == [' ', 'We', '.']
|
|
212
|
+
assert [t.text for t in first + second + third] == ['We', '.', ' ', 'We', '.', ' ', 'We', '.']
|
|
213
|
+
joined = ''.join(t.text for t in first + second + third)
|
|
214
|
+
assert joined == 'We. We. We.'
|
|
215
|
+
assert ' ' not in joined
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def test_intra_utterance_space_run_collapses_before_the_next_word(shared):
|
|
219
|
+
# Mid-utterance the decoder emits a standalone space token and then a
|
|
220
|
+
# space-prefixed word in the SAME batch ('the. The'), a seam an utterance-start
|
|
221
|
+
# rule never reaches. The run collapses to one space; the word keeps its letters.
|
|
222
|
+
shared.model.vocabulary = ['<unk>', '▁', 'We', '.', '▁The']
|
|
223
|
+
shared.model.blank_id = 5
|
|
224
|
+
scripted = []
|
|
225
|
+
|
|
226
|
+
def joint(feature, decoded):
|
|
227
|
+
prediction = scripted.pop(0)
|
|
228
|
+
return mx.array([int(i == prediction) for i in range(shared.model.blank_id + 1)])
|
|
229
|
+
|
|
230
|
+
shared.model.joint = joint
|
|
231
|
+
online = NemotronMLXOnlineProcessor(shared)
|
|
232
|
+
pieces = [2, 3, 1, 4]
|
|
233
|
+
scripted.extend(value for piece in pieces for value in (piece, shared.model.blank_id))
|
|
234
|
+
online.insert_audio_chunk(np.zeros(1280 * len(pieces), dtype=np.float32), 0.16 * len(pieces))
|
|
235
|
+
tokens, _ = online.process_iter()
|
|
236
|
+
assert online._encoder is not None # One decode batch, nothing finalised.
|
|
237
|
+
assert [t.text for t in tokens] == ['We', '.', ' ', 'The']
|
|
238
|
+
joined = ''.join(t.text for t in tokens)
|
|
239
|
+
assert joined == 'We. The'
|
|
240
|
+
assert ' ' not in joined
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def test_space_run_across_decode_batches_collapses(shared):
|
|
244
|
+
# The run can also straddle two batches of one utterance: a batch ends on a
|
|
245
|
+
# standalone space token and the next opens with a space-prefixed word, which the
|
|
246
|
+
# utterance-flush trim never sees. Joined verbatim that read 'We. The'.
|
|
247
|
+
shared.model.vocabulary = ['<unk>', '▁', 'We', '.', '▁The']
|
|
248
|
+
shared.model.blank_id = 5
|
|
249
|
+
scripted = []
|
|
250
|
+
|
|
251
|
+
def joint(feature, decoded):
|
|
252
|
+
prediction = scripted.pop(0)
|
|
253
|
+
return mx.array([int(i == prediction) for i in range(shared.model.blank_id + 1)])
|
|
254
|
+
|
|
255
|
+
shared.model.joint = joint
|
|
256
|
+
online = NemotronMLXOnlineProcessor(shared)
|
|
257
|
+
scripted.extend(value for piece in [2, 3, 1] for value in (piece, shared.model.blank_id))
|
|
258
|
+
online.insert_audio_chunk(np.zeros(1280 * 3, dtype=np.float32), 0.48)
|
|
259
|
+
head, _ = online.process_iter()
|
|
260
|
+
assert ''.join(t.text for t in head) == 'We. '
|
|
261
|
+
scripted.extend(value for piece in [1, 4] for value in (piece, shared.model.blank_id))
|
|
262
|
+
online.insert_audio_chunk(np.zeros(1280 * 2, dtype=np.float32), 0.80)
|
|
263
|
+
tail, _ = online.finish()
|
|
264
|
+
assert ''.join(t.text for t in tail) == 'The'
|
|
265
|
+
joined = ''.join(t.text for t in head + tail)
|
|
266
|
+
assert joined == 'We. The'
|
|
267
|
+
assert ' ' not in joined
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def test_interleaved_sessions_have_separate_decoder_state_and_language(shared):
|
|
271
|
+
args = SimpleNamespace(backend='nemotron-mlx-asr', backend_policy='simulstreaming')
|
|
272
|
+
chinese = online_factory(args, shared, language='zh')
|
|
273
|
+
french = online_factory(args, shared, language='fr')
|
|
274
|
+
automatic = online_factory(args, shared, language='auto')
|
|
275
|
+
chinese.insert_audio_chunk(np.full(1280, 4), 0.08)
|
|
276
|
+
fr_audio = np.concatenate([np.ones(1280), np.full(1280, 2)])
|
|
277
|
+
french.insert_audio_chunk(fr_audio, 0.16)
|
|
278
|
+
with ThreadPoolExecutor(max_workers=2) as callers:
|
|
279
|
+
zh_result = callers.submit(chinese.process_iter)
|
|
280
|
+
fr_result = callers.submit(french.process_iter)
|
|
281
|
+
zh_head, _ = zh_result.result()
|
|
282
|
+
fr_tokens, _ = fr_result.result()
|
|
283
|
+
chinese.insert_audio_chunk(np.full(800, 5), 0.13)
|
|
284
|
+
zh_tail, _ = chinese.new_speaker()
|
|
285
|
+
assert ''.join(t.text for t in zh_head + zh_tail) == '你好世界'
|
|
286
|
+
assert ''.join(t.text for t in fr_tokens) == ' Bonjour monde'
|
|
287
|
+
assert all(t.detected_language == 'zh-CN' for t in zh_head + zh_tail)
|
|
288
|
+
assert all(t.detected_language == 'fr-FR' for t in fr_tokens)
|
|
289
|
+
assert automatic.language is None and shared.original_language == 'fr-FR'
|
|
290
|
+
assert french.finish()[0] == []
|
|
291
|
+
# A lock alone allows calls to migrate between caller threads, which breaks
|
|
292
|
+
# cached MLX streams. Both sessions must keep the same model thread.
|
|
293
|
+
assert len(shared.model.thread_ids) == 1
|
|
294
|
+
with pytest.raises(ValueError, match='not supported'):
|
|
295
|
+
online_factory(args, shared, language='not-a-language')
|
|
296
|
+
with pytest.raises(ValueError, match='context'):
|
|
297
|
+
online_factory(args, shared, context='unsupported terminology prompt')
|