live-translation 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- live_translation-0.1.0/LICENSE +20 -0
- live_translation-0.1.0/PKG-INFO +296 -0
- live_translation-0.1.0/README.md +273 -0
- live_translation-0.1.0/live_translation/__init__.py +6 -0
- live_translation-0.1.0/live_translation/_args.py +140 -0
- live_translation-0.1.0/live_translation/_audio/__init__.py +0 -0
- live_translation-0.1.0/live_translation/_audio/_processor.py +151 -0
- live_translation-0.1.0/live_translation/_audio/_recorder.py +62 -0
- live_translation-0.1.0/live_translation/_audio/_vad.py +33 -0
- live_translation-0.1.0/live_translation/_output.py +88 -0
- live_translation-0.1.0/live_translation/_pipeline.py +130 -0
- live_translation-0.1.0/live_translation/_transcription/__init__.py +0 -0
- live_translation-0.1.0/live_translation/_transcription/_transcriber.py +83 -0
- live_translation-0.1.0/live_translation/_translation/__init__.py +0 -0
- live_translation-0.1.0/live_translation/_translation/_translator.py +92 -0
- live_translation-0.1.0/live_translation/_ws.py +81 -0
- live_translation-0.1.0/live_translation/app.py +27 -0
- live_translation-0.1.0/live_translation/cli.py +34 -0
- live_translation-0.1.0/live_translation/config.py +219 -0
- live_translation-0.1.0/live_translation.egg-info/PKG-INFO +296 -0
- live_translation-0.1.0/live_translation.egg-info/SOURCES.txt +34 -0
- live_translation-0.1.0/live_translation.egg-info/dependency_links.txt +1 -0
- live_translation-0.1.0/live_translation.egg-info/entry_points.txt +2 -0
- live_translation-0.1.0/live_translation.egg-info/requires.txt +10 -0
- live_translation-0.1.0/live_translation.egg-info/top_level.txt +1 -0
- live_translation-0.1.0/pyproject.toml +35 -0
- live_translation-0.1.0/setup.cfg +4 -0
- live_translation-0.1.0/tests/test_audio_processor.py +85 -0
- live_translation-0.1.0/tests/test_audio_recorder.py +68 -0
- live_translation-0.1.0/tests/test_audio_vad.py +45 -0
- live_translation-0.1.0/tests/test_cli.py +43 -0
- live_translation-0.1.0/tests/test_config.py +81 -0
- live_translation-0.1.0/tests/test_output.py +83 -0
- live_translation-0.1.0/tests/test_pipeline.py +49 -0
- live_translation-0.1.0/tests/test_transcription_transcriber.py +83 -0
- live_translation-0.1.0/tests/test_translation_translator.py +89 -0
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2024 Abdullah Hendy
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is furnished
|
|
10
|
+
to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
|
17
|
+
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
|
18
|
+
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
|
19
|
+
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
|
20
|
+
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: live-translation
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A real-time translation tool using Whisper & Opus-MT
|
|
5
|
+
Author-email: Abdullah Hendy <abdullah.a.hendy@gmail.com>
|
|
6
|
+
Project-URL: Homepage, https://github.com/AbdullahHendy/live-translation
|
|
7
|
+
Project-URL: Repository, https://github.com/AbdullahHendy/live-translation
|
|
8
|
+
Keywords: translation,transcription,speech-to-text,real-time translation,whisper,marianMT,Opus-MT,live transcription
|
|
9
|
+
Requires-Python: >=3.8
|
|
10
|
+
Description-Content-Type: text/markdown
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Requires-Dist: torch
|
|
13
|
+
Requires-Dist: torchaudio
|
|
14
|
+
Requires-Dist: transformers
|
|
15
|
+
Requires-Dist: numpy
|
|
16
|
+
Requires-Dist: pyaudio
|
|
17
|
+
Requires-Dist: websockets
|
|
18
|
+
Requires-Dist: huggingface_hub
|
|
19
|
+
Requires-Dist: sacremoses
|
|
20
|
+
Requires-Dist: faster-whisper
|
|
21
|
+
Requires-Dist: sentencepiece
|
|
22
|
+
Dynamic: license-file
|
|
23
|
+
|
|
24
|
+
# Real-time Speech-to-Text Translation
|
|
25
|
+
|
|
26
|
+
This project provides a real-time speech-to-text translation solution. It captures audio from the microphone, processes it, transcribes it into text, and translates it to a target language. It uses the **Silero** model for processing (Voice Activity Detection), **Whisper** model for transcription and **Opus-MT** for translation. The output can be through ***stdout***, a ***JSON file***, or ***websockets***.
|
|
27
|
+
|
|
28
|
+
## Features
|
|
29
|
+
|
|
30
|
+
- Real-time speech capture and processing using **Silero** VAD (Voice Activity Detection)
|
|
31
|
+
- Speech-to-text transcription using the Whisper model
|
|
32
|
+
- Translation of transcriptions from a source language to a target language
|
|
33
|
+
- Multithreaded design for efficient processing
|
|
34
|
+
- Different output modes: stdout, **JSON** file, websocket server
|
|
35
|
+
|
|
36
|
+
## Prerequisites
|
|
37
|
+
|
|
38
|
+
Before running the project, you need to install the following system dependencies:
|
|
39
|
+
|
|
40
|
+
- **PortAudio** (for audio input handling)
|
|
41
|
+
- **FFmpeg** (for audio and video processing)
|
|
42
|
+
- On Ubuntu/Debian-based systems, you can install it with:
|
|
43
|
+
```bash
|
|
44
|
+
sudo apt-get install portaudio19-dev ffmpeg
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
## Installation
|
|
48
|
+
|
|
49
|
+
**(RECOMMENDED)**: install this package inside a virtual environment to avoid dependency conflicts.
|
|
50
|
+
```bash
|
|
51
|
+
python -m venv .venv
|
|
52
|
+
source .venv/bin/activate
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
**Install** the package from PyPI:
|
|
56
|
+
```bash
|
|
57
|
+
pip install live-translation
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
**Verify** the installation:
|
|
61
|
+
```bash
|
|
62
|
+
python -c "import live_translation; print('live-translation installed successfully!')"
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
## Usage
|
|
66
|
+
|
|
67
|
+
> **NOTE**: One can safely ignore the following warning that might appear on **Linux** systems:
|
|
68
|
+
>
|
|
69
|
+
> ALSA lib pcm_dsnoop.c:567:(snd_pcm_dsnoop_open) unable to open slave
|
|
70
|
+
> ALSA lib pcm_dmix.c:1000:(snd_pcm_dmix_open) unable to open slave
|
|
71
|
+
> ALSA lib pcm.c:2722:(snd_pcm_open_noupdate) Unknown PCM cards.pcm.rear
|
|
72
|
+
> ALSA lib pcm.c:2722:(snd_pcm_open_noupdate) Unknown PCM cards.pcm.center_lfe
|
|
73
|
+
> ALSA lib pcm.c:2722:(snd_pcm_open_noupdate) Unknown PCM cards.pcm.side
|
|
74
|
+
> ALSA lib pcm_dmix.c:1000:(snd_pcm_dmix_open) unable to open slave
|
|
75
|
+
> Cannot connect to server socket err = No such file or directory
|
|
76
|
+
> Cannot connect to server request channel
|
|
77
|
+
> jack server is not running or cannot be started
|
|
78
|
+
> JackShmReadWritePtr::~JackShmReadWritePtr - Init not done for -1, skipping unlock
|
|
79
|
+
> JackShmReadWritePtr::~JackShmReadWritePtr - Init not done for -1, skipping unlock
|
|
80
|
+
>
|
|
81
|
+
|
|
82
|
+
### CLI
|
|
83
|
+
live-translation can be run directly from the command line:
|
|
84
|
+
```bash
|
|
85
|
+
live-translate [OPTIONS]
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
**[OPTIONS]**
|
|
89
|
+
```bash
|
|
90
|
+
usage: live-translate [-h] [--silence_threshold SILENCE_THRESHOLD] [--vad_aggressiveness {0,1,2,3,4,5,6,7,8,9}] [--max_buffer_duration {5,6,7,8,9,10}] [--device {cpu,cuda}] [--whisper_model {tiny,base,small,medium,large,large-v2}]
|
|
91
|
+
[--trans_model {Helsinki-NLP/opus-mt,Helsinki-NLP/opus-mt-tc-big}] [--src_lang SRC_LANG] [--tgt_lang TGT_LANG] [--output {print,file,websocket}] [--ws_port WS_PORT] [--transcribe_only]
|
|
92
|
+
|
|
93
|
+
Live Translation Pipeline - Configure runtime settings.
|
|
94
|
+
|
|
95
|
+
options:
|
|
96
|
+
-h, --help show this help message and exit
|
|
97
|
+
--silence_threshold SILENCE_THRESHOLD
|
|
98
|
+
Number of consecutive 32ms silent chunks to detect SILENCE.
|
|
99
|
+
SILENCE clears the audio buffer for transcription/translation.
|
|
100
|
+
NOTE: Minimum value is 16.
|
|
101
|
+
Default is 65 (~ 2s).
|
|
102
|
+
--vad_aggressiveness {0,1,2,3,4,5,6,7,8,9}
|
|
103
|
+
Voice Activity Detection (VAD) aggressiveness level (0-9).
|
|
104
|
+
Higher values mean VAD has to be more confident to detect speech vs silence.
|
|
105
|
+
Default is 8.
|
|
106
|
+
--max_buffer_duration {5,6,7,8,9,10}
|
|
107
|
+
Max audio buffer duration in seconds before trimming it.
|
|
108
|
+
Default is 7 seconds.
|
|
109
|
+
--device {cpu,cuda} Device for processing ('cpu', 'cuda').
|
|
110
|
+
Default is 'cpu'.
|
|
111
|
+
--whisper_model {tiny,base,small,medium,large,large-v2}
|
|
112
|
+
Whisper model size ('tiny', 'base', 'small', 'medium', 'large', 'large-v2').
|
|
113
|
+
Default is 'base'.
|
|
114
|
+
--trans_model {Helsinki-NLP/opus-mt,Helsinki-NLP/opus-mt-tc-big}
|
|
115
|
+
Translation model ('Helsinki-NLP/opus-mt', 'Helsinki-NLP/opus-mt-tc-big').
|
|
116
|
+
NOTE: Don't include source and target languages here.
|
|
117
|
+
Default is 'Helsinki-NLP/opus-mt'.
|
|
118
|
+
--src_lang SRC_LANG Source/Input language for transcription (e.g., 'en', 'fr').
|
|
119
|
+
Default is 'en'.
|
|
120
|
+
--tgt_lang TGT_LANG Target language for translation (e.g., 'es', 'de').
|
|
121
|
+
Default is 'es'.
|
|
122
|
+
--output {print,file,websocket}
|
|
123
|
+
Output method ('print', 'file', 'websocket').
|
|
124
|
+
- 'print': Prints transcriptions and translations to stdout.
|
|
125
|
+
- 'file': Saves structured JSON data (see below) in ./transcripts/transcriptions.json.
|
|
126
|
+
- 'websocket': Sends structured JSON data (see below) over WebSocket.
|
|
127
|
+
JSON format for 'file' and 'websocket':
|
|
128
|
+
{
|
|
129
|
+
"timestamp": "2025-03-06T12:34:56.789Z",
|
|
130
|
+
"transcription": "Hello world",
|
|
131
|
+
"translation": "Hola mundo"
|
|
132
|
+
}.
|
|
133
|
+
Default is 'print'.
|
|
134
|
+
--ws_port WS_PORT WebSocket port for sending transcriptions.
|
|
135
|
+
Required if --output is 'websocket'.
|
|
136
|
+
--transcribe_only Transcribe only mode. No translations are performed.
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
- in case of **websockets**, one can connect to the server using **curl**, **wscat**, etc..
|
|
140
|
+
```bash
|
|
141
|
+
curl --include --no-buffer ws://localhost:<PORT_NUM>
|
|
142
|
+
```
|
|
143
|
+
```bash
|
|
144
|
+
wscat -c ws://localhost:<PORT_NUM>
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
### API
|
|
148
|
+
You can also import and use live_translation directly in your Python code.
|
|
149
|
+
The following is a ***simple*** example of running *live_translation* in a server/client fashion.
|
|
150
|
+
For more detailed examples see [examples/](/examples/).
|
|
151
|
+
|
|
152
|
+
- **Server**
|
|
153
|
+
```python
|
|
154
|
+
from live_translation.config import Config
|
|
155
|
+
from live_translation.app import LiveTranslationApp
|
|
156
|
+
|
|
157
|
+
# Initialize the app with custom configuration
|
|
158
|
+
# Configure to be CPU-based and output to WebSocket
|
|
159
|
+
config = Config(
|
|
160
|
+
device="cpu",
|
|
161
|
+
output="websocket",
|
|
162
|
+
ws_port=8765,
|
|
163
|
+
)
|
|
164
|
+
|
|
165
|
+
# Create and start the Live Translation App
|
|
166
|
+
app = LiveTranslationApp(config)
|
|
167
|
+
app.run()
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
- **Client**
|
|
171
|
+
```python
|
|
172
|
+
import asyncio
|
|
173
|
+
import websockets
|
|
174
|
+
import json
|
|
175
|
+
|
|
176
|
+
async def listen():
|
|
177
|
+
uri = "ws://localhost:8765"
|
|
178
|
+
async with websockets.connect(uri) as websocket:
|
|
179
|
+
print("🔌 Connected to Live Translation WebSocket server.")
|
|
180
|
+
|
|
181
|
+
try:
|
|
182
|
+
while True:
|
|
183
|
+
message = await websocket.recv()
|
|
184
|
+
data = json.loads(message)
|
|
185
|
+
|
|
186
|
+
print(f"⏳ Timestamp: {data['timestamp']}")
|
|
187
|
+
print(f"📝 Transcription: {data['transcription']}")
|
|
188
|
+
print(f"🌍 Translation: {data['translation']}\n")
|
|
189
|
+
|
|
190
|
+
except websockets.exceptions.ConnectionClosed:
|
|
191
|
+
print("WebSocket connection closed.")
|
|
192
|
+
|
|
193
|
+
asyncio.run(listen())
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
## Development
|
|
197
|
+
|
|
198
|
+
To contribute or modify this project, these steps might be helpful:
|
|
199
|
+
|
|
200
|
+
**Clone** the repository:
|
|
201
|
+
```bash
|
|
202
|
+
git clone git@github.com:AbdullahHendy/live-translation.git
|
|
203
|
+
cd live-translation
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
**Ceate** a virtual environment:
|
|
207
|
+
```bash
|
|
208
|
+
python -m venv .venv
|
|
209
|
+
source .venv/bin/activate
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
**Install** Dependencies:
|
|
213
|
+
```bash
|
|
214
|
+
pip install --upgrade pip
|
|
215
|
+
pip install -r requirements.txt
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
**Test** the package:
|
|
219
|
+
```bash
|
|
220
|
+
make test
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
**Build** the package:
|
|
224
|
+
```bash
|
|
225
|
+
make build
|
|
226
|
+
```
|
|
227
|
+
> **NOTE**: Building generates a ***.whl*** file that can be ***pip*** insalled in a new environment for testing
|
|
228
|
+
|
|
229
|
+
**If needed**, run the program within the virtual environment:
|
|
230
|
+
```bash
|
|
231
|
+
python -m live_translation.cli [OPTIONS]
|
|
232
|
+
```
|
|
233
|
+
|
|
234
|
+
## Tested Environment
|
|
235
|
+
|
|
236
|
+
This project was tested and developed on the following system configuration:
|
|
237
|
+
|
|
238
|
+
- **Architecture**: x86_64 (64-bit)
|
|
239
|
+
- **Operating System**: Ubuntu 24.10 (Oracular Oriole)
|
|
240
|
+
- **Kernel Version**: 6.11.0-18-generic
|
|
241
|
+
- **Python Version**: 3.12.7
|
|
242
|
+
- **Processor**: 13th Gen Intel(R) Core(TM) i9-13900HX
|
|
243
|
+
- **GPU**: GeForce RTX 4070 Max-Q / Mobile [^1]
|
|
244
|
+
- **RAM**: 16GB DDR5
|
|
245
|
+
- **Dependencies**: All required dependencies are listed in `requirements.txt` and [Prerequisites](#prerequisites)
|
|
246
|
+
|
|
247
|
+
[^1]: CUDA not utilized, as the `DEVICE` configuration is set to `"cpu"`. Additional Nvidia drivers, CUDA, cuDNN installation needed if option `"cuda"` were to be used.
|
|
248
|
+
|
|
249
|
+
## Improvements
|
|
250
|
+
|
|
251
|
+
- **Block Diagram**: Include a block diagram to visually represent the flow and architecture of the system, making it easier to understand the overall design.
|
|
252
|
+
- **Better Error Handling**: Improve error handling across various components (audio, transcription, translation) to ensure the system is robust and can handle unexpected scenarios gracefully.
|
|
253
|
+
- **Performance Optimization**: Investigate performance bottlenecks including checking sleep durations and optimizing concurrency management to minimize lag.
|
|
254
|
+
- **Concurrency Design Check**: Review and optimize the threading design to ensure thread safety and prevent issues like race conditions or deadlocks, etc., revisit the current design of ***AudioRecorder*** being a thread while ***AudioProcessor***, ***Transcriber***, and ***Translator*** being processes.
|
|
255
|
+
- **Logging**: Integrate detailed logging to track system activity, errors, and performance metrics using a more formal logging framework.
|
|
256
|
+
|
|
257
|
+
## Citations
|
|
258
|
+
```bibtex
|
|
259
|
+
@article{Whisper,
|
|
260
|
+
title = {Robust Speech Recognition via Large-Scale Weak Supervision},
|
|
261
|
+
url = {https://arxiv.org/abs/2212.04356},
|
|
262
|
+
author = {Radford, Alec and Kim, Jong Wook and Xu, Tao and Brockman, Greg and McLeavey, Christine and Sutskever, Ilya},
|
|
263
|
+
publisher = {arXiv},
|
|
264
|
+
year = {2022}
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
@misc{Silero VAD,
|
|
268
|
+
author = {Silero Team},
|
|
269
|
+
title = {Silero VAD: pre-trained enterprise-grade Voice Activity Detector (VAD), Number Detector and Language Classifier},
|
|
270
|
+
year = {2021},
|
|
271
|
+
publisher = {GitHub},
|
|
272
|
+
journal = {GitHub repository},
|
|
273
|
+
howpublished = {\url{https://github.com/snakers4/silero-vad}},
|
|
274
|
+
email = {hello@silero.ai}
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
@article{tiedemann2023democratizing,
|
|
278
|
+
title={Democratizing neural machine translation with {OPUS-MT}},
|
|
279
|
+
author={Tiedemann, J{\"o}rg and Aulamo, Mikko and Bakshandaeva, Daria and Boggia, Michele and Gr{\"o}nroos, Stig-Arne and Nieminen, Tommi and Raganato, Alessandro and Scherrer, Yves and Vazquez, Raul and Virpioja, Sami},
|
|
280
|
+
journal={Language Resources and Evaluation},
|
|
281
|
+
number={58},
|
|
282
|
+
pages={713--755},
|
|
283
|
+
year={2023},
|
|
284
|
+
publisher={Springer Nature},
|
|
285
|
+
issn={1574-0218},
|
|
286
|
+
doi={10.1007/s10579-023-09704-w}
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
@InProceedings{TiedemannThottingal:EAMT2020,
|
|
290
|
+
author = {J{\"o}rg Tiedemann and Santhosh Thottingal},
|
|
291
|
+
title = {{OPUS-MT} — {B}uilding open translation services for the {W}orld},
|
|
292
|
+
booktitle = {Proceedings of the 22nd Annual Conference of the European Association for Machine Translation (EAMT)},
|
|
293
|
+
year = {2020},
|
|
294
|
+
address = {Lisbon, Portugal}
|
|
295
|
+
}
|
|
296
|
+
```
|
|
@@ -0,0 +1,273 @@
|
|
|
1
|
+
# Real-time Speech-to-Text Translation
|
|
2
|
+
|
|
3
|
+
This project provides a real-time speech-to-text translation solution. It captures audio from the microphone, processes it, transcribes it into text, and translates it to a target language. It uses the **Silero** model for processing (Voice Activity Detection), **Whisper** model for transcription and **Opus-MT** for translation. The output can be through ***stdout***, a ***JSON file***, or ***websockets***.
|
|
4
|
+
|
|
5
|
+
## Features
|
|
6
|
+
|
|
7
|
+
- Real-time speech capture and processing using **Silero** VAD (Voice Activity Detection)
|
|
8
|
+
- Speech-to-text transcription using the Whisper model
|
|
9
|
+
- Translation of transcriptions from a source language to a target language
|
|
10
|
+
- Multithreaded design for efficient processing
|
|
11
|
+
- Different output modes: stdout, **JSON** file, websocket server
|
|
12
|
+
|
|
13
|
+
## Prerequisites
|
|
14
|
+
|
|
15
|
+
Before running the project, you need to install the following system dependencies:
|
|
16
|
+
|
|
17
|
+
- **PortAudio** (for audio input handling)
|
|
18
|
+
- **FFmpeg** (for audio and video processing)
|
|
19
|
+
- On Ubuntu/Debian-based systems, you can install it with:
|
|
20
|
+
```bash
|
|
21
|
+
sudo apt-get install portaudio19-dev ffmpeg
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
## Installation
|
|
25
|
+
|
|
26
|
+
**(RECOMMENDED)**: install this package inside a virtual environment to avoid dependency conflicts.
|
|
27
|
+
```bash
|
|
28
|
+
python -m venv .venv
|
|
29
|
+
source .venv/bin/activate
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
**Install** the package from PyPI:
|
|
33
|
+
```bash
|
|
34
|
+
pip install live-translation
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
**Verify** the installation:
|
|
38
|
+
```bash
|
|
39
|
+
python -c "import live_translation; print('live-translation installed successfully!')"
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
## Usage
|
|
43
|
+
|
|
44
|
+
> **NOTE**: One can safely ignore the following warning that might appear on **Linux** systems:
|
|
45
|
+
>
|
|
46
|
+
> ALSA lib pcm_dsnoop.c:567:(snd_pcm_dsnoop_open) unable to open slave
|
|
47
|
+
> ALSA lib pcm_dmix.c:1000:(snd_pcm_dmix_open) unable to open slave
|
|
48
|
+
> ALSA lib pcm.c:2722:(snd_pcm_open_noupdate) Unknown PCM cards.pcm.rear
|
|
49
|
+
> ALSA lib pcm.c:2722:(snd_pcm_open_noupdate) Unknown PCM cards.pcm.center_lfe
|
|
50
|
+
> ALSA lib pcm.c:2722:(snd_pcm_open_noupdate) Unknown PCM cards.pcm.side
|
|
51
|
+
> ALSA lib pcm_dmix.c:1000:(snd_pcm_dmix_open) unable to open slave
|
|
52
|
+
> Cannot connect to server socket err = No such file or directory
|
|
53
|
+
> Cannot connect to server request channel
|
|
54
|
+
> jack server is not running or cannot be started
|
|
55
|
+
> JackShmReadWritePtr::~JackShmReadWritePtr - Init not done for -1, skipping unlock
|
|
56
|
+
> JackShmReadWritePtr::~JackShmReadWritePtr - Init not done for -1, skipping unlock
|
|
57
|
+
>
|
|
58
|
+
|
|
59
|
+
### CLI
|
|
60
|
+
live-translation can be run directly from the command line:
|
|
61
|
+
```bash
|
|
62
|
+
live-translate [OPTIONS]
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
**[OPTIONS]**
|
|
66
|
+
```bash
|
|
67
|
+
usage: live-translate [-h] [--silence_threshold SILENCE_THRESHOLD] [--vad_aggressiveness {0,1,2,3,4,5,6,7,8,9}] [--max_buffer_duration {5,6,7,8,9,10}] [--device {cpu,cuda}] [--whisper_model {tiny,base,small,medium,large,large-v2}]
|
|
68
|
+
[--trans_model {Helsinki-NLP/opus-mt,Helsinki-NLP/opus-mt-tc-big}] [--src_lang SRC_LANG] [--tgt_lang TGT_LANG] [--output {print,file,websocket}] [--ws_port WS_PORT] [--transcribe_only]
|
|
69
|
+
|
|
70
|
+
Live Translation Pipeline - Configure runtime settings.
|
|
71
|
+
|
|
72
|
+
options:
|
|
73
|
+
-h, --help show this help message and exit
|
|
74
|
+
--silence_threshold SILENCE_THRESHOLD
|
|
75
|
+
Number of consecutive 32ms silent chunks to detect SILENCE.
|
|
76
|
+
SILENCE clears the audio buffer for transcription/translation.
|
|
77
|
+
NOTE: Minimum value is 16.
|
|
78
|
+
Default is 65 (~ 2s).
|
|
79
|
+
--vad_aggressiveness {0,1,2,3,4,5,6,7,8,9}
|
|
80
|
+
Voice Activity Detection (VAD) aggressiveness level (0-9).
|
|
81
|
+
Higher values mean VAD has to be more confident to detect speech vs silence.
|
|
82
|
+
Default is 8.
|
|
83
|
+
--max_buffer_duration {5,6,7,8,9,10}
|
|
84
|
+
Max audio buffer duration in seconds before trimming it.
|
|
85
|
+
Default is 7 seconds.
|
|
86
|
+
--device {cpu,cuda} Device for processing ('cpu', 'cuda').
|
|
87
|
+
Default is 'cpu'.
|
|
88
|
+
--whisper_model {tiny,base,small,medium,large,large-v2}
|
|
89
|
+
Whisper model size ('tiny', 'base', 'small', 'medium', 'large', 'large-v2').
|
|
90
|
+
Default is 'base'.
|
|
91
|
+
--trans_model {Helsinki-NLP/opus-mt,Helsinki-NLP/opus-mt-tc-big}
|
|
92
|
+
Translation model ('Helsinki-NLP/opus-mt', 'Helsinki-NLP/opus-mt-tc-big').
|
|
93
|
+
NOTE: Don't include source and target languages here.
|
|
94
|
+
Default is 'Helsinki-NLP/opus-mt'.
|
|
95
|
+
--src_lang SRC_LANG Source/Input language for transcription (e.g., 'en', 'fr').
|
|
96
|
+
Default is 'en'.
|
|
97
|
+
--tgt_lang TGT_LANG Target language for translation (e.g., 'es', 'de').
|
|
98
|
+
Default is 'es'.
|
|
99
|
+
--output {print,file,websocket}
|
|
100
|
+
Output method ('print', 'file', 'websocket').
|
|
101
|
+
- 'print': Prints transcriptions and translations to stdout.
|
|
102
|
+
- 'file': Saves structured JSON data (see below) in ./transcripts/transcriptions.json.
|
|
103
|
+
- 'websocket': Sends structured JSON data (see below) over WebSocket.
|
|
104
|
+
JSON format for 'file' and 'websocket':
|
|
105
|
+
{
|
|
106
|
+
"timestamp": "2025-03-06T12:34:56.789Z",
|
|
107
|
+
"transcription": "Hello world",
|
|
108
|
+
"translation": "Hola mundo"
|
|
109
|
+
}.
|
|
110
|
+
Default is 'print'.
|
|
111
|
+
--ws_port WS_PORT WebSocket port for sending transcriptions.
|
|
112
|
+
Required if --output is 'websocket'.
|
|
113
|
+
--transcribe_only Transcribe only mode. No translations are performed.
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
- in case of **websockets**, one can connect to the server using **curl**, **wscat**, etc..
|
|
117
|
+
```bash
|
|
118
|
+
curl --include --no-buffer ws://localhost:<PORT_NUM>
|
|
119
|
+
```
|
|
120
|
+
```bash
|
|
121
|
+
wscat -c ws://localhost:<PORT_NUM>
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
### API
|
|
125
|
+
You can also import and use live_translation directly in your Python code.
|
|
126
|
+
The following is a ***simple*** example of running *live_translation* in a server/client fashion.
|
|
127
|
+
For more detailed examples see [examples/](/examples/).
|
|
128
|
+
|
|
129
|
+
- **Server**
|
|
130
|
+
```python
|
|
131
|
+
from live_translation.config import Config
|
|
132
|
+
from live_translation.app import LiveTranslationApp
|
|
133
|
+
|
|
134
|
+
# Initialize the app with custom configuration
|
|
135
|
+
# Configure to be CPU-based and output to WebSocket
|
|
136
|
+
config = Config(
|
|
137
|
+
device="cpu",
|
|
138
|
+
output="websocket",
|
|
139
|
+
ws_port=8765,
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
# Create and start the Live Translation App
|
|
143
|
+
app = LiveTranslationApp(config)
|
|
144
|
+
app.run()
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
- **Client**
|
|
148
|
+
```python
|
|
149
|
+
import asyncio
|
|
150
|
+
import websockets
|
|
151
|
+
import json
|
|
152
|
+
|
|
153
|
+
async def listen():
|
|
154
|
+
uri = "ws://localhost:8765"
|
|
155
|
+
async with websockets.connect(uri) as websocket:
|
|
156
|
+
print("🔌 Connected to Live Translation WebSocket server.")
|
|
157
|
+
|
|
158
|
+
try:
|
|
159
|
+
while True:
|
|
160
|
+
message = await websocket.recv()
|
|
161
|
+
data = json.loads(message)
|
|
162
|
+
|
|
163
|
+
print(f"⏳ Timestamp: {data['timestamp']}")
|
|
164
|
+
print(f"📝 Transcription: {data['transcription']}")
|
|
165
|
+
print(f"🌍 Translation: {data['translation']}\n")
|
|
166
|
+
|
|
167
|
+
except websockets.exceptions.ConnectionClosed:
|
|
168
|
+
print("WebSocket connection closed.")
|
|
169
|
+
|
|
170
|
+
asyncio.run(listen())
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
## Development
|
|
174
|
+
|
|
175
|
+
To contribute or modify this project, these steps might be helpful:
|
|
176
|
+
|
|
177
|
+
**Clone** the repository:
|
|
178
|
+
```bash
|
|
179
|
+
git clone git@github.com:AbdullahHendy/live-translation.git
|
|
180
|
+
cd live-translation
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
**Ceate** a virtual environment:
|
|
184
|
+
```bash
|
|
185
|
+
python -m venv .venv
|
|
186
|
+
source .venv/bin/activate
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
**Install** Dependencies:
|
|
190
|
+
```bash
|
|
191
|
+
pip install --upgrade pip
|
|
192
|
+
pip install -r requirements.txt
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
**Test** the package:
|
|
196
|
+
```bash
|
|
197
|
+
make test
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
**Build** the package:
|
|
201
|
+
```bash
|
|
202
|
+
make build
|
|
203
|
+
```
|
|
204
|
+
> **NOTE**: Building generates a ***.whl*** file that can be ***pip*** insalled in a new environment for testing
|
|
205
|
+
|
|
206
|
+
**If needed**, run the program within the virtual environment:
|
|
207
|
+
```bash
|
|
208
|
+
python -m live_translation.cli [OPTIONS]
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
## Tested Environment
|
|
212
|
+
|
|
213
|
+
This project was tested and developed on the following system configuration:
|
|
214
|
+
|
|
215
|
+
- **Architecture**: x86_64 (64-bit)
|
|
216
|
+
- **Operating System**: Ubuntu 24.10 (Oracular Oriole)
|
|
217
|
+
- **Kernel Version**: 6.11.0-18-generic
|
|
218
|
+
- **Python Version**: 3.12.7
|
|
219
|
+
- **Processor**: 13th Gen Intel(R) Core(TM) i9-13900HX
|
|
220
|
+
- **GPU**: GeForce RTX 4070 Max-Q / Mobile [^1]
|
|
221
|
+
- **RAM**: 16GB DDR5
|
|
222
|
+
- **Dependencies**: All required dependencies are listed in `requirements.txt` and [Prerequisites](#prerequisites)
|
|
223
|
+
|
|
224
|
+
[^1]: CUDA not utilized, as the `DEVICE` configuration is set to `"cpu"`. Additional Nvidia drivers, CUDA, cuDNN installation needed if option `"cuda"` were to be used.
|
|
225
|
+
|
|
226
|
+
## Improvements
|
|
227
|
+
|
|
228
|
+
- **Block Diagram**: Include a block diagram to visually represent the flow and architecture of the system, making it easier to understand the overall design.
|
|
229
|
+
- **Better Error Handling**: Improve error handling across various components (audio, transcription, translation) to ensure the system is robust and can handle unexpected scenarios gracefully.
|
|
230
|
+
- **Performance Optimization**: Investigate performance bottlenecks including checking sleep durations and optimizing concurrency management to minimize lag.
|
|
231
|
+
- **Concurrency Design Check**: Review and optimize the threading design to ensure thread safety and prevent issues like race conditions or deadlocks, etc., revisit the current design of ***AudioRecorder*** being a thread while ***AudioProcessor***, ***Transcriber***, and ***Translator*** being processes.
|
|
232
|
+
- **Logging**: Integrate detailed logging to track system activity, errors, and performance metrics using a more formal logging framework.
|
|
233
|
+
|
|
234
|
+
## Citations
|
|
235
|
+
```bibtex
|
|
236
|
+
@article{Whisper,
|
|
237
|
+
title = {Robust Speech Recognition via Large-Scale Weak Supervision},
|
|
238
|
+
url = {https://arxiv.org/abs/2212.04356},
|
|
239
|
+
author = {Radford, Alec and Kim, Jong Wook and Xu, Tao and Brockman, Greg and McLeavey, Christine and Sutskever, Ilya},
|
|
240
|
+
publisher = {arXiv},
|
|
241
|
+
year = {2022}
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
@misc{Silero VAD,
|
|
245
|
+
author = {Silero Team},
|
|
246
|
+
title = {Silero VAD: pre-trained enterprise-grade Voice Activity Detector (VAD), Number Detector and Language Classifier},
|
|
247
|
+
year = {2021},
|
|
248
|
+
publisher = {GitHub},
|
|
249
|
+
journal = {GitHub repository},
|
|
250
|
+
howpublished = {\url{https://github.com/snakers4/silero-vad}},
|
|
251
|
+
email = {hello@silero.ai}
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
@article{tiedemann2023democratizing,
|
|
255
|
+
title={Democratizing neural machine translation with {OPUS-MT}},
|
|
256
|
+
author={Tiedemann, J{\"o}rg and Aulamo, Mikko and Bakshandaeva, Daria and Boggia, Michele and Gr{\"o}nroos, Stig-Arne and Nieminen, Tommi and Raganato, Alessandro and Scherrer, Yves and Vazquez, Raul and Virpioja, Sami},
|
|
257
|
+
journal={Language Resources and Evaluation},
|
|
258
|
+
number={58},
|
|
259
|
+
pages={713--755},
|
|
260
|
+
year={2023},
|
|
261
|
+
publisher={Springer Nature},
|
|
262
|
+
issn={1574-0218},
|
|
263
|
+
doi={10.1007/s10579-023-09704-w}
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
@InProceedings{TiedemannThottingal:EAMT2020,
|
|
267
|
+
author = {J{\"o}rg Tiedemann and Santhosh Thottingal},
|
|
268
|
+
title = {{OPUS-MT} — {B}uilding open translation services for the {W}orld},
|
|
269
|
+
booktitle = {Proceedings of the 22nd Annual Conference of the European Association for Machine Translation (EAMT)},
|
|
270
|
+
year = {2020},
|
|
271
|
+
address = {Lisbon, Portugal}
|
|
272
|
+
}
|
|
273
|
+
```
|