stream-translator-gpt 2024.3.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stream-translator-gpt-2024.3.3/LICENSE +21 -0
- stream-translator-gpt-2024.3.3/PKG-INFO +116 -0
- stream-translator-gpt-2024.3.3/README.md +85 -0
- stream-translator-gpt-2024.3.3/pyproject.toml +62 -0
- stream-translator-gpt-2024.3.3/setup.cfg +4 -0
- stream-translator-gpt-2024.3.3/stream_translator_gpt/__init__.py +0 -0
- stream-translator-gpt-2024.3.3/stream_translator_gpt/__main__.py +4 -0
- stream-translator-gpt-2024.3.3/stream_translator_gpt/audio_getter.py +142 -0
- stream-translator-gpt-2024.3.3/stream_translator_gpt/audio_slicer.py +107 -0
- stream-translator-gpt-2024.3.3/stream_translator_gpt/audio_transcriber.py +86 -0
- stream-translator-gpt-2024.3.3/stream_translator_gpt/common.py +30 -0
- stream-translator-gpt-2024.3.3/stream_translator_gpt/filters.py +26 -0
- stream-translator-gpt-2024.3.3/stream_translator_gpt/llm_translator.py +190 -0
- stream-translator-gpt-2024.3.3/stream_translator_gpt/result_exporter.py +43 -0
- stream-translator-gpt-2024.3.3/stream_translator_gpt/silero_vad.jit +0 -0
- stream-translator-gpt-2024.3.3/stream_translator_gpt/translator.py +337 -0
- stream-translator-gpt-2024.3.3/stream_translator_gpt.egg-info/PKG-INFO +116 -0
- stream-translator-gpt-2024.3.3/stream_translator_gpt.egg-info/SOURCES.txt +20 -0
- stream-translator-gpt-2024.3.3/stream_translator_gpt.egg-info/dependency_links.txt +1 -0
- stream-translator-gpt-2024.3.3/stream_translator_gpt.egg-info/entry_points.txt +2 -0
- stream-translator-gpt-2024.3.3/stream_translator_gpt.egg-info/requires.txt +8 -0
- stream-translator-gpt-2024.3.3/stream_translator_gpt.egg-info/top_level.txt +2 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2022 fortypercnt
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: stream-translator-gpt
|
|
3
|
+
Version: 2024.3.3
|
|
4
|
+
Summary: Command line tool to transcribe & translate audio from livestreams in real time
|
|
5
|
+
Author-email: ion <ionicbond3@gmail.com>
|
|
6
|
+
Project-URL: Homepage, https://github.com/ionic-bond/stream-translator-gpt
|
|
7
|
+
Project-URL: Issues, https://github.com/ionic-bond/stream-translator-gpt/issues
|
|
8
|
+
Keywords: translator,translation,translate,transcribe,yt-dlp,vad,whisper,faster-whisper,whisper-api,gpt,gemini
|
|
9
|
+
Classifier: Topic :: Multimedia :: Sound/Audio :: Analysis
|
|
10
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
11
|
+
Classifier: Environment :: Console
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
19
|
+
Classifier: Operating System :: OS Independent
|
|
20
|
+
Requires-Python: >=3.9
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
License-File: LICENSE
|
|
23
|
+
Requires-Dist: numpy
|
|
24
|
+
Requires-Dist: yt-dlp
|
|
25
|
+
Requires-Dist: ffmpeg-python<0.3,>=0.2.0
|
|
26
|
+
Requires-Dist: sounddevice<1.0
|
|
27
|
+
Requires-Dist: openai-whisper<=20231117
|
|
28
|
+
Requires-Dist: faster-whisper<1.0.0,>=0.8.0
|
|
29
|
+
Requires-Dist: openai<2.0,>=1.0
|
|
30
|
+
Requires-Dist: google-generativeai<1.0
|
|
31
|
+
|
|
32
|
+
# stream-translator-gpt
|
|
33
|
+
Command line utility to transcribe or translate audio from livestreams in real time. Uses [yt-dlp](https://github.com/yt-dlp/yt-dlp) to
|
|
34
|
+
get livestream URLs from various services and OpenAI's [Whisper](https://github.com/openai/whisper) for transcription/translation.
|
|
35
|
+
|
|
36
|
+
This fork optimized the audio slicing logic based on [VAD](https://github.com/snakers4/silero-vad),
|
|
37
|
+
introduced OpenAI's [GPT API](https://platform.openai.com/docs/api-reference/chat/create) / Google's [Gemini API](https://makersuite.google.com/app/apikey) to support language translation beyond English,
|
|
38
|
+
and supports getting audio from the devices.
|
|
39
|
+
|
|
40
|
+
Sample: [](https://colab.research.google.com/github/ionic-bond/stream-translator-gpt/blob/main/stream_translator.ipynb)
|
|
41
|
+
|
|
42
|
+
## Prerequisites
|
|
43
|
+
|
|
44
|
+
1. [**Install and add ffmpeg to your PATH**](https://www.thewindowsclub.com/how-to-install-ffmpeg-on-windows-10#:~:text=Click%20New%20and%20type%20the,Click%20OK%20to%20apply%20changes.)
|
|
45
|
+
2. [**Install CUDA on your system.**](https://developer.nvidia.com/cuda-downloads) You can check the installed CUDA version with ```nvcc --version```.
|
|
46
|
+
|
|
47
|
+
## Setup
|
|
48
|
+
|
|
49
|
+
1. Setup a virtual environment.
|
|
50
|
+
2. ```pip install stream-translator-gpt```
|
|
51
|
+
3. Make sure that pytorch is installed with CUDA support. Whisper will probably not run in real time on a CPU.
|
|
52
|
+
|
|
53
|
+
## Usage
|
|
54
|
+
|
|
55
|
+
1. Translate live streaming:
|
|
56
|
+
|
|
57
|
+
```stream-translator-gpt {URL} {flags...}```
|
|
58
|
+
|
|
59
|
+
By default, the URL can be of the form ```twitch.tv/forsen``` and yt-dlp is used to obtain the .m3u8 link which is passed to ffmpeg.
|
|
60
|
+
|
|
61
|
+
2. Translate PC device audio:
|
|
62
|
+
|
|
63
|
+
```stream-translator-gpt device {flags...}```
|
|
64
|
+
|
|
65
|
+
Will use the system's default audio device as input.
|
|
66
|
+
|
|
67
|
+
If need to use another audio input device, `stream-translator-gpt device --print_all_devices` get device index and run the CLI with `--device_index`.
|
|
68
|
+
|
|
69
|
+
## Flags
|
|
70
|
+
|
|
71
|
+
| --flags | Default Value | Description |
|
|
72
|
+
| :--------------------------------: | :-----------: | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: |
|
|
73
|
+
| `URL` | | The URL of the stream. If fill in "device", the audio will be obtained from your PC device. |
|
|
74
|
+
| `--format` | wa* | Stream format code, this parameter will be passed directly to yt-dlp. |
|
|
75
|
+
| `--cookies` | | Used to open member-only stream, this parameter will be passed directly to yt-dlp. |
|
|
76
|
+
| `--device_index` | | The index of the device that needs to be recorded. If not set, the system default recording device will be used. |
|
|
77
|
+
| `--frame_duration` | 0.1 | The unit that processes live streaming data in seconds. |
|
|
78
|
+
| `--continuous_no_speech_threshold` | 0.8 | Slice if there is no speech for a continuous period in second. |
|
|
79
|
+
| `--min_audio_length` | 3.0 | Minimum slice audio length in seconds. |
|
|
80
|
+
| `--max_audio_length` | 30.0 | Maximum slice audio length in seconds. |
|
|
81
|
+
| `--prefix_retention_length` | 0.8 | The length of the retention prefix audio during slicing. |
|
|
82
|
+
| `--vad_threshold` | 0.5 | The threshold of Voice activity detection. if the speech probability of a frame is higher than this value, then this frame is speech. |
|
|
83
|
+
| `--model` | small | Select model size. See [here](https://github.com/openai/whisper#available-models-and-languages) for available models. |
|
|
84
|
+
| `--task` | translate | Whether to transcribe the audio (keep original language) or translate to english. |
|
|
85
|
+
| `--language` | auto | Language spoken in the stream. See [here](https://github.com/openai/whisper#available-models-and-languages) for available languages. |
|
|
86
|
+
| `--beam_size` | 5 | Number of beams in beam search. Set to 0 to use greedy algorithm instead (faster but less accurate). |
|
|
87
|
+
| `--best_of` | 5 | Number of candidates when sampling with non-zero temperature. |
|
|
88
|
+
| `--direct_url` | | Set this flag to pass the URL directly to ffmpeg. Otherwise, yt-dlp is used to obtain the stream URL. |
|
|
89
|
+
| `--use_faster_whisper` | | Set this flag to use faster_whisper implementation instead of the original OpenAI implementation |
|
|
90
|
+
| `--use_whisper_api` | | Set this flag to use OpenAI Whisper API instead of the original local Whipser. |
|
|
91
|
+
| `--whisper_filters` | emoji_filter | Filters apply to whisper results, separated by ",". |
|
|
92
|
+
| `--hide_whisper_result` | | Hide the result of Whisper transcribe. |
|
|
93
|
+
| `--openai_api_key` | | OpenAI API key if using GPT translation / Whisper API. |
|
|
94
|
+
| `--google_api_key` | | Google API key if using Gemini translation. |
|
|
95
|
+
| `--gpt_model` | gpt-3.5-turbo | GPT model name, gpt-3.5-turbo or gpt-4. (If using Gemini, not need to change this) |
|
|
96
|
+
| `--gpt_translation_prompt` | | If set, will translate the result text to target language via GPT / Gemini API (According to which API key is filled in). Example: "Translate from Japanese to Chinese" |
|
|
97
|
+
| `--gpt_translation_history_size` | 0 | The number of previous messages sent when calling the GPT / Gemini API. If the history size is 0, the translation will be run parallelly. If the history size > 0, the translation will be run serially. |
|
|
98
|
+
| `--gpt_translation_timeout` | 15 | If the GPT / Gemini translation exceeds this number of seconds, the translation will be discarded. |
|
|
99
|
+
| `--gpt_base_url` | | Customize the API endpoint of GPT. |
|
|
100
|
+
| `--retry_if_translation_fails` | | Retry when translation times out/fails. Used to generate subtitles offline. |
|
|
101
|
+
| `--output_timestamps` | | Output the timestamp of the text when outputting the text. |
|
|
102
|
+
| `--cqhttp_url` | | If set, will send the result text to the cqhttp server. |
|
|
103
|
+
| `--cqhttp_token` | | Token of cqhttp, if it is not set on the server side, it does not need to fill in. |
|
|
104
|
+
|
|
105
|
+
## Using faster-whisper
|
|
106
|
+
|
|
107
|
+
faster-whisper provides significant performance upgrades over the original OpenAI implementation (~ 4x faster, ~ 2x less memory).
|
|
108
|
+
To use it, install the [cuDNN](https://developer.nvidia.com/cudnn) to your CUDA dir, Then you can run the CLI with `--use_faster_whisper`.
|
|
109
|
+
|
|
110
|
+
## Contact me
|
|
111
|
+
|
|
112
|
+
Telegram: [@ionic_bond](https://t.me/ionic_bond)
|
|
113
|
+
|
|
114
|
+
## Donate
|
|
115
|
+
|
|
116
|
+
[PayPal](https://www.paypal.com/donate/?hosted_button_id=U9WR47CFGPBPU)
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
# stream-translator-gpt
|
|
2
|
+
Command line utility to transcribe or translate audio from livestreams in real time. Uses [yt-dlp](https://github.com/yt-dlp/yt-dlp) to
|
|
3
|
+
get livestream URLs from various services and OpenAI's [Whisper](https://github.com/openai/whisper) for transcription/translation.
|
|
4
|
+
|
|
5
|
+
This fork optimized the audio slicing logic based on [VAD](https://github.com/snakers4/silero-vad),
|
|
6
|
+
introduced OpenAI's [GPT API](https://platform.openai.com/docs/api-reference/chat/create) / Google's [Gemini API](https://makersuite.google.com/app/apikey) to support language translation beyond English,
|
|
7
|
+
and supports getting audio from the devices.
|
|
8
|
+
|
|
9
|
+
Sample: [](https://colab.research.google.com/github/ionic-bond/stream-translator-gpt/blob/main/stream_translator.ipynb)
|
|
10
|
+
|
|
11
|
+
## Prerequisites
|
|
12
|
+
|
|
13
|
+
1. [**Install and add ffmpeg to your PATH**](https://www.thewindowsclub.com/how-to-install-ffmpeg-on-windows-10#:~:text=Click%20New%20and%20type%20the,Click%20OK%20to%20apply%20changes.)
|
|
14
|
+
2. [**Install CUDA on your system.**](https://developer.nvidia.com/cuda-downloads) You can check the installed CUDA version with ```nvcc --version```.
|
|
15
|
+
|
|
16
|
+
## Setup
|
|
17
|
+
|
|
18
|
+
1. Setup a virtual environment.
|
|
19
|
+
2. ```pip install stream-translator-gpt```
|
|
20
|
+
3. Make sure that pytorch is installed with CUDA support. Whisper will probably not run in real time on a CPU.
|
|
21
|
+
|
|
22
|
+
## Usage
|
|
23
|
+
|
|
24
|
+
1. Translate live streaming:
|
|
25
|
+
|
|
26
|
+
```stream-translator-gpt {URL} {flags...}```
|
|
27
|
+
|
|
28
|
+
By default, the URL can be of the form ```twitch.tv/forsen``` and yt-dlp is used to obtain the .m3u8 link which is passed to ffmpeg.
|
|
29
|
+
|
|
30
|
+
2. Translate PC device audio:
|
|
31
|
+
|
|
32
|
+
```stream-translator-gpt device {flags...}```
|
|
33
|
+
|
|
34
|
+
Will use the system's default audio device as input.
|
|
35
|
+
|
|
36
|
+
If need to use another audio input device, `stream-translator-gpt device --print_all_devices` get device index and run the CLI with `--device_index`.
|
|
37
|
+
|
|
38
|
+
## Flags
|
|
39
|
+
|
|
40
|
+
| --flags | Default Value | Description |
|
|
41
|
+
| :--------------------------------: | :-----------: | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: |
|
|
42
|
+
| `URL` | | The URL of the stream. If fill in "device", the audio will be obtained from your PC device. |
|
|
43
|
+
| `--format` | wa* | Stream format code, this parameter will be passed directly to yt-dlp. |
|
|
44
|
+
| `--cookies` | | Used to open member-only stream, this parameter will be passed directly to yt-dlp. |
|
|
45
|
+
| `--device_index` | | The index of the device that needs to be recorded. If not set, the system default recording device will be used. |
|
|
46
|
+
| `--frame_duration` | 0.1 | The unit that processes live streaming data in seconds. |
|
|
47
|
+
| `--continuous_no_speech_threshold` | 0.8 | Slice if there is no speech for a continuous period in second. |
|
|
48
|
+
| `--min_audio_length` | 3.0 | Minimum slice audio length in seconds. |
|
|
49
|
+
| `--max_audio_length` | 30.0 | Maximum slice audio length in seconds. |
|
|
50
|
+
| `--prefix_retention_length` | 0.8 | The length of the retention prefix audio during slicing. |
|
|
51
|
+
| `--vad_threshold` | 0.5 | The threshold of Voice activity detection. if the speech probability of a frame is higher than this value, then this frame is speech. |
|
|
52
|
+
| `--model` | small | Select model size. See [here](https://github.com/openai/whisper#available-models-and-languages) for available models. |
|
|
53
|
+
| `--task` | translate | Whether to transcribe the audio (keep original language) or translate to english. |
|
|
54
|
+
| `--language` | auto | Language spoken in the stream. See [here](https://github.com/openai/whisper#available-models-and-languages) for available languages. |
|
|
55
|
+
| `--beam_size` | 5 | Number of beams in beam search. Set to 0 to use greedy algorithm instead (faster but less accurate). |
|
|
56
|
+
| `--best_of` | 5 | Number of candidates when sampling with non-zero temperature. |
|
|
57
|
+
| `--direct_url` | | Set this flag to pass the URL directly to ffmpeg. Otherwise, yt-dlp is used to obtain the stream URL. |
|
|
58
|
+
| `--use_faster_whisper` | | Set this flag to use faster_whisper implementation instead of the original OpenAI implementation |
|
|
59
|
+
| `--use_whisper_api` | | Set this flag to use OpenAI Whisper API instead of the original local Whipser. |
|
|
60
|
+
| `--whisper_filters` | emoji_filter | Filters apply to whisper results, separated by ",". |
|
|
61
|
+
| `--hide_whisper_result` | | Hide the result of Whisper transcribe. |
|
|
62
|
+
| `--openai_api_key` | | OpenAI API key if using GPT translation / Whisper API. |
|
|
63
|
+
| `--google_api_key` | | Google API key if using Gemini translation. |
|
|
64
|
+
| `--gpt_model` | gpt-3.5-turbo | GPT model name, gpt-3.5-turbo or gpt-4. (If using Gemini, not need to change this) |
|
|
65
|
+
| `--gpt_translation_prompt` | | If set, will translate the result text to target language via GPT / Gemini API (According to which API key is filled in). Example: "Translate from Japanese to Chinese" |
|
|
66
|
+
| `--gpt_translation_history_size` | 0 | The number of previous messages sent when calling the GPT / Gemini API. If the history size is 0, the translation will be run parallelly. If the history size > 0, the translation will be run serially. |
|
|
67
|
+
| `--gpt_translation_timeout` | 15 | If the GPT / Gemini translation exceeds this number of seconds, the translation will be discarded. |
|
|
68
|
+
| `--gpt_base_url` | | Customize the API endpoint of GPT. |
|
|
69
|
+
| `--retry_if_translation_fails` | | Retry when translation times out/fails. Used to generate subtitles offline. |
|
|
70
|
+
| `--output_timestamps` | | Output the timestamp of the text when outputting the text. |
|
|
71
|
+
| `--cqhttp_url` | | If set, will send the result text to the cqhttp server. |
|
|
72
|
+
| `--cqhttp_token` | | Token of cqhttp, if it is not set on the server side, it does not need to fill in. |
|
|
73
|
+
|
|
74
|
+
## Using faster-whisper
|
|
75
|
+
|
|
76
|
+
faster-whisper provides significant performance upgrades over the original OpenAI implementation (~ 4x faster, ~ 2x less memory).
|
|
77
|
+
To use it, install the [cuDNN](https://developer.nvidia.com/cudnn) to your CUDA dir, Then you can run the CLI with `--use_faster_whisper`.
|
|
78
|
+
|
|
79
|
+
## Contact me
|
|
80
|
+
|
|
81
|
+
Telegram: [@ionic_bond](https://t.me/ionic_bond)
|
|
82
|
+
|
|
83
|
+
## Donate
|
|
84
|
+
|
|
85
|
+
[PayPal](https://www.paypal.com/donate/?hosted_button_id=U9WR47CFGPBPU)
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61.0"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "stream-translator-gpt"
|
|
7
|
+
version = "2024.3.3"
|
|
8
|
+
authors = [
|
|
9
|
+
{ name="ion", email="ionicbond3@gmail.com" },
|
|
10
|
+
]
|
|
11
|
+
description = "Command line tool to transcribe & translate audio from livestreams in real time"
|
|
12
|
+
readme = "README.md"
|
|
13
|
+
requires-python = ">=3.9"
|
|
14
|
+
keywords = [
|
|
15
|
+
"translator",
|
|
16
|
+
"translation",
|
|
17
|
+
"translate",
|
|
18
|
+
"transcribe",
|
|
19
|
+
"yt-dlp",
|
|
20
|
+
"vad",
|
|
21
|
+
"whisper",
|
|
22
|
+
"faster-whisper",
|
|
23
|
+
"whisper-api",
|
|
24
|
+
"gpt",
|
|
25
|
+
"gemini",
|
|
26
|
+
]
|
|
27
|
+
classifiers = [
|
|
28
|
+
"Topic :: Multimedia :: Sound/Audio :: Analysis",
|
|
29
|
+
"Development Status :: 5 - Production/Stable",
|
|
30
|
+
"Environment :: Console",
|
|
31
|
+
"Programming Language :: Python :: 3",
|
|
32
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
33
|
+
"Programming Language :: Python :: 3.9",
|
|
34
|
+
"Programming Language :: Python :: 3.10",
|
|
35
|
+
"Programming Language :: Python :: 3.11",
|
|
36
|
+
"Programming Language :: Python :: 3.12",
|
|
37
|
+
"License :: OSI Approved :: MIT License",
|
|
38
|
+
"Operating System :: OS Independent",
|
|
39
|
+
]
|
|
40
|
+
dependencies = [
|
|
41
|
+
"numpy",
|
|
42
|
+
"yt-dlp",
|
|
43
|
+
"ffmpeg-python>=0.2.0,<0.3",
|
|
44
|
+
"sounddevice<1.0",
|
|
45
|
+
"openai-whisper<=20231117",
|
|
46
|
+
"faster-whisper>=0.8.0,<1.0.0",
|
|
47
|
+
"openai>=1.0,<2.0",
|
|
48
|
+
"google-generativeai<1.0",
|
|
49
|
+
]
|
|
50
|
+
|
|
51
|
+
[project.scripts]
|
|
52
|
+
stream-translator-gpt = "stream_translator_gpt.translator:cli"
|
|
53
|
+
|
|
54
|
+
[project.urls]
|
|
55
|
+
Homepage = "https://github.com/ionic-bond/stream-translator-gpt"
|
|
56
|
+
Issues = "https://github.com/ionic-bond/stream-translator-gpt/issues"
|
|
57
|
+
|
|
58
|
+
[tool.setuptools.packages.find]
|
|
59
|
+
where = ["."]
|
|
60
|
+
|
|
61
|
+
[tool.setuptools.package-data]
|
|
62
|
+
stream_translator_gpt = ["*.jit"]
|
|
File without changes
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
import queue
|
|
2
|
+
import signal
|
|
3
|
+
import subprocess
|
|
4
|
+
import sys
|
|
5
|
+
import threading
|
|
6
|
+
|
|
7
|
+
import ffmpeg
|
|
8
|
+
import numpy as np
|
|
9
|
+
|
|
10
|
+
from .common import SAMPLE_RATE, LoopWorkerBase
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _transport(ytdlp_proc, ffmpeg_proc):
|
|
14
|
+
while (ytdlp_proc.poll() is None) and (ffmpeg_proc.poll() is None):
|
|
15
|
+
try:
|
|
16
|
+
chunk = ytdlp_proc.stdout.read(1024)
|
|
17
|
+
ffmpeg_proc.stdin.write(chunk)
|
|
18
|
+
except (BrokenPipeError, OSError):
|
|
19
|
+
pass
|
|
20
|
+
ytdlp_proc.kill()
|
|
21
|
+
ffmpeg_proc.kill()
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _open_stream(url: str, direct_url: bool, format: str, cookies: str):
|
|
25
|
+
if direct_url:
|
|
26
|
+
try:
|
|
27
|
+
process = (ffmpeg.input(
|
|
28
|
+
url, loglevel='panic').output('pipe:',
|
|
29
|
+
format='s16le',
|
|
30
|
+
acodec='pcm_s16le',
|
|
31
|
+
ac=1,
|
|
32
|
+
ar=SAMPLE_RATE).run_async(pipe_stdout=True))
|
|
33
|
+
except ffmpeg.Error as e:
|
|
34
|
+
raise RuntimeError(f'Failed to load audio: {e.stderr.decode()}') from e
|
|
35
|
+
|
|
36
|
+
return process, None
|
|
37
|
+
|
|
38
|
+
cmd = ['yt-dlp', url, '-f', format, '-o', '-', '-q']
|
|
39
|
+
if cookies:
|
|
40
|
+
cmd.extend(['--cookies', cookies])
|
|
41
|
+
ytdlp_process = subprocess.Popen(cmd, stdout=subprocess.PIPE)
|
|
42
|
+
|
|
43
|
+
try:
|
|
44
|
+
ffmpeg_process = (ffmpeg.input('pipe:', loglevel='panic').output('pipe:',
|
|
45
|
+
format='s16le',
|
|
46
|
+
acodec='pcm_s16le',
|
|
47
|
+
ac=1,
|
|
48
|
+
ar=SAMPLE_RATE).run_async(
|
|
49
|
+
pipe_stdin=True,
|
|
50
|
+
pipe_stdout=True))
|
|
51
|
+
except ffmpeg.Error as e:
|
|
52
|
+
raise RuntimeError(f'Failed to load audio: {e.stderr.decode()}') from e
|
|
53
|
+
|
|
54
|
+
thread = threading.Thread(target=_transport, args=(ytdlp_process, ffmpeg_process))
|
|
55
|
+
thread.start()
|
|
56
|
+
return ffmpeg_process, ytdlp_process
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class StreamAudioGetter(LoopWorkerBase):
|
|
60
|
+
|
|
61
|
+
def __init__(self, url: str, direct_url: bool, format: str, cookies: str,
|
|
62
|
+
frame_duration: float) -> None:
|
|
63
|
+
print('Opening stream: {}'.format(url))
|
|
64
|
+
self.ffmpeg_process, self.ytdlp_process = _open_stream(url, direct_url, format, cookies)
|
|
65
|
+
self.byte_size = round(frame_duration * SAMPLE_RATE *
|
|
66
|
+
2) # Factor 2 comes from reading the int16 stream as bytes
|
|
67
|
+
signal.signal(signal.SIGINT, self._exit_handler)
|
|
68
|
+
|
|
69
|
+
def _exit_handler(self, signum, frame):
|
|
70
|
+
self.ffmpeg_process.kill()
|
|
71
|
+
if self.ytdlp_process:
|
|
72
|
+
self.ytdlp_process.kill()
|
|
73
|
+
sys.exit(0)
|
|
74
|
+
|
|
75
|
+
def loop(self, output_queue: queue.SimpleQueue[np.array]):
|
|
76
|
+
while self.ffmpeg_process.poll() is None:
|
|
77
|
+
in_bytes = self.ffmpeg_process.stdout.read(self.byte_size)
|
|
78
|
+
if not in_bytes:
|
|
79
|
+
break
|
|
80
|
+
if len(in_bytes) != self.byte_size:
|
|
81
|
+
continue
|
|
82
|
+
audio = np.frombuffer(in_bytes, np.int16).flatten().astype(np.float32) / 32768.0
|
|
83
|
+
output_queue.put(audio)
|
|
84
|
+
|
|
85
|
+
self.ffmpeg_process.kill()
|
|
86
|
+
if self.ytdlp_process:
|
|
87
|
+
self.ytdlp_process.kill()
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class LocalFileAudioGetter(LoopWorkerBase):
|
|
91
|
+
|
|
92
|
+
def __init__(self, file_path: str, frame_duration: float) -> None:
|
|
93
|
+
print('Opening local file: {}'.format(file_path))
|
|
94
|
+
try:
|
|
95
|
+
self.ffmpeg_process = (ffmpeg.input(
|
|
96
|
+
file_path, loglevel='panic').output('pipe:',
|
|
97
|
+
format='s16le',
|
|
98
|
+
acodec='pcm_s16le',
|
|
99
|
+
ac=1,
|
|
100
|
+
ar=SAMPLE_RATE).run_async(pipe_stdin=True,
|
|
101
|
+
pipe_stdout=True))
|
|
102
|
+
except ffmpeg.Error as e:
|
|
103
|
+
raise RuntimeError(f'Failed to load audio: {e.stderr.decode()}') from e
|
|
104
|
+
self.byte_size = round(frame_duration * SAMPLE_RATE *
|
|
105
|
+
2) # Factor 2 comes from reading the int16 stream as bytes
|
|
106
|
+
signal.signal(signal.SIGINT, self._exit_handler)
|
|
107
|
+
|
|
108
|
+
def _exit_handler(self, signum, frame):
|
|
109
|
+
self.ffmpeg_process.kill()
|
|
110
|
+
sys.exit(0)
|
|
111
|
+
|
|
112
|
+
def loop(self, output_queue: queue.SimpleQueue[np.array]):
|
|
113
|
+
while self.ffmpeg_process.poll() is None:
|
|
114
|
+
in_bytes = self.ffmpeg_process.stdout.read(self.byte_size)
|
|
115
|
+
if not in_bytes:
|
|
116
|
+
break
|
|
117
|
+
if len(in_bytes) != self.byte_size:
|
|
118
|
+
continue
|
|
119
|
+
audio = np.frombuffer(in_bytes, np.int16).flatten().astype(np.float32) / 32768.0
|
|
120
|
+
output_queue.put(audio)
|
|
121
|
+
|
|
122
|
+
self.ffmpeg_process.kill()
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
class DeviceAudioGetter(LoopWorkerBase):
|
|
126
|
+
|
|
127
|
+
def __init__(self, device_index: int, frame_duration: float) -> None:
|
|
128
|
+
import sounddevice as sd
|
|
129
|
+
if device_index:
|
|
130
|
+
sd.default.device[0] = device_index
|
|
131
|
+
sd.default.dtype[0] = np.float32
|
|
132
|
+
self.frame_duration = frame_duration
|
|
133
|
+
print('Recording device: {}'.format(sd.query_devices(sd.default.device[0])['name']))
|
|
134
|
+
|
|
135
|
+
def loop(self, output_queue: queue.SimpleQueue[np.array]):
|
|
136
|
+
import sounddevice as sd
|
|
137
|
+
while True:
|
|
138
|
+
audio = sd.rec(frames=round(SAMPLE_RATE * self.frame_duration),
|
|
139
|
+
samplerate=SAMPLE_RATE,
|
|
140
|
+
channels=1,
|
|
141
|
+
blocking=True).flatten()
|
|
142
|
+
output_queue.put(audio)
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import queue
|
|
3
|
+
import torch
|
|
4
|
+
import warnings
|
|
5
|
+
|
|
6
|
+
import numpy as np
|
|
7
|
+
|
|
8
|
+
from .common import TranslationTask, SAMPLE_RATE, LoopWorkerBase
|
|
9
|
+
|
|
10
|
+
warnings.filterwarnings('ignore')
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _init_jit_model(model_path: str, device=torch.device('cpu')):
|
|
14
|
+
torch.set_grad_enabled(False)
|
|
15
|
+
model = torch.jit.load(model_path, map_location=device)
|
|
16
|
+
model.eval()
|
|
17
|
+
return model
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class VAD:
|
|
21
|
+
|
|
22
|
+
def __init__(self):
|
|
23
|
+
current_dir = os.path.dirname(__file__)
|
|
24
|
+
self.model = _init_jit_model(os.path.join(current_dir, 'silero_vad.jit'))
|
|
25
|
+
|
|
26
|
+
def is_speech(self, audio: np.array, threshold: float = 0.5, sampling_rate: int = 16000):
|
|
27
|
+
if not torch.is_tensor(audio):
|
|
28
|
+
try:
|
|
29
|
+
audio = torch.Tensor(audio)
|
|
30
|
+
except:
|
|
31
|
+
raise TypeError('Audio cannot be casted to tensor. Cast it manually')
|
|
32
|
+
speech_prob = self.model(audio, sampling_rate).item()
|
|
33
|
+
return speech_prob >= threshold
|
|
34
|
+
|
|
35
|
+
def reset_states(self):
|
|
36
|
+
self.model.reset_states()
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class AudioSlicer(LoopWorkerBase):
|
|
40
|
+
|
|
41
|
+
def __init__(self, frame_duration: float, continuous_no_speech_threshold: float,
|
|
42
|
+
min_audio_length: float, max_audio_length: float, prefix_retention_length: float,
|
|
43
|
+
vad_threshold: float):
|
|
44
|
+
self.vad = VAD()
|
|
45
|
+
self.continuous_no_speech_threshold = round(continuous_no_speech_threshold / frame_duration)
|
|
46
|
+
self.min_audio_length = round(min_audio_length / frame_duration)
|
|
47
|
+
self.max_audio_length = round(max_audio_length / frame_duration)
|
|
48
|
+
self.prefix_retention_length = round(prefix_retention_length / frame_duration)
|
|
49
|
+
self.vad_threshold = vad_threshold
|
|
50
|
+
self.sampling_rate = SAMPLE_RATE
|
|
51
|
+
self.audio_buffer = []
|
|
52
|
+
self.prefix_audio_buffer = []
|
|
53
|
+
self.speech_count = 0
|
|
54
|
+
self.no_speech_count = 0
|
|
55
|
+
self.continuous_no_speech_count = 0
|
|
56
|
+
self.frame_duration = frame_duration
|
|
57
|
+
self.counter = 0
|
|
58
|
+
self.last_slice_second = 0.0
|
|
59
|
+
|
|
60
|
+
def put(self, audio: np.array):
|
|
61
|
+
self.counter += 1
|
|
62
|
+
if self.vad.is_speech(audio, self.vad_threshold, self.sampling_rate):
|
|
63
|
+
self.audio_buffer.append(audio)
|
|
64
|
+
self.speech_count += 1
|
|
65
|
+
self.continuous_no_speech_count = 0
|
|
66
|
+
else:
|
|
67
|
+
if self.speech_count == 0 and self.no_speech_count == 1:
|
|
68
|
+
self.slice()
|
|
69
|
+
self.audio_buffer.append(audio)
|
|
70
|
+
self.no_speech_count += 1
|
|
71
|
+
self.continuous_no_speech_count += 1
|
|
72
|
+
if self.speech_count and self.no_speech_count / 4 > self.speech_count:
|
|
73
|
+
self.slice()
|
|
74
|
+
|
|
75
|
+
def should_slice(self):
|
|
76
|
+
audio_len = len(self.audio_buffer)
|
|
77
|
+
if audio_len < self.min_audio_length:
|
|
78
|
+
return False
|
|
79
|
+
if audio_len > self.max_audio_length:
|
|
80
|
+
return True
|
|
81
|
+
if self.continuous_no_speech_count >= self.continuous_no_speech_threshold:
|
|
82
|
+
return True
|
|
83
|
+
return False
|
|
84
|
+
|
|
85
|
+
def slice(self):
|
|
86
|
+
concatenate_buffer = self.prefix_audio_buffer + self.audio_buffer
|
|
87
|
+
concatenate_audio = np.concatenate(concatenate_buffer)
|
|
88
|
+
self.audio_buffer = []
|
|
89
|
+
self.prefix_audio_buffer = concatenate_buffer[-self.prefix_retention_length:]
|
|
90
|
+
self.speech_count = 0
|
|
91
|
+
self.no_speech_count = 0
|
|
92
|
+
self.continuous_no_speech_count = 0
|
|
93
|
+
# self.vad.reset_states()
|
|
94
|
+
slice_second = self.counter * self.frame_duration
|
|
95
|
+
last_slice_second = self.last_slice_second
|
|
96
|
+
self.last_slice_second = slice_second
|
|
97
|
+
return concatenate_audio, (last_slice_second, slice_second)
|
|
98
|
+
|
|
99
|
+
def loop(self, input_queue: queue.SimpleQueue[np.array],
|
|
100
|
+
output_queue: queue.SimpleQueue[TranslationTask]):
|
|
101
|
+
while True:
|
|
102
|
+
audio = input_queue.get()
|
|
103
|
+
self.put(audio)
|
|
104
|
+
if self.should_slice():
|
|
105
|
+
sliced_audio, time_range = self.slice()
|
|
106
|
+
task = TranslationTask(sliced_audio, time_range)
|
|
107
|
+
output_queue.put(task)
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import queue
|
|
3
|
+
from scipy.io.wavfile import write as write_audio
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
from openai import OpenAI
|
|
7
|
+
|
|
8
|
+
from . import filters
|
|
9
|
+
from .common import TranslationTask, SAMPLE_RATE, LoopWorkerBase
|
|
10
|
+
|
|
11
|
+
TEMP_AUDIO_FILE_NAME = 'temp.wav'
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _filter_text(text: str, whisper_filters: str):
|
|
15
|
+
filter_name_list = whisper_filters.split(',')
|
|
16
|
+
for filter_name in filter_name_list:
|
|
17
|
+
filter = getattr(filters, filter_name)
|
|
18
|
+
if not filter:
|
|
19
|
+
raise Exception('Unknown filter: %s' % filter_name)
|
|
20
|
+
text = filter(text)
|
|
21
|
+
return text
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class OpenaiWhisper(LoopWorkerBase):
|
|
25
|
+
|
|
26
|
+
def __init__(self, model: str, language: str) -> None:
|
|
27
|
+
print('Loading whisper model: {}'.format(model))
|
|
28
|
+
import whisper
|
|
29
|
+
self.model = whisper.load_model(model)
|
|
30
|
+
self.language = language
|
|
31
|
+
|
|
32
|
+
def transcribe(self, audio: np.array, **transcribe_options) -> str:
|
|
33
|
+
result = self.model.transcribe(audio,
|
|
34
|
+
without_timestamps=True,
|
|
35
|
+
language=self.language,
|
|
36
|
+
**transcribe_options)
|
|
37
|
+
return result.get('text')
|
|
38
|
+
|
|
39
|
+
def loop(self, input_queue: queue.SimpleQueue[TranslationTask],
|
|
40
|
+
output_queue: queue.SimpleQueue[TranslationTask], whisper_filters: str,
|
|
41
|
+
print_result: bool, **transcribe_options):
|
|
42
|
+
while True:
|
|
43
|
+
task = input_queue.get()
|
|
44
|
+
task.transcribed_text = _filter_text(self.transcribe(task.audio, **transcribe_options),
|
|
45
|
+
whisper_filters).strip()
|
|
46
|
+
if not task.transcribed_text:
|
|
47
|
+
if print_result:
|
|
48
|
+
print('skip...')
|
|
49
|
+
continue
|
|
50
|
+
if print_result:
|
|
51
|
+
print(task.transcribed_text)
|
|
52
|
+
output_queue.put(task)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class FasterWhisper(OpenaiWhisper):
|
|
56
|
+
|
|
57
|
+
def __init__(self, model: str, language: str) -> None:
|
|
58
|
+
print('Loading faster-whisper model: {}'.format(model))
|
|
59
|
+
from faster_whisper import WhisperModel
|
|
60
|
+
self.model = WhisperModel(model)
|
|
61
|
+
self.language = language
|
|
62
|
+
|
|
63
|
+
def transcribe(self, audio: np.array, **transcribe_options) -> str:
|
|
64
|
+
segments, info = self.model.transcribe(audio, language=self.language, **transcribe_options)
|
|
65
|
+
transcribed_text = ''
|
|
66
|
+
for segment in segments:
|
|
67
|
+
transcribed_text += segment.text
|
|
68
|
+
return transcribed_text
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class RemoteOpenaiWhisper(OpenaiWhisper):
|
|
72
|
+
# https://platform.openai.com/docs/api-reference/audio/createTranscription?lang=python
|
|
73
|
+
|
|
74
|
+
def __init__(self, language: str) -> None:
|
|
75
|
+
self.client = OpenAI()
|
|
76
|
+
self.language = language
|
|
77
|
+
|
|
78
|
+
def transcribe(self, audio: np.array, **transcribe_options) -> str:
|
|
79
|
+
with open(TEMP_AUDIO_FILE_NAME, 'wb') as audio_file:
|
|
80
|
+
write_audio(audio_file, SAMPLE_RATE, audio)
|
|
81
|
+
with open(TEMP_AUDIO_FILE_NAME, 'rb') as audio_file:
|
|
82
|
+
result = self.client.audio.transcriptions.create(model='whisper-1',
|
|
83
|
+
file=audio_file,
|
|
84
|
+
language=self.language).text
|
|
85
|
+
os.remove(TEMP_AUDIO_FILE_NAME)
|
|
86
|
+
return result
|