OpenSTBench 0.3.2__tar.gz → 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- openstbench-1.0.0/PKG-INFO +134 -0
- openstbench-1.0.0/README.md +85 -0
- {openstbench-0.3.2 → openstbench-1.0.0}/pyproject.toml +1 -1
- openstbench-1.0.0/src/OpenSTBench.egg-info/PKG-INFO +134 -0
- {openstbench-0.3.2 → openstbench-1.0.0}/src/openstbench/__init__.py +5 -11
- openstbench-1.0.0/src/openstbench/paralinguistic_evaluator.py +1054 -0
- openstbench-1.0.0/src/openstbench/speaker_similarity_evaluator.py +163 -0
- {openstbench-0.3.2 → openstbench-1.0.0}/src/openstbench/speech_quality_evaluator.py +31 -1
- openstbench-0.3.2/PKG-INFO +0 -212
- openstbench-0.3.2/README.md +0 -163
- openstbench-0.3.2/src/OpenSTBench.egg-info/PKG-INFO +0 -212
- openstbench-0.3.2/src/openstbench/paralinguistic_evaluator.py +0 -1331
- openstbench-0.3.2/src/openstbench/speaker_similarity_evaluator.py +0 -155
- {openstbench-0.3.2 → openstbench-1.0.0}/setup.cfg +0 -0
- {openstbench-0.3.2 → openstbench-1.0.0}/src/OpenSTBench.egg-info/SOURCES.txt +0 -0
- {openstbench-0.3.2 → openstbench-1.0.0}/src/OpenSTBench.egg-info/dependency_links.txt +0 -0
- {openstbench-0.3.2 → openstbench-1.0.0}/src/OpenSTBench.egg-info/requires.txt +0 -0
- {openstbench-0.3.2 → openstbench-1.0.0}/src/OpenSTBench.egg-info/top_level.txt +0 -0
- {openstbench-0.3.2 → openstbench-1.0.0}/src/openstbench/_model_loading.py +0 -0
- {openstbench-0.3.2 → openstbench-1.0.0}/src/openstbench/dataset.py +0 -0
- {openstbench-0.3.2 → openstbench-1.0.0}/src/openstbench/emotion_evaluator.py +0 -0
- {openstbench-0.3.2 → openstbench-1.0.0}/src/openstbench/latency/__init__.py +0 -0
- {openstbench-0.3.2 → openstbench-1.0.0}/src/openstbench/latency/agent.py +0 -0
- {openstbench-0.3.2 → openstbench-1.0.0}/src/openstbench/latency/basics.py +0 -0
- {openstbench-0.3.2 → openstbench-1.0.0}/src/openstbench/latency/cli.py +0 -0
- {openstbench-0.3.2 → openstbench-1.0.0}/src/openstbench/latency/instance.py +0 -0
- {openstbench-0.3.2 → openstbench-1.0.0}/src/openstbench/latency/metrics.py +0 -0
- {openstbench-0.3.2 → openstbench-1.0.0}/src/openstbench/latency/utils.py +0 -0
- {openstbench-0.3.2 → openstbench-1.0.0}/src/openstbench/temporal_consistency_evaluator.py +0 -0
- {openstbench-0.3.2 → openstbench-1.0.0}/src/openstbench/translation_evaluator.py +0 -0
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: OpenSTBench
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Open speech and translation benchmarking toolkit supporting MT, ASR, TTS, SimulST, VC, and paralinguistics with optimized CJK language support
|
|
5
|
+
Author-email: Yanjie An <691476922@qq.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/sjtuayj/OpenSTBench
|
|
8
|
+
Project-URL: Repository, https://github.com/sjtuayj/OpenSTBench
|
|
9
|
+
Project-URL: Documentation, https://github.com/sjtuayj/OpenSTBench#readme
|
|
10
|
+
Project-URL: Bug Tracker, https://github.com/sjtuayj/OpenSTBench/issues
|
|
11
|
+
Keywords: translation,evaluation,BLEU,COMET,BLEURT,ASR,speech-synthesis,voice-conversion,UTMOS,WER,machine-translation,multilingual,CJK,emotion,paralinguistics,CLAP,audio-events
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Requires-Python: <3.11,>=3.9
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
Requires-Dist: torch<2.7,>=2.6
|
|
20
|
+
Requires-Dist: torchaudio<2.7,>=2.6
|
|
21
|
+
Requires-Dist: numpy
|
|
22
|
+
Requires-Dist: sacrebleu>=2.0.0
|
|
23
|
+
Requires-Dist: pandas
|
|
24
|
+
Requires-Dist: tqdm
|
|
25
|
+
Requires-Dist: scipy
|
|
26
|
+
Requires-Dist: soundfile
|
|
27
|
+
Requires-Dist: textgrid
|
|
28
|
+
Requires-Dist: transformers
|
|
29
|
+
Requires-Dist: matplotlib
|
|
30
|
+
Requires-Dist: resemblyzer
|
|
31
|
+
Provides-Extra: comet
|
|
32
|
+
Requires-Dist: unbabel-comet>=2.2.2; extra == "comet"
|
|
33
|
+
Provides-Extra: whisper
|
|
34
|
+
Requires-Dist: openai-whisper; extra == "whisper"
|
|
35
|
+
Provides-Extra: speech-quality
|
|
36
|
+
Requires-Dist: jiwer; extra == "speech-quality"
|
|
37
|
+
Provides-Extra: emotion
|
|
38
|
+
Requires-Dist: funasr; extra == "emotion"
|
|
39
|
+
Requires-Dist: modelscope; extra == "emotion"
|
|
40
|
+
Provides-Extra: paralinguistics
|
|
41
|
+
Requires-Dist: transformers; extra == "paralinguistics"
|
|
42
|
+
Provides-Extra: all
|
|
43
|
+
Requires-Dist: unbabel-comet>=2.2.2; extra == "all"
|
|
44
|
+
Requires-Dist: openai-whisper; extra == "all"
|
|
45
|
+
Requires-Dist: jiwer; extra == "all"
|
|
46
|
+
Requires-Dist: funasr; extra == "all"
|
|
47
|
+
Requires-Dist: modelscope; extra == "all"
|
|
48
|
+
Requires-Dist: transformers; extra == "all"
|
|
49
|
+
|
|
50
|
+
# OpenSTBench
|
|
51
|
+
|
|
52
|
+
English | [Chinese](./README_zh.md)
|
|
53
|
+
|
|
54
|
+
[](https://pypi.org/project/OpenSTBench/)
|
|
55
|
+
[](https://www.python.org/downloads/)
|
|
56
|
+
[](https://opensource.org/licenses/MIT)
|
|
57
|
+
|
|
58
|
+
OpenSTBench is a speech and translation evaluation toolkit. It covers text translation quality, speech quality, speaker and style preservation, temporal consistency, and streaming latency.
|
|
59
|
+
|
|
60
|
+
## Installation
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
pip install OpenSTBench
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
For local development:
|
|
67
|
+
|
|
68
|
+
```bash
|
|
69
|
+
pip install -e .
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
Optional extras:
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
pip install "OpenSTBench[comet]"
|
|
76
|
+
pip install "OpenSTBench[whisper]"
|
|
77
|
+
pip install "OpenSTBench[speech_quality]"
|
|
78
|
+
pip install "OpenSTBench[emotion]"
|
|
79
|
+
pip install "OpenSTBench[paralinguistics]"
|
|
80
|
+
pip install "OpenSTBench[all]"
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
BLEURT is installed separately:
|
|
84
|
+
|
|
85
|
+
```bash
|
|
86
|
+
pip install git+https://github.com/lucadiliello/bleurt-pytorch.git
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
## Package Names
|
|
90
|
+
|
|
91
|
+
- PyPI package: `OpenSTBench`
|
|
92
|
+
- Python import: `openstbench`
|
|
93
|
+
|
|
94
|
+
## Evaluators
|
|
95
|
+
|
|
96
|
+
| Evaluator | Scope | Main outputs |
|
|
97
|
+
| :--- | :--- | :--- |
|
|
98
|
+
| `TranslationEvaluator` | MT and S2TT text quality | `sacreBLEU`, `chrF++`, `COMET`, `BLEURT` |
|
|
99
|
+
| `SpeechQualityEvaluator` | Generated speech quality and text consistency | `UTMOS`, `WER_Consistency`, `CER_Consistency` |
|
|
100
|
+
| `SpeakerSimilarityEvaluator` | Speaker preservation | `wavlm_similarity`, `resemblyzer_similarity` |
|
|
101
|
+
| `EmotionEvaluator` | Emotion preservation or emotion classification | `Emotion2Vec_Cosine_Similarity`, `Audio_Emotion_Accuracy` |
|
|
102
|
+
| `ParalinguisticEvaluator` | Non-verbal and paralinguistic preservation | `Paralinguistic_Fidelity_Cosine`, `Acoustic_Event_Count_F1`, `Acoustic_Event_Localization_F1` |
|
|
103
|
+
| `TemporalConsistencyEvaluator` | Source-target duration consistency | `Duration_Consistency_SLC_*`, duration diagnostics |
|
|
104
|
+
| `LatencyEvaluator` | Streaming and simultaneous ST latency | `StartOffset`, `ATD`, `CustomATD`, `RTF` |
|
|
105
|
+
|
|
106
|
+
## Examples
|
|
107
|
+
|
|
108
|
+
Usage examples are kept under `examples/`.
|
|
109
|
+
|
|
110
|
+
- `examples/python/translation_eval.py`
|
|
111
|
+
- `examples/python/speech_quality_eval.py`
|
|
112
|
+
- `examples/python/speaker_similarity_eval.py`
|
|
113
|
+
- `examples/python/emotion_eval.py`
|
|
114
|
+
- `examples/python/paralinguistic_eval.py`
|
|
115
|
+
- `examples/python/paralinguistic_identity_baseline.py`
|
|
116
|
+
- `examples/python/temporal_consistency_eval.py`
|
|
117
|
+
- `examples/python/latency_eval.py`
|
|
118
|
+
- `examples/bash/install_extras.sh`
|
|
119
|
+
- `examples/bash/run_latency_cli.sh`
|
|
120
|
+
|
|
121
|
+
Latency can also be run from the module CLI:
|
|
122
|
+
|
|
123
|
+
```bash
|
|
124
|
+
python -m openstbench.latency.cli --help
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
## Conventions
|
|
128
|
+
|
|
129
|
+
- Text inputs generally accept `list[str]`, one-sample-per-line `.txt` files, and `.json` files.
|
|
130
|
+
- Audio inputs generally accept folders, `list[str]`, `.txt` path lists, and `.json` path lists.
|
|
131
|
+
- For `zh`, `ja`, and `ko`, text-side evaluation uses CJK-aware handling; speech consistency reports `CER_Consistency` instead of `WER_Consistency`.
|
|
132
|
+
- Model path arguments use a local-first rule. If the supplied local path exists, OpenSTBench uses it; otherwise it falls back to the configured remote model id.
|
|
133
|
+
- Optional dependencies are loaded only by the evaluator that needs them.
|
|
134
|
+
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
# OpenSTBench
|
|
2
|
+
|
|
3
|
+
English | [Chinese](./README_zh.md)
|
|
4
|
+
|
|
5
|
+
[](https://pypi.org/project/OpenSTBench/)
|
|
6
|
+
[](https://www.python.org/downloads/)
|
|
7
|
+
[](https://opensource.org/licenses/MIT)
|
|
8
|
+
|
|
9
|
+
OpenSTBench is a speech and translation evaluation toolkit. It covers text translation quality, speech quality, speaker and style preservation, temporal consistency, and streaming latency.
|
|
10
|
+
|
|
11
|
+
## Installation
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
pip install OpenSTBench
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
For local development:
|
|
18
|
+
|
|
19
|
+
```bash
|
|
20
|
+
pip install -e .
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
Optional extras:
|
|
24
|
+
|
|
25
|
+
```bash
|
|
26
|
+
pip install "OpenSTBench[comet]"
|
|
27
|
+
pip install "OpenSTBench[whisper]"
|
|
28
|
+
pip install "OpenSTBench[speech_quality]"
|
|
29
|
+
pip install "OpenSTBench[emotion]"
|
|
30
|
+
pip install "OpenSTBench[paralinguistics]"
|
|
31
|
+
pip install "OpenSTBench[all]"
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
BLEURT is installed separately:
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
pip install git+https://github.com/lucadiliello/bleurt-pytorch.git
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
## Package Names
|
|
41
|
+
|
|
42
|
+
- PyPI package: `OpenSTBench`
|
|
43
|
+
- Python import: `openstbench`
|
|
44
|
+
|
|
45
|
+
## Evaluators
|
|
46
|
+
|
|
47
|
+
| Evaluator | Scope | Main outputs |
|
|
48
|
+
| :--- | :--- | :--- |
|
|
49
|
+
| `TranslationEvaluator` | MT and S2TT text quality | `sacreBLEU`, `chrF++`, `COMET`, `BLEURT` |
|
|
50
|
+
| `SpeechQualityEvaluator` | Generated speech quality and text consistency | `UTMOS`, `WER_Consistency`, `CER_Consistency` |
|
|
51
|
+
| `SpeakerSimilarityEvaluator` | Speaker preservation | `wavlm_similarity`, `resemblyzer_similarity` |
|
|
52
|
+
| `EmotionEvaluator` | Emotion preservation or emotion classification | `Emotion2Vec_Cosine_Similarity`, `Audio_Emotion_Accuracy` |
|
|
53
|
+
| `ParalinguisticEvaluator` | Non-verbal and paralinguistic preservation | `Paralinguistic_Fidelity_Cosine`, `Acoustic_Event_Count_F1`, `Acoustic_Event_Localization_F1` |
|
|
54
|
+
| `TemporalConsistencyEvaluator` | Source-target duration consistency | `Duration_Consistency_SLC_*`, duration diagnostics |
|
|
55
|
+
| `LatencyEvaluator` | Streaming and simultaneous ST latency | `StartOffset`, `ATD`, `CustomATD`, `RTF` |
|
|
56
|
+
|
|
57
|
+
## Examples
|
|
58
|
+
|
|
59
|
+
Usage examples are kept under `examples/`.
|
|
60
|
+
|
|
61
|
+
- `examples/python/translation_eval.py`
|
|
62
|
+
- `examples/python/speech_quality_eval.py`
|
|
63
|
+
- `examples/python/speaker_similarity_eval.py`
|
|
64
|
+
- `examples/python/emotion_eval.py`
|
|
65
|
+
- `examples/python/paralinguistic_eval.py`
|
|
66
|
+
- `examples/python/paralinguistic_identity_baseline.py`
|
|
67
|
+
- `examples/python/temporal_consistency_eval.py`
|
|
68
|
+
- `examples/python/latency_eval.py`
|
|
69
|
+
- `examples/bash/install_extras.sh`
|
|
70
|
+
- `examples/bash/run_latency_cli.sh`
|
|
71
|
+
|
|
72
|
+
Latency can also be run from the module CLI:
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
python -m openstbench.latency.cli --help
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
## Conventions
|
|
79
|
+
|
|
80
|
+
- Text inputs generally accept `list[str]`, one-sample-per-line `.txt` files, and `.json` files.
|
|
81
|
+
- Audio inputs generally accept folders, `list[str]`, `.txt` path lists, and `.json` path lists.
|
|
82
|
+
- For `zh`, `ja`, and `ko`, text-side evaluation uses CJK-aware handling; speech consistency reports `CER_Consistency` instead of `WER_Consistency`.
|
|
83
|
+
- Model path arguments use a local-first rule. If the supplied local path exists, OpenSTBench uses it; otherwise it falls back to the configured remote model id.
|
|
84
|
+
- Optional dependencies are loaded only by the evaluator that needs them.
|
|
85
|
+
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "OpenSTBench"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "1.0.0"
|
|
8
8
|
description = "Open speech and translation benchmarking toolkit supporting MT, ASR, TTS, SimulST, VC, and paralinguistics with optimized CJK language support"
|
|
9
9
|
readme = {file = "README.md", content-type = "text/markdown"}
|
|
10
10
|
requires-python = ">=3.9,<3.11"
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: OpenSTBench
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Open speech and translation benchmarking toolkit supporting MT, ASR, TTS, SimulST, VC, and paralinguistics with optimized CJK language support
|
|
5
|
+
Author-email: Yanjie An <691476922@qq.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/sjtuayj/OpenSTBench
|
|
8
|
+
Project-URL: Repository, https://github.com/sjtuayj/OpenSTBench
|
|
9
|
+
Project-URL: Documentation, https://github.com/sjtuayj/OpenSTBench#readme
|
|
10
|
+
Project-URL: Bug Tracker, https://github.com/sjtuayj/OpenSTBench/issues
|
|
11
|
+
Keywords: translation,evaluation,BLEU,COMET,BLEURT,ASR,speech-synthesis,voice-conversion,UTMOS,WER,machine-translation,multilingual,CJK,emotion,paralinguistics,CLAP,audio-events
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Requires-Python: <3.11,>=3.9
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
Requires-Dist: torch<2.7,>=2.6
|
|
20
|
+
Requires-Dist: torchaudio<2.7,>=2.6
|
|
21
|
+
Requires-Dist: numpy
|
|
22
|
+
Requires-Dist: sacrebleu>=2.0.0
|
|
23
|
+
Requires-Dist: pandas
|
|
24
|
+
Requires-Dist: tqdm
|
|
25
|
+
Requires-Dist: scipy
|
|
26
|
+
Requires-Dist: soundfile
|
|
27
|
+
Requires-Dist: textgrid
|
|
28
|
+
Requires-Dist: transformers
|
|
29
|
+
Requires-Dist: matplotlib
|
|
30
|
+
Requires-Dist: resemblyzer
|
|
31
|
+
Provides-Extra: comet
|
|
32
|
+
Requires-Dist: unbabel-comet>=2.2.2; extra == "comet"
|
|
33
|
+
Provides-Extra: whisper
|
|
34
|
+
Requires-Dist: openai-whisper; extra == "whisper"
|
|
35
|
+
Provides-Extra: speech-quality
|
|
36
|
+
Requires-Dist: jiwer; extra == "speech-quality"
|
|
37
|
+
Provides-Extra: emotion
|
|
38
|
+
Requires-Dist: funasr; extra == "emotion"
|
|
39
|
+
Requires-Dist: modelscope; extra == "emotion"
|
|
40
|
+
Provides-Extra: paralinguistics
|
|
41
|
+
Requires-Dist: transformers; extra == "paralinguistics"
|
|
42
|
+
Provides-Extra: all
|
|
43
|
+
Requires-Dist: unbabel-comet>=2.2.2; extra == "all"
|
|
44
|
+
Requires-Dist: openai-whisper; extra == "all"
|
|
45
|
+
Requires-Dist: jiwer; extra == "all"
|
|
46
|
+
Requires-Dist: funasr; extra == "all"
|
|
47
|
+
Requires-Dist: modelscope; extra == "all"
|
|
48
|
+
Requires-Dist: transformers; extra == "all"
|
|
49
|
+
|
|
50
|
+
# OpenSTBench
|
|
51
|
+
|
|
52
|
+
English | [Chinese](./README_zh.md)
|
|
53
|
+
|
|
54
|
+
[](https://pypi.org/project/OpenSTBench/)
|
|
55
|
+
[](https://www.python.org/downloads/)
|
|
56
|
+
[](https://opensource.org/licenses/MIT)
|
|
57
|
+
|
|
58
|
+
OpenSTBench is a speech and translation evaluation toolkit. It covers text translation quality, speech quality, speaker and style preservation, temporal consistency, and streaming latency.
|
|
59
|
+
|
|
60
|
+
## Installation
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
pip install OpenSTBench
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
For local development:
|
|
67
|
+
|
|
68
|
+
```bash
|
|
69
|
+
pip install -e .
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
Optional extras:
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
pip install "OpenSTBench[comet]"
|
|
76
|
+
pip install "OpenSTBench[whisper]"
|
|
77
|
+
pip install "OpenSTBench[speech_quality]"
|
|
78
|
+
pip install "OpenSTBench[emotion]"
|
|
79
|
+
pip install "OpenSTBench[paralinguistics]"
|
|
80
|
+
pip install "OpenSTBench[all]"
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
BLEURT is installed separately:
|
|
84
|
+
|
|
85
|
+
```bash
|
|
86
|
+
pip install git+https://github.com/lucadiliello/bleurt-pytorch.git
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
## Package Names
|
|
90
|
+
|
|
91
|
+
- PyPI package: `OpenSTBench`
|
|
92
|
+
- Python import: `openstbench`
|
|
93
|
+
|
|
94
|
+
## Evaluators
|
|
95
|
+
|
|
96
|
+
| Evaluator | Scope | Main outputs |
|
|
97
|
+
| :--- | :--- | :--- |
|
|
98
|
+
| `TranslationEvaluator` | MT and S2TT text quality | `sacreBLEU`, `chrF++`, `COMET`, `BLEURT` |
|
|
99
|
+
| `SpeechQualityEvaluator` | Generated speech quality and text consistency | `UTMOS`, `WER_Consistency`, `CER_Consistency` |
|
|
100
|
+
| `SpeakerSimilarityEvaluator` | Speaker preservation | `wavlm_similarity`, `resemblyzer_similarity` |
|
|
101
|
+
| `EmotionEvaluator` | Emotion preservation or emotion classification | `Emotion2Vec_Cosine_Similarity`, `Audio_Emotion_Accuracy` |
|
|
102
|
+
| `ParalinguisticEvaluator` | Non-verbal and paralinguistic preservation | `Paralinguistic_Fidelity_Cosine`, `Acoustic_Event_Count_F1`, `Acoustic_Event_Localization_F1` |
|
|
103
|
+
| `TemporalConsistencyEvaluator` | Source-target duration consistency | `Duration_Consistency_SLC_*`, duration diagnostics |
|
|
104
|
+
| `LatencyEvaluator` | Streaming and simultaneous ST latency | `StartOffset`, `ATD`, `CustomATD`, `RTF` |
|
|
105
|
+
|
|
106
|
+
## Examples
|
|
107
|
+
|
|
108
|
+
Usage examples are kept under `examples/`.
|
|
109
|
+
|
|
110
|
+
- `examples/python/translation_eval.py`
|
|
111
|
+
- `examples/python/speech_quality_eval.py`
|
|
112
|
+
- `examples/python/speaker_similarity_eval.py`
|
|
113
|
+
- `examples/python/emotion_eval.py`
|
|
114
|
+
- `examples/python/paralinguistic_eval.py`
|
|
115
|
+
- `examples/python/paralinguistic_identity_baseline.py`
|
|
116
|
+
- `examples/python/temporal_consistency_eval.py`
|
|
117
|
+
- `examples/python/latency_eval.py`
|
|
118
|
+
- `examples/bash/install_extras.sh`
|
|
119
|
+
- `examples/bash/run_latency_cli.sh`
|
|
120
|
+
|
|
121
|
+
Latency can also be run from the module CLI:
|
|
122
|
+
|
|
123
|
+
```bash
|
|
124
|
+
python -m openstbench.latency.cli --help
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
## Conventions
|
|
128
|
+
|
|
129
|
+
- Text inputs generally accept `list[str]`, one-sample-per-line `.txt` files, and `.json` files.
|
|
130
|
+
- Audio inputs generally accept folders, `list[str]`, `.txt` path lists, and `.json` path lists.
|
|
131
|
+
- For `zh`, `ja`, and `ko`, text-side evaluation uses CJK-aware handling; speech consistency reports `CER_Consistency` instead of `WER_Consistency`.
|
|
132
|
+
- Model path arguments use a local-first rule. If the supplied local path exists, OpenSTBench uses it; otherwise it falls back to the configured remote model id.
|
|
133
|
+
- Optional dependencies are loaded only by the evaluator that needs them.
|
|
134
|
+
|
|
@@ -9,20 +9,17 @@ actually accessed.
|
|
|
9
9
|
from importlib import import_module
|
|
10
10
|
from typing import Dict, Tuple
|
|
11
11
|
|
|
12
|
-
__version__ = "0.
|
|
12
|
+
__version__ = "1.0.0"
|
|
13
13
|
|
|
14
14
|
__all__ = [
|
|
15
15
|
"TranslationEvaluator",
|
|
16
16
|
"EmotionEvaluator",
|
|
17
|
+
"AcousticEvent",
|
|
17
18
|
"ParalinguisticEvaluator",
|
|
18
|
-
"BaseAudioEventPredictor",
|
|
19
19
|
"BaseAudioEventLocalizer",
|
|
20
|
-
"ClapAudioEventPredictor",
|
|
21
20
|
"ClapSlidingWindowEventLocalizer",
|
|
22
|
-
"
|
|
23
|
-
"EventLocalization",
|
|
21
|
+
"EventMatchingConfig",
|
|
24
22
|
"EventLocalizationConfig",
|
|
25
|
-
"EventPrediction",
|
|
26
23
|
"EventPredictionConfig",
|
|
27
24
|
"ParalinguisticSample",
|
|
28
25
|
"load_paralinguistic_manifest",
|
|
@@ -61,14 +58,11 @@ _EXPORT_SPECS: Tuple[Tuple[str, Tuple[str, ...]], ...] = (
|
|
|
61
58
|
(
|
|
62
59
|
"paralinguistic_evaluator",
|
|
63
60
|
(
|
|
64
|
-
"
|
|
61
|
+
"AcousticEvent",
|
|
65
62
|
"BaseAudioEventLocalizer",
|
|
66
|
-
"ClapAudioEventPredictor",
|
|
67
63
|
"ClapSlidingWindowEventLocalizer",
|
|
68
|
-
"
|
|
69
|
-
"EventLocalization",
|
|
64
|
+
"EventMatchingConfig",
|
|
70
65
|
"EventLocalizationConfig",
|
|
71
|
-
"EventPrediction",
|
|
72
66
|
"EventPredictionConfig",
|
|
73
67
|
"ParalinguisticEvaluator",
|
|
74
68
|
"ParalinguisticSample",
|