rumblex 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rumblex/__init__.py +72 -0
- rumblex/__main__.py +5 -0
- rumblex/aligner.py +536 -0
- rumblex/audio.py +104 -0
- rumblex/bench.py +564 -0
- rumblex/cli.py +274 -0
- rumblex/diarizer.py +181 -0
- rumblex/py.typed +0 -0
- rumblex/transcriber.py +319 -0
- rumblex/types.py +46 -0
- rumblex/vad.py +261 -0
- rumblex/vllm_backend.py +227 -0
- rumblex/writers.py +281 -0
- rumblex-0.1.0.dist-info/METADATA +230 -0
- rumblex-0.1.0.dist-info/RECORD +18 -0
- rumblex-0.1.0.dist-info/WHEEL +4 -0
- rumblex-0.1.0.dist-info/entry_points.txt +2 -0
- rumblex-0.1.0.dist-info/licenses/LICENSE +202 -0
rumblex/audio.py
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
"""Audio loading: any file ffmpeg can read (wav, mp3, m4a, mp4, mkv, ...) or an in-memory array,
|
|
2
|
+
returned as 16 kHz mono float32 numpy."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import os
|
|
7
|
+
import shutil
|
|
8
|
+
import subprocess
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
import numpy as np
|
|
12
|
+
import torch
|
|
13
|
+
|
|
14
|
+
SAMPLE_RATE = 16_000
|
|
15
|
+
|
|
16
|
+
AudioInput = str | os.PathLike | np.ndarray | torch.Tensor
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class AudioLoadError(RuntimeError):
|
|
20
|
+
pass
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def find_ffmpeg() -> str | None:
|
|
24
|
+
"""`RUMBLEX_FFMPEG` if set, else whatever `ffmpeg` is on PATH."""
|
|
25
|
+
env = os.environ.get("RUMBLEX_FFMPEG")
|
|
26
|
+
if env:
|
|
27
|
+
return env
|
|
28
|
+
return shutil.which("ffmpeg")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def load_audio(source: AudioInput, *, sr: int | None = None) -> np.ndarray:
|
|
32
|
+
"""Return `source` as a 1-D float32 array at 16 kHz.
|
|
33
|
+
|
|
34
|
+
`source` is a path, or an array/tensor of samples. For arrays, `sr` is their sample rate
|
|
35
|
+
(default 16 kHz); 2-D arrays are downmixed, with the shorter axis taken as channels.
|
|
36
|
+
"""
|
|
37
|
+
if isinstance(source, (np.ndarray, torch.Tensor)):
|
|
38
|
+
return _from_array(source, sr if sr is not None else SAMPLE_RATE)
|
|
39
|
+
if sr is not None:
|
|
40
|
+
raise ValueError("sr is only meaningful for array input; files are resampled by ffmpeg")
|
|
41
|
+
return _from_file(Path(source))
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _from_array(x: np.ndarray | torch.Tensor, sr: int) -> np.ndarray:
|
|
45
|
+
if isinstance(x, torch.Tensor):
|
|
46
|
+
x = x.detach().cpu()
|
|
47
|
+
if x.dtype in (torch.bfloat16, torch.float16): # numpy has no bf16
|
|
48
|
+
x = x.float()
|
|
49
|
+
x = x.numpy()
|
|
50
|
+
x = np.asarray(x)
|
|
51
|
+
if x.dtype == np.uint8: # 8-bit PCM is unsigned, centred on 128
|
|
52
|
+
x = (x.astype(np.float32) - 128) / 128
|
|
53
|
+
elif np.issubdtype(x.dtype, np.integer):
|
|
54
|
+
x = x.astype(np.float32) / np.iinfo(x.dtype).max
|
|
55
|
+
x = x.astype(np.float32, copy=False)
|
|
56
|
+
if x.ndim == 2:
|
|
57
|
+
x = x.mean(axis=0 if x.shape[0] <= x.shape[1] else 1)
|
|
58
|
+
elif x.ndim != 1:
|
|
59
|
+
raise ValueError(f"expected 1-D or 2-D audio, got shape {x.shape}")
|
|
60
|
+
if sr != SAMPLE_RATE:
|
|
61
|
+
x = _resample(x, sr)
|
|
62
|
+
return np.ascontiguousarray(x)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _resample(x: np.ndarray, sr: int) -> np.ndarray:
|
|
66
|
+
import torchaudio.functional as F
|
|
67
|
+
|
|
68
|
+
return F.resample(torch.from_numpy(x), sr, SAMPLE_RATE).numpy()
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _from_file(path: Path) -> np.ndarray:
|
|
72
|
+
if not path.is_file():
|
|
73
|
+
raise FileNotFoundError(path)
|
|
74
|
+
ffmpeg = find_ffmpeg()
|
|
75
|
+
if ffmpeg is not None:
|
|
76
|
+
return _decode_ffmpeg(ffmpeg, path)
|
|
77
|
+
# No ffmpeg: libsndfile covers wav/flac/ogg/mp3, but not video containers or m4a.
|
|
78
|
+
try:
|
|
79
|
+
import soundfile as sf
|
|
80
|
+
|
|
81
|
+
data, file_sr = sf.read(str(path), dtype="float32", always_2d=True)
|
|
82
|
+
except Exception as e:
|
|
83
|
+
raise AudioLoadError(
|
|
84
|
+
f"ffmpeg not found and soundfile can't read {path.name} ({e}). "
|
|
85
|
+
"Install ffmpeg or set RUMBLEX_FFMPEG to its path."
|
|
86
|
+
) from e
|
|
87
|
+
return _from_array(data.T, file_sr)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _decode_ffmpeg(ffmpeg: str, path: Path) -> np.ndarray:
|
|
91
|
+
cmd = [
|
|
92
|
+
ffmpeg, "-nostdin", "-hide_banner", "-loglevel", "error", "-threads", "0",
|
|
93
|
+
"-i", str(path),
|
|
94
|
+
"-vn", "-map", "0:a:0", # first audio stream only, skip video
|
|
95
|
+
"-ac", "1", "-ar", str(SAMPLE_RATE), "-f", "f32le", "-acodec", "pcm_f32le", "-",
|
|
96
|
+
] # fmt: skip
|
|
97
|
+
proc = subprocess.run(cmd, check=False, capture_output=True)
|
|
98
|
+
if proc.returncode != 0:
|
|
99
|
+
err = proc.stderr.decode(errors="replace").strip().splitlines()
|
|
100
|
+
raise AudioLoadError(f"ffmpeg failed on {path.name}: {' | '.join(err[-3:])}")
|
|
101
|
+
audio = np.frombuffer(proc.stdout, dtype=np.float32)
|
|
102
|
+
if audio.size == 0:
|
|
103
|
+
raise AudioLoadError(f"no audio decoded from {path.name}")
|
|
104
|
+
return audio.copy() # frombuffer is read-only
|
rumblex/bench.py
ADDED
|
@@ -0,0 +1,564 @@
|
|
|
1
|
+
"""Benchmark building blocks, used by scripts/benchmark.py and the evaluation scripts.
|
|
2
|
+
|
|
3
|
+
- Word-timing scoring against LibriSpeech's word alignments: `gilkeyio/librispeech-alignments` on
|
|
4
|
+
Hugging Face (CC-BY-4.0), which repackages Loren Lugosch's Montreal Forced Aligner alignments
|
|
5
|
+
(zenodo.org/record/2619474) with the audio. These are MFA output, not human labels; in them a word's
|
|
6
|
+
end usually equals the next word's start.
|
|
7
|
+
- WER-only evaluation sets (`EVAL_SETS`): AMI and Earnings-22 from the Open ASR Leaderboard's test sets,
|
|
8
|
+
FLEURS Dutch. Revisions are pinned.
|
|
9
|
+
- WER with Whisper's normalizers (`normalizer_for`), as the Open ASR Leaderboard does.
|
|
10
|
+
- The systems under test (rumblex, rumblex with vLLM, WhisperX) behind one `System.run` interface, and
|
|
11
|
+
the accuracy and long-file speed runs.
|
|
12
|
+
|
|
13
|
+
pyarrow, jiwer and whisper-normalizer come with the `[dev]` extra; WhisperX is not a rumblex dependency.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import io
|
|
19
|
+
import re
|
|
20
|
+
import statistics
|
|
21
|
+
from collections.abc import Callable, Iterator, Sequence
|
|
22
|
+
from dataclasses import dataclass, field
|
|
23
|
+
|
|
24
|
+
import numpy as np
|
|
25
|
+
|
|
26
|
+
from rumblex.audio import SAMPLE_RATE
|
|
27
|
+
|
|
28
|
+
ALIGNMENT_REPO = "gilkeyio/librispeech-alignments"
|
|
29
|
+
ALIGNMENT_REVISION = "0daa1eb43dda38ee6ce752e785555380e5628f5c"
|
|
30
|
+
TOLERANCES_MS = (20, 50, 100, 200)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass(frozen=True)
|
|
34
|
+
class TimedWord:
|
|
35
|
+
word: str
|
|
36
|
+
start: float
|
|
37
|
+
end: float
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass
|
|
41
|
+
class Utterance:
|
|
42
|
+
id: str
|
|
43
|
+
audio: np.ndarray # 16 kHz mono float32
|
|
44
|
+
transcript: str
|
|
45
|
+
words: list[TimedWord]
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
_PUNCT = re.compile(r"[^\w']+")
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def normalize_word(w: str) -> str:
|
|
52
|
+
"""Lowercase, drop punctuation (keep apostrophes: LibriSpeech has "don't", "o'clock")."""
|
|
53
|
+
return _PUNCT.sub("", w.lower()).strip("'")
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def match_words(ref: Sequence[str], hyp: Sequence[str]) -> list[tuple[int, int]]:
|
|
57
|
+
"""Index pairs (i_ref, j_hyp) of equal words on a minimum-edit-distance alignment.
|
|
58
|
+
Among alignments of equal cost, the one with the most matches wins."""
|
|
59
|
+
n, m = len(ref), len(hyp)
|
|
60
|
+
big = n + m + 1 # score = edits * big - matches: fewer edits first, then more matches
|
|
61
|
+
d = np.zeros((n + 1, m + 1), dtype=np.int64)
|
|
62
|
+
d[:, 0] = np.arange(n + 1) * big
|
|
63
|
+
d[0, :] = np.arange(m + 1) * big
|
|
64
|
+
for i in range(1, n + 1):
|
|
65
|
+
for j in range(1, m + 1):
|
|
66
|
+
diag = d[i - 1, j - 1] + (-1 if ref[i - 1] == hyp[j - 1] else big)
|
|
67
|
+
d[i, j] = min(diag, d[i - 1, j] + big, d[i, j - 1] + big)
|
|
68
|
+
pairs = []
|
|
69
|
+
i, j = n, m
|
|
70
|
+
while i > 0 and j > 0:
|
|
71
|
+
eq = ref[i - 1] == hyp[j - 1]
|
|
72
|
+
if d[i, j] == d[i - 1, j - 1] + (-1 if eq else big):
|
|
73
|
+
if eq:
|
|
74
|
+
pairs.append((i - 1, j - 1))
|
|
75
|
+
i, j = i - 1, j - 1
|
|
76
|
+
elif d[i, j] == d[i - 1, j] + big:
|
|
77
|
+
i -= 1
|
|
78
|
+
else:
|
|
79
|
+
j -= 1
|
|
80
|
+
return pairs[::-1]
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
@dataclass
|
|
84
|
+
class TimingScore:
|
|
85
|
+
n_ref: int = 0
|
|
86
|
+
n_hyp: int = 0
|
|
87
|
+
start_err_ms: list[float] = field(default_factory=list) # signed hyp - ref, matched words only
|
|
88
|
+
end_err_ms: list[float] = field(default_factory=list)
|
|
89
|
+
|
|
90
|
+
@property
|
|
91
|
+
def n_matched(self) -> int:
|
|
92
|
+
return len(self.start_err_ms)
|
|
93
|
+
|
|
94
|
+
def add(self, other: TimingScore) -> TimingScore:
|
|
95
|
+
self.n_ref += other.n_ref
|
|
96
|
+
self.n_hyp += other.n_hyp
|
|
97
|
+
self.start_err_ms += other.start_err_ms
|
|
98
|
+
self.end_err_ms += other.end_err_ms
|
|
99
|
+
return self
|
|
100
|
+
|
|
101
|
+
def summary(self) -> dict:
|
|
102
|
+
out: dict = {"n_ref": self.n_ref, "n_hyp": self.n_hyp, "n_matched": self.n_matched}
|
|
103
|
+
for name, errs in (("start", self.start_err_ms), ("end", self.end_err_ms)):
|
|
104
|
+
if not errs:
|
|
105
|
+
continue
|
|
106
|
+
a = [abs(e) for e in errs]
|
|
107
|
+
out[f"{name}_mae_ms"] = round(statistics.fmean(a), 1)
|
|
108
|
+
out[f"{name}_median_ae_ms"] = round(statistics.median(a), 1)
|
|
109
|
+
out[f"{name}_mean_signed_ms"] = round(statistics.fmean(errs), 1)
|
|
110
|
+
for tol in TOLERANCES_MS:
|
|
111
|
+
out[f"{name}_within_{tol}ms"] = round(sum(x <= tol for x in a) / len(a), 4)
|
|
112
|
+
return out
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def score_timings(ref: Sequence[TimedWord], hyp: Sequence[TimedWord]) -> TimingScore:
|
|
116
|
+
"""Compare word timings on the words both sides agree on (after normalisation).
|
|
117
|
+
Words that differ (ASR errors) are counted in n_ref/n_hyp but not timed."""
|
|
118
|
+
ref = [w for w in ref if normalize_word(w.word)]
|
|
119
|
+
hyp = [w for w in hyp if normalize_word(w.word)]
|
|
120
|
+
pairs = match_words([normalize_word(w.word) for w in ref], [normalize_word(w.word) for w in hyp])
|
|
121
|
+
s = TimingScore(n_ref=len(ref), n_hyp=len(hyp))
|
|
122
|
+
for i, j in pairs:
|
|
123
|
+
# rounded to µs so float noise doesn't push e.g. 50 ms over a 50 ms tolerance
|
|
124
|
+
s.start_err_ms.append(round((hyp[j].start - ref[i].start) * 1000, 3))
|
|
125
|
+
s.end_err_ms.append(round((hyp[j].end - ref[i].end) * 1000, 3))
|
|
126
|
+
return s
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def alignment_parquet(split: str = "test_clean") -> str:
|
|
130
|
+
from huggingface_hub import hf_hub_download
|
|
131
|
+
|
|
132
|
+
return hf_hub_download(
|
|
133
|
+
ALIGNMENT_REPO,
|
|
134
|
+
f"data/{split}-00000-of-00001.parquet",
|
|
135
|
+
repo_type="dataset",
|
|
136
|
+
revision=ALIGNMENT_REVISION,
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def librispeech_ids(split: str = "test_clean", path: str | None = None) -> list[str]:
|
|
141
|
+
import pyarrow.parquet as pq
|
|
142
|
+
|
|
143
|
+
return sorted(pq.read_table(path or alignment_parquet(split), columns=["id"]).column("id").to_pylist())
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def sample_ids(split: str, n: int, seed: int = 0, path: str | None = None) -> list[str]:
|
|
147
|
+
"""A fixed random subset of utterance ids (sorted), so runs are comparable."""
|
|
148
|
+
ids = librispeech_ids(split, path)
|
|
149
|
+
rng = np.random.default_rng(seed)
|
|
150
|
+
return sorted(rng.choice(ids, size=min(n, len(ids)), replace=False).tolist())
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def iter_librispeech(
|
|
154
|
+
split: str = "test_clean",
|
|
155
|
+
limit: int | None = None,
|
|
156
|
+
path: str | None = None,
|
|
157
|
+
ids: Sequence[str] | None = None,
|
|
158
|
+
) -> Iterator[Utterance]:
|
|
159
|
+
"""Utterances in id order (deterministic), audio decoded to float32 16 kHz. `ids` restricts to
|
|
160
|
+
those utterances. Empty-word entries (MFA's silence markers) are dropped."""
|
|
161
|
+
import pyarrow.parquet as pq
|
|
162
|
+
import soundfile as sf
|
|
163
|
+
|
|
164
|
+
table = pq.read_table(path or alignment_parquet(split), columns=["id", "audio", "transcript", "words"])
|
|
165
|
+
all_ids = np.asarray(table.column("id").to_pylist())
|
|
166
|
+
order = np.argsort(all_ids)
|
|
167
|
+
if ids is not None:
|
|
168
|
+
wanted = set(ids)
|
|
169
|
+
order = np.array([k for k in order if all_ids[k] in wanted], dtype=np.int64)
|
|
170
|
+
if limit is not None:
|
|
171
|
+
order = order[:limit]
|
|
172
|
+
for k in order:
|
|
173
|
+
row = table.slice(int(k), 1).to_pylist()[0]
|
|
174
|
+
audio, sr = sf.read(io.BytesIO(row["audio"]["bytes"]), dtype="float32")
|
|
175
|
+
if sr != SAMPLE_RATE:
|
|
176
|
+
raise ValueError(f"{row['id']}: expected 16 kHz, got {sr}")
|
|
177
|
+
words = [TimedWord(w["word"], w["start"], w["end"]) for w in row["words"] if w["word"].strip()]
|
|
178
|
+
yield Utterance(row["id"], audio, row["transcript"], words)
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def normalize_text(s: str) -> str:
|
|
182
|
+
"""Lowercase, drop punctuation except apostrophes, collapse spaces (LibriSpeech refs are already so)."""
|
|
183
|
+
s = re.sub(r"[^\w' ]+", " ", s.lower())
|
|
184
|
+
return " ".join(w.strip("'") for w in s.split() if w.strip("'"))
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def english_normalizer() -> tuple[str, Callable[[str], str]]:
|
|
188
|
+
"""Whisper's English normalizer (numbers, spellings, contractions) when the `whisper-normalizer`
|
|
189
|
+
package is installed, as on the Open ASR Leaderboard; otherwise `normalize_text`. Returns (name, fn)."""
|
|
190
|
+
try:
|
|
191
|
+
from whisper_normalizer.english import EnglishTextNormalizer
|
|
192
|
+
except ImportError:
|
|
193
|
+
return "basic", normalize_text
|
|
194
|
+
return "whisper-english", EnglishTextNormalizer()
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def normalizer_for(language: str | None) -> tuple[str, Callable[[str], str]]:
|
|
198
|
+
"""English: Whisper's English normalizer. Other languages: Whisper's basic normalizer (lowercase, drop
|
|
199
|
+
punctuation and bracketed text, keep diacritics), as the Open ASR Leaderboard's multilingual track."""
|
|
200
|
+
if language is None or language.lower() in ("english", "en"):
|
|
201
|
+
return english_normalizer()
|
|
202
|
+
try:
|
|
203
|
+
from whisper_normalizer.basic import BasicTextNormalizer
|
|
204
|
+
except ImportError:
|
|
205
|
+
return "basic", normalize_text
|
|
206
|
+
return "whisper-basic", BasicTextNormalizer()
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
# Extra evaluation sets: WER (and speed) only, no reference word timings.
|
|
210
|
+
ESB_REPO, ESB_REVISION = "hf-audio/esb-datasets-test-only-sorted", "de79e57adf9983993bcc72586d7313460825f5eb"
|
|
211
|
+
FLEURS_REPO, FLEURS_REVISION = "google/fleurs", "70bb2e84b976b7e960aa89f1c648e09c59f894dd"
|
|
212
|
+
EVAL_SETS = {
|
|
213
|
+
# name: (repo, revision, parquet prefix, id column, text column, language)
|
|
214
|
+
"ami": (ESB_REPO, ESB_REVISION, "ami/test-", "id", "text", "English"), # meetings, CC-BY-4.0
|
|
215
|
+
"earnings22": (ESB_REPO, ESB_REVISION, "earnings22/test-", "id", "text", "English"), # CC-BY-SA-4.0
|
|
216
|
+
"fleurs_nl": (
|
|
217
|
+
FLEURS_REPO,
|
|
218
|
+
FLEURS_REVISION,
|
|
219
|
+
"parquet-data/nl_nl/test-",
|
|
220
|
+
"id",
|
|
221
|
+
"raw_transcription",
|
|
222
|
+
"Dutch",
|
|
223
|
+
),
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def load_eval_set(name: str, n: int, seed: int = 0) -> tuple[list[Utterance], str]:
|
|
228
|
+
"""A fixed random sample of `n` utterances (all if fewer) and the set's language. Utterances whose
|
|
229
|
+
reference is empty are skipped before sampling."""
|
|
230
|
+
import pyarrow.parquet as pq
|
|
231
|
+
import soundfile as sf
|
|
232
|
+
from huggingface_hub import HfApi, hf_hub_download
|
|
233
|
+
|
|
234
|
+
repo, rev, prefix, id_col, text_col, language = EVAL_SETS[name]
|
|
235
|
+
files = sorted(
|
|
236
|
+
f for f in HfApi().list_repo_files(repo, repo_type="dataset", revision=rev)
|
|
237
|
+
if f.startswith(prefix) and f.endswith(".parquet")
|
|
238
|
+
) # fmt: skip
|
|
239
|
+
tables = [pq.read_table(hf_hub_download(repo, f, repo_type="dataset", revision=rev)) for f in files]
|
|
240
|
+
rows = [(t, i) for t in tables for i in range(t.num_rows) if t.column(text_col)[i].as_py().strip()]
|
|
241
|
+
rng = np.random.default_rng(seed)
|
|
242
|
+
pick = sorted(rng.choice(len(rows), size=min(n, len(rows)), replace=False).tolist())
|
|
243
|
+
utts = []
|
|
244
|
+
for k in pick:
|
|
245
|
+
t, i = rows[k]
|
|
246
|
+
audio, sr = sf.read(io.BytesIO(t.column("audio")[i].as_py()["bytes"]), dtype="float32")
|
|
247
|
+
if audio.ndim == 2:
|
|
248
|
+
audio = audio.mean(axis=1)
|
|
249
|
+
if sr != SAMPLE_RATE:
|
|
250
|
+
from rumblex.audio import load_audio
|
|
251
|
+
|
|
252
|
+
audio = load_audio(audio, sr=sr)
|
|
253
|
+
utts.append(Utterance(str(t.column(id_col)[i].as_py()), audio, t.column(text_col)[i].as_py(), []))
|
|
254
|
+
return utts, language
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def wer(refs: Sequence[str], hyps: Sequence[str], norm: Callable[[str], str] = normalize_text) -> dict:
|
|
258
|
+
import jiwer
|
|
259
|
+
|
|
260
|
+
r, h = [norm(x) for x in refs], [norm(x) for x in hyps]
|
|
261
|
+
keep = [i for i, x in enumerate(r) if x.strip()] # jiwer rejects empty references
|
|
262
|
+
o = jiwer.process_words([r[i] for i in keep], [h[i] for i in keep])
|
|
263
|
+
return {
|
|
264
|
+
"wer": round(o.wer, 4),
|
|
265
|
+
"sub": o.substitutions,
|
|
266
|
+
"del": o.deletions,
|
|
267
|
+
"ins": o.insertions,
|
|
268
|
+
"ref_words": o.hits + o.substitutions + o.deletions,
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
@dataclass
|
|
273
|
+
class SystemOutput:
|
|
274
|
+
text: str
|
|
275
|
+
words: list[TimedWord]
|
|
276
|
+
timings: dict[str, float] = field(default_factory=dict) # seconds per stage
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
class System:
|
|
280
|
+
"""One ASR + word-alignment system under test. `load` is timed separately from `run`."""
|
|
281
|
+
|
|
282
|
+
name = "system"
|
|
283
|
+
|
|
284
|
+
def load(self) -> None:
|
|
285
|
+
pass
|
|
286
|
+
|
|
287
|
+
def run(self, audio: np.ndarray, language: str | None) -> SystemOutput:
|
|
288
|
+
raise NotImplementedError
|
|
289
|
+
|
|
290
|
+
def info(self) -> dict:
|
|
291
|
+
return {}
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
class RumbleXSystem(System):
|
|
295
|
+
name = "rumblex"
|
|
296
|
+
|
|
297
|
+
def __init__(
|
|
298
|
+
self,
|
|
299
|
+
model: str = "Reson8/Rumble-2-Flash",
|
|
300
|
+
device: str | None = None,
|
|
301
|
+
compute_type: str | None = None,
|
|
302
|
+
batch_size: int = 8,
|
|
303
|
+
digits: bool = False,
|
|
304
|
+
max_chunk_s: float = 20.0,
|
|
305
|
+
):
|
|
306
|
+
self.kw = {"model": model, "device": device, "compute_type": compute_type, "batch_size": batch_size}
|
|
307
|
+
self.digits = digits
|
|
308
|
+
self.max_chunk_s = max_chunk_s
|
|
309
|
+
|
|
310
|
+
def load(self) -> None:
|
|
311
|
+
from rumblex import load_model
|
|
312
|
+
from rumblex.aligner import Aligner
|
|
313
|
+
|
|
314
|
+
self.tr = load_model(**self.kw)
|
|
315
|
+
self.al = Aligner.from_transcriber(self.tr)
|
|
316
|
+
|
|
317
|
+
def run(self, audio: np.ndarray, language: str | None) -> SystemOutput:
|
|
318
|
+
t0 = _now()
|
|
319
|
+
res = self.tr.transcribe(
|
|
320
|
+
audio,
|
|
321
|
+
language=language,
|
|
322
|
+
digits=self.digits,
|
|
323
|
+
keep_encoder_states=True,
|
|
324
|
+
max_chunk_s=self.max_chunk_s,
|
|
325
|
+
)
|
|
326
|
+
t1 = _now()
|
|
327
|
+
self.al.align(res, audio)
|
|
328
|
+
t2 = _now()
|
|
329
|
+
words = [TimedWord(w.word, w.start, w.end) for s in res.segments for w in s.words]
|
|
330
|
+
text = " ".join(s.text for s in res.segments)
|
|
331
|
+
return SystemOutput(text, words, {"transcribe": t1 - t0, "align": t2 - t1})
|
|
332
|
+
|
|
333
|
+
def info(self) -> dict:
|
|
334
|
+
return {
|
|
335
|
+
**self.kw,
|
|
336
|
+
"dtype": str(self.tr.dtype),
|
|
337
|
+
"digits": self.digits,
|
|
338
|
+
"max_chunk_s": self.max_chunk_s,
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
class RumbleXVLLMSystem(RumbleXSystem):
|
|
343
|
+
"""rumblex with decoding on a vLLM server (`rumblex.vllm_backend`); alignment as in `RumbleXSystem`."""
|
|
344
|
+
|
|
345
|
+
name = "rumblex-vllm"
|
|
346
|
+
|
|
347
|
+
def __init__(self, base_url: str = "http://localhost:8000/v1", concurrency: int = 32, **kw):
|
|
348
|
+
super().__init__(**kw)
|
|
349
|
+
self.base_url, self.concurrency = base_url, concurrency
|
|
350
|
+
|
|
351
|
+
def load(self) -> None:
|
|
352
|
+
from rumblex.aligner import Aligner, AlignOptions, CTCHead
|
|
353
|
+
from rumblex.vllm_backend import VLLMTranscriber
|
|
354
|
+
|
|
355
|
+
self.tr = VLLMTranscriber(
|
|
356
|
+
self.base_url,
|
|
357
|
+
self.kw["model"],
|
|
358
|
+
concurrency=self.concurrency,
|
|
359
|
+
batch_size=self.kw["batch_size"],
|
|
360
|
+
device=self.kw["device"],
|
|
361
|
+
dtype=self.kw["compute_type"],
|
|
362
|
+
)
|
|
363
|
+
head = CTCHead.from_pretrained(self.kw["model"], device=self.tr.device)
|
|
364
|
+
self.al = Aligner(head, self.tr.tokenizer, AlignOptions())
|
|
365
|
+
|
|
366
|
+
def info(self) -> dict:
|
|
367
|
+
return {
|
|
368
|
+
**super().info(),
|
|
369
|
+
"backend": "vllm",
|
|
370
|
+
"base_url": self.base_url,
|
|
371
|
+
"concurrency": self.concurrency,
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
class WhisperXSystem(System):
|
|
376
|
+
"""WhisperX (faster-whisper + wav2vec2 alignment). Needs `pip install whisperx` (not a rumblex dependency)."""
|
|
377
|
+
|
|
378
|
+
name = "whisperx"
|
|
379
|
+
|
|
380
|
+
def __init__(
|
|
381
|
+
self,
|
|
382
|
+
model: str = "large-v3",
|
|
383
|
+
device: str = "cuda",
|
|
384
|
+
compute_type: str = "float16",
|
|
385
|
+
batch_size: int = 16,
|
|
386
|
+
):
|
|
387
|
+
self.kw = {"model": model, "device": device, "compute_type": compute_type, "batch_size": batch_size}
|
|
388
|
+
self._align: dict = {}
|
|
389
|
+
|
|
390
|
+
def load(self) -> None:
|
|
391
|
+
import whisperx
|
|
392
|
+
|
|
393
|
+
self.wx = whisperx
|
|
394
|
+
self.model = whisperx.load_model(
|
|
395
|
+
self.kw["model"], self.kw["device"], compute_type=self.kw["compute_type"]
|
|
396
|
+
)
|
|
397
|
+
|
|
398
|
+
def run(self, audio: np.ndarray, language: str | None) -> SystemOutput:
|
|
399
|
+
lang = _iso(language)
|
|
400
|
+
t0 = _now()
|
|
401
|
+
res = self.model.transcribe(audio, batch_size=self.kw["batch_size"], language=lang)
|
|
402
|
+
t1 = _now()
|
|
403
|
+
code = res.get("language") or lang or "en"
|
|
404
|
+
if code not in self._align: # alignment model load is a one-off, kept out of the timing
|
|
405
|
+
self._align[code] = self.wx.load_align_model(language_code=code, device=self.kw["device"])
|
|
406
|
+
t2 = _now()
|
|
407
|
+
model_a, meta = self._align[code]
|
|
408
|
+
aligned = self.wx.align(
|
|
409
|
+
res["segments"], model_a, meta, audio, self.kw["device"], return_char_alignments=False
|
|
410
|
+
)
|
|
411
|
+
t3 = _now()
|
|
412
|
+
words = [
|
|
413
|
+
TimedWord(w["word"], float(w["start"]), float(w["end"]))
|
|
414
|
+
for s in aligned["segments"]
|
|
415
|
+
for w in s.get("words", [])
|
|
416
|
+
if "start" in w and "end" in w # WhisperX leaves some tokens (e.g. bare numbers) untimed
|
|
417
|
+
]
|
|
418
|
+
text = " ".join(s["text"].strip() for s in res["segments"])
|
|
419
|
+
return SystemOutput(text, words, {"transcribe": t1 - t0, "align": t3 - t2})
|
|
420
|
+
|
|
421
|
+
def info(self) -> dict:
|
|
422
|
+
import importlib.metadata as md
|
|
423
|
+
|
|
424
|
+
return {**self.kw, "whisperx": md.version("whisperx")}
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
_ISO = {"english": "en", "dutch": "nl", "german": "de", "french": "fr", "spanish": "es"}
|
|
428
|
+
|
|
429
|
+
|
|
430
|
+
def _iso(language: str | None) -> str | None:
|
|
431
|
+
if language is None:
|
|
432
|
+
return None
|
|
433
|
+
return _ISO.get(language.lower(), language.lower())
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
def _now() -> float:
|
|
437
|
+
import time
|
|
438
|
+
|
|
439
|
+
try:
|
|
440
|
+
import torch
|
|
441
|
+
|
|
442
|
+
if torch.cuda.is_available():
|
|
443
|
+
torch.cuda.synchronize()
|
|
444
|
+
except ImportError:
|
|
445
|
+
pass
|
|
446
|
+
return time.perf_counter()
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
class GpuMemorySampler:
|
|
450
|
+
"""Peak memory used on the GPU while running, sampled from nvidia-smi. It counts memory torch doesn't
|
|
451
|
+
see (e.g. CTranslate2's), so both systems are measured the same way. None without nvidia-smi."""
|
|
452
|
+
|
|
453
|
+
def __init__(self, interval: float = 0.2, index: int = 0):
|
|
454
|
+
self.interval, self.index, self.peak_mb = interval, index, None
|
|
455
|
+
|
|
456
|
+
def _read(self) -> int | None:
|
|
457
|
+
import subprocess
|
|
458
|
+
|
|
459
|
+
cmd = ["nvidia-smi", f"--id={self.index}", "--query-gpu=memory.used", "--format=csv,noheader,nounits"]
|
|
460
|
+
try:
|
|
461
|
+
out = subprocess.run(cmd, capture_output=True, text=True, timeout=5, check=True).stdout
|
|
462
|
+
return int(out.strip().splitlines()[0])
|
|
463
|
+
except Exception: # noqa: BLE001 - no GPU / no nvidia-smi: just no reading
|
|
464
|
+
return None
|
|
465
|
+
|
|
466
|
+
def __enter__(self):
|
|
467
|
+
import threading
|
|
468
|
+
|
|
469
|
+
self._stop = threading.Event()
|
|
470
|
+
|
|
471
|
+
def loop():
|
|
472
|
+
while not self._stop.is_set():
|
|
473
|
+
v = self._read()
|
|
474
|
+
if v is not None:
|
|
475
|
+
self.peak_mb = max(self.peak_mb or 0, v)
|
|
476
|
+
self._stop.wait(self.interval)
|
|
477
|
+
|
|
478
|
+
self._t = threading.Thread(target=loop, daemon=True)
|
|
479
|
+
self._t.start()
|
|
480
|
+
return self
|
|
481
|
+
|
|
482
|
+
def __exit__(self, *exc):
|
|
483
|
+
self._stop.set()
|
|
484
|
+
self._t.join()
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
def run_accuracy(
|
|
488
|
+
system: System,
|
|
489
|
+
utts: Sequence[Utterance],
|
|
490
|
+
language: str | None,
|
|
491
|
+
norm: Callable[[str], str] = normalize_text,
|
|
492
|
+
) -> dict:
|
|
493
|
+
"""One run per utterance: WER over all of them, and word timing errors against the references."""
|
|
494
|
+
hyps, score, secs = [], TimingScore(), 0.0
|
|
495
|
+
for u in utts:
|
|
496
|
+
out = system.run(u.audio, language)
|
|
497
|
+
secs += sum(out.timings.values())
|
|
498
|
+
hyps.append(out.text)
|
|
499
|
+
score.add(score_timings(u.words, out.words))
|
|
500
|
+
audio_s = sum(len(u.audio) for u in utts) / SAMPLE_RATE
|
|
501
|
+
return {
|
|
502
|
+
"utterances": len(utts),
|
|
503
|
+
"audio_seconds": round(audio_s, 1),
|
|
504
|
+
**wer([u.transcript for u in utts], hyps, norm),
|
|
505
|
+
"timing": score.summary(),
|
|
506
|
+
"rtfx_per_utterance": round(audio_s / secs, 1) if secs else None,
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
|
|
510
|
+
def run_speed(system: System, audio: np.ndarray, language: str | None) -> dict:
|
|
511
|
+
"""One long recording end to end: stage times, real-time factor, peak GPU memory."""
|
|
512
|
+
with GpuMemorySampler() as mem:
|
|
513
|
+
t0 = _now()
|
|
514
|
+
out = system.run(audio, language)
|
|
515
|
+
total = _now() - t0
|
|
516
|
+
dur = len(audio) / SAMPLE_RATE
|
|
517
|
+
return {
|
|
518
|
+
"audio_seconds": round(dur, 1),
|
|
519
|
+
"total_seconds": round(total, 2),
|
|
520
|
+
"rtfx": round(dur / total, 1) if total else None,
|
|
521
|
+
"stages": {k: round(v, 2) for k, v in out.timings.items()},
|
|
522
|
+
"words": len(out.words),
|
|
523
|
+
"peak_gpu_mb": mem.peak_mb,
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
|
|
527
|
+
def long_audio_from_librispeech(utts: Sequence[Utterance], seconds: float, gap: float = 0.5) -> np.ndarray:
|
|
528
|
+
"""Real speech for speed runs: utterances back to back with short gaps, cycled up to `seconds`."""
|
|
529
|
+
if not utts:
|
|
530
|
+
return np.zeros(0, np.float32)
|
|
531
|
+
target = int(seconds * SAMPLE_RATE)
|
|
532
|
+
g = np.zeros(int(gap * SAMPLE_RATE), np.float32)
|
|
533
|
+
pieces, total = [], 0
|
|
534
|
+
while total < target:
|
|
535
|
+
for u in utts:
|
|
536
|
+
pieces += [u.audio, g]
|
|
537
|
+
total += len(u.audio) + len(g)
|
|
538
|
+
if total >= target:
|
|
539
|
+
break
|
|
540
|
+
return np.concatenate(pieces)[:target]
|
|
541
|
+
|
|
542
|
+
|
|
543
|
+
def _pct(x) -> str:
|
|
544
|
+
return f"{100 * x:.1f}%" if isinstance(x, (int, float)) else "n/a"
|
|
545
|
+
|
|
546
|
+
|
|
547
|
+
def _unit(x, unit: str) -> str:
|
|
548
|
+
return "n/a" if x is None else f"{x}{unit}"
|
|
549
|
+
|
|
550
|
+
|
|
551
|
+
def markdown_table(results: dict) -> str:
|
|
552
|
+
rows = [
|
|
553
|
+
"| system | WER | start MAE | starts ≤ 100 ms | end MAE | ends ≤ 100 ms | long file speed | peak GPU |",
|
|
554
|
+
"|---|---|---|---|---|---|---|---|",
|
|
555
|
+
]
|
|
556
|
+
for name, r in results.items():
|
|
557
|
+
a, s = r.get("accuracy", {}), r.get("speed", {})
|
|
558
|
+
t = a.get("timing", {})
|
|
559
|
+
rows.append(
|
|
560
|
+
f"| {name} | {_pct(a.get('wer'))} | {_unit(t.get('start_mae_ms'), ' ms')}"
|
|
561
|
+
f" | {_pct(t.get('start_within_100ms'))} | {_unit(t.get('end_mae_ms'), ' ms')}"
|
|
562
|
+
f" | {_pct(t.get('end_within_100ms'))} | {_unit(s.get('rtfx'), 'x')} | {_unit(s.get('peak_gpu_mb'), ' MB')} |"
|
|
563
|
+
)
|
|
564
|
+
return "\n".join(rows)
|