rumblex 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
rumblex/audio.py ADDED
@@ -0,0 +1,104 @@
1
+ """Audio loading: any file ffmpeg can read (wav, mp3, m4a, mp4, mkv, ...) or an in-memory array,
2
+ returned as 16 kHz mono float32 numpy."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import os
7
+ import shutil
8
+ import subprocess
9
+ from pathlib import Path
10
+
11
+ import numpy as np
12
+ import torch
13
+
14
+ SAMPLE_RATE = 16_000
15
+
16
+ AudioInput = str | os.PathLike | np.ndarray | torch.Tensor
17
+
18
+
19
+ class AudioLoadError(RuntimeError):
20
+ pass
21
+
22
+
23
+ def find_ffmpeg() -> str | None:
24
+ """`RUMBLEX_FFMPEG` if set, else whatever `ffmpeg` is on PATH."""
25
+ env = os.environ.get("RUMBLEX_FFMPEG")
26
+ if env:
27
+ return env
28
+ return shutil.which("ffmpeg")
29
+
30
+
31
+ def load_audio(source: AudioInput, *, sr: int | None = None) -> np.ndarray:
32
+ """Return `source` as a 1-D float32 array at 16 kHz.
33
+
34
+ `source` is a path, or an array/tensor of samples. For arrays, `sr` is their sample rate
35
+ (default 16 kHz); 2-D arrays are downmixed, with the shorter axis taken as channels.
36
+ """
37
+ if isinstance(source, (np.ndarray, torch.Tensor)):
38
+ return _from_array(source, sr if sr is not None else SAMPLE_RATE)
39
+ if sr is not None:
40
+ raise ValueError("sr is only meaningful for array input; files are resampled by ffmpeg")
41
+ return _from_file(Path(source))
42
+
43
+
44
+ def _from_array(x: np.ndarray | torch.Tensor, sr: int) -> np.ndarray:
45
+ if isinstance(x, torch.Tensor):
46
+ x = x.detach().cpu()
47
+ if x.dtype in (torch.bfloat16, torch.float16): # numpy has no bf16
48
+ x = x.float()
49
+ x = x.numpy()
50
+ x = np.asarray(x)
51
+ if x.dtype == np.uint8: # 8-bit PCM is unsigned, centred on 128
52
+ x = (x.astype(np.float32) - 128) / 128
53
+ elif np.issubdtype(x.dtype, np.integer):
54
+ x = x.astype(np.float32) / np.iinfo(x.dtype).max
55
+ x = x.astype(np.float32, copy=False)
56
+ if x.ndim == 2:
57
+ x = x.mean(axis=0 if x.shape[0] <= x.shape[1] else 1)
58
+ elif x.ndim != 1:
59
+ raise ValueError(f"expected 1-D or 2-D audio, got shape {x.shape}")
60
+ if sr != SAMPLE_RATE:
61
+ x = _resample(x, sr)
62
+ return np.ascontiguousarray(x)
63
+
64
+
65
+ def _resample(x: np.ndarray, sr: int) -> np.ndarray:
66
+ import torchaudio.functional as F
67
+
68
+ return F.resample(torch.from_numpy(x), sr, SAMPLE_RATE).numpy()
69
+
70
+
71
+ def _from_file(path: Path) -> np.ndarray:
72
+ if not path.is_file():
73
+ raise FileNotFoundError(path)
74
+ ffmpeg = find_ffmpeg()
75
+ if ffmpeg is not None:
76
+ return _decode_ffmpeg(ffmpeg, path)
77
+ # No ffmpeg: libsndfile covers wav/flac/ogg/mp3, but not video containers or m4a.
78
+ try:
79
+ import soundfile as sf
80
+
81
+ data, file_sr = sf.read(str(path), dtype="float32", always_2d=True)
82
+ except Exception as e:
83
+ raise AudioLoadError(
84
+ f"ffmpeg not found and soundfile can't read {path.name} ({e}). "
85
+ "Install ffmpeg or set RUMBLEX_FFMPEG to its path."
86
+ ) from e
87
+ return _from_array(data.T, file_sr)
88
+
89
+
90
+ def _decode_ffmpeg(ffmpeg: str, path: Path) -> np.ndarray:
91
+ cmd = [
92
+ ffmpeg, "-nostdin", "-hide_banner", "-loglevel", "error", "-threads", "0",
93
+ "-i", str(path),
94
+ "-vn", "-map", "0:a:0", # first audio stream only, skip video
95
+ "-ac", "1", "-ar", str(SAMPLE_RATE), "-f", "f32le", "-acodec", "pcm_f32le", "-",
96
+ ] # fmt: skip
97
+ proc = subprocess.run(cmd, check=False, capture_output=True)
98
+ if proc.returncode != 0:
99
+ err = proc.stderr.decode(errors="replace").strip().splitlines()
100
+ raise AudioLoadError(f"ffmpeg failed on {path.name}: {' | '.join(err[-3:])}")
101
+ audio = np.frombuffer(proc.stdout, dtype=np.float32)
102
+ if audio.size == 0:
103
+ raise AudioLoadError(f"no audio decoded from {path.name}")
104
+ return audio.copy() # frombuffer is read-only
rumblex/bench.py ADDED
@@ -0,0 +1,564 @@
1
+ """Benchmark building blocks, used by scripts/benchmark.py and the evaluation scripts.
2
+
3
+ - Word-timing scoring against LibriSpeech's word alignments: `gilkeyio/librispeech-alignments` on
4
+ Hugging Face (CC-BY-4.0), which repackages Loren Lugosch's Montreal Forced Aligner alignments
5
+ (zenodo.org/record/2619474) with the audio. These are MFA output, not human labels; in them a word's
6
+ end usually equals the next word's start.
7
+ - WER-only evaluation sets (`EVAL_SETS`): AMI and Earnings-22 from the Open ASR Leaderboard's test sets,
8
+ FLEURS Dutch. Revisions are pinned.
9
+ - WER with Whisper's normalizers (`normalizer_for`), as the Open ASR Leaderboard does.
10
+ - The systems under test (rumblex, rumblex with vLLM, WhisperX) behind one `System.run` interface, and
11
+ the accuracy and long-file speed runs.
12
+
13
+ pyarrow, jiwer and whisper-normalizer come with the `[dev]` extra; WhisperX is not a rumblex dependency.
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import io
19
+ import re
20
+ import statistics
21
+ from collections.abc import Callable, Iterator, Sequence
22
+ from dataclasses import dataclass, field
23
+
24
+ import numpy as np
25
+
26
+ from rumblex.audio import SAMPLE_RATE
27
+
28
+ ALIGNMENT_REPO = "gilkeyio/librispeech-alignments"
29
+ ALIGNMENT_REVISION = "0daa1eb43dda38ee6ce752e785555380e5628f5c"
30
+ TOLERANCES_MS = (20, 50, 100, 200)
31
+
32
+
33
+ @dataclass(frozen=True)
34
+ class TimedWord:
35
+ word: str
36
+ start: float
37
+ end: float
38
+
39
+
40
+ @dataclass
41
+ class Utterance:
42
+ id: str
43
+ audio: np.ndarray # 16 kHz mono float32
44
+ transcript: str
45
+ words: list[TimedWord]
46
+
47
+
48
+ _PUNCT = re.compile(r"[^\w']+")
49
+
50
+
51
+ def normalize_word(w: str) -> str:
52
+ """Lowercase, drop punctuation (keep apostrophes: LibriSpeech has "don't", "o'clock")."""
53
+ return _PUNCT.sub("", w.lower()).strip("'")
54
+
55
+
56
+ def match_words(ref: Sequence[str], hyp: Sequence[str]) -> list[tuple[int, int]]:
57
+ """Index pairs (i_ref, j_hyp) of equal words on a minimum-edit-distance alignment.
58
+ Among alignments of equal cost, the one with the most matches wins."""
59
+ n, m = len(ref), len(hyp)
60
+ big = n + m + 1 # score = edits * big - matches: fewer edits first, then more matches
61
+ d = np.zeros((n + 1, m + 1), dtype=np.int64)
62
+ d[:, 0] = np.arange(n + 1) * big
63
+ d[0, :] = np.arange(m + 1) * big
64
+ for i in range(1, n + 1):
65
+ for j in range(1, m + 1):
66
+ diag = d[i - 1, j - 1] + (-1 if ref[i - 1] == hyp[j - 1] else big)
67
+ d[i, j] = min(diag, d[i - 1, j] + big, d[i, j - 1] + big)
68
+ pairs = []
69
+ i, j = n, m
70
+ while i > 0 and j > 0:
71
+ eq = ref[i - 1] == hyp[j - 1]
72
+ if d[i, j] == d[i - 1, j - 1] + (-1 if eq else big):
73
+ if eq:
74
+ pairs.append((i - 1, j - 1))
75
+ i, j = i - 1, j - 1
76
+ elif d[i, j] == d[i - 1, j] + big:
77
+ i -= 1
78
+ else:
79
+ j -= 1
80
+ return pairs[::-1]
81
+
82
+
83
+ @dataclass
84
+ class TimingScore:
85
+ n_ref: int = 0
86
+ n_hyp: int = 0
87
+ start_err_ms: list[float] = field(default_factory=list) # signed hyp - ref, matched words only
88
+ end_err_ms: list[float] = field(default_factory=list)
89
+
90
+ @property
91
+ def n_matched(self) -> int:
92
+ return len(self.start_err_ms)
93
+
94
+ def add(self, other: TimingScore) -> TimingScore:
95
+ self.n_ref += other.n_ref
96
+ self.n_hyp += other.n_hyp
97
+ self.start_err_ms += other.start_err_ms
98
+ self.end_err_ms += other.end_err_ms
99
+ return self
100
+
101
+ def summary(self) -> dict:
102
+ out: dict = {"n_ref": self.n_ref, "n_hyp": self.n_hyp, "n_matched": self.n_matched}
103
+ for name, errs in (("start", self.start_err_ms), ("end", self.end_err_ms)):
104
+ if not errs:
105
+ continue
106
+ a = [abs(e) for e in errs]
107
+ out[f"{name}_mae_ms"] = round(statistics.fmean(a), 1)
108
+ out[f"{name}_median_ae_ms"] = round(statistics.median(a), 1)
109
+ out[f"{name}_mean_signed_ms"] = round(statistics.fmean(errs), 1)
110
+ for tol in TOLERANCES_MS:
111
+ out[f"{name}_within_{tol}ms"] = round(sum(x <= tol for x in a) / len(a), 4)
112
+ return out
113
+
114
+
115
+ def score_timings(ref: Sequence[TimedWord], hyp: Sequence[TimedWord]) -> TimingScore:
116
+ """Compare word timings on the words both sides agree on (after normalisation).
117
+ Words that differ (ASR errors) are counted in n_ref/n_hyp but not timed."""
118
+ ref = [w for w in ref if normalize_word(w.word)]
119
+ hyp = [w for w in hyp if normalize_word(w.word)]
120
+ pairs = match_words([normalize_word(w.word) for w in ref], [normalize_word(w.word) for w in hyp])
121
+ s = TimingScore(n_ref=len(ref), n_hyp=len(hyp))
122
+ for i, j in pairs:
123
+ # rounded to µs so float noise doesn't push e.g. 50 ms over a 50 ms tolerance
124
+ s.start_err_ms.append(round((hyp[j].start - ref[i].start) * 1000, 3))
125
+ s.end_err_ms.append(round((hyp[j].end - ref[i].end) * 1000, 3))
126
+ return s
127
+
128
+
129
+ def alignment_parquet(split: str = "test_clean") -> str:
130
+ from huggingface_hub import hf_hub_download
131
+
132
+ return hf_hub_download(
133
+ ALIGNMENT_REPO,
134
+ f"data/{split}-00000-of-00001.parquet",
135
+ repo_type="dataset",
136
+ revision=ALIGNMENT_REVISION,
137
+ )
138
+
139
+
140
+ def librispeech_ids(split: str = "test_clean", path: str | None = None) -> list[str]:
141
+ import pyarrow.parquet as pq
142
+
143
+ return sorted(pq.read_table(path or alignment_parquet(split), columns=["id"]).column("id").to_pylist())
144
+
145
+
146
+ def sample_ids(split: str, n: int, seed: int = 0, path: str | None = None) -> list[str]:
147
+ """A fixed random subset of utterance ids (sorted), so runs are comparable."""
148
+ ids = librispeech_ids(split, path)
149
+ rng = np.random.default_rng(seed)
150
+ return sorted(rng.choice(ids, size=min(n, len(ids)), replace=False).tolist())
151
+
152
+
153
+ def iter_librispeech(
154
+ split: str = "test_clean",
155
+ limit: int | None = None,
156
+ path: str | None = None,
157
+ ids: Sequence[str] | None = None,
158
+ ) -> Iterator[Utterance]:
159
+ """Utterances in id order (deterministic), audio decoded to float32 16 kHz. `ids` restricts to
160
+ those utterances. Empty-word entries (MFA's silence markers) are dropped."""
161
+ import pyarrow.parquet as pq
162
+ import soundfile as sf
163
+
164
+ table = pq.read_table(path or alignment_parquet(split), columns=["id", "audio", "transcript", "words"])
165
+ all_ids = np.asarray(table.column("id").to_pylist())
166
+ order = np.argsort(all_ids)
167
+ if ids is not None:
168
+ wanted = set(ids)
169
+ order = np.array([k for k in order if all_ids[k] in wanted], dtype=np.int64)
170
+ if limit is not None:
171
+ order = order[:limit]
172
+ for k in order:
173
+ row = table.slice(int(k), 1).to_pylist()[0]
174
+ audio, sr = sf.read(io.BytesIO(row["audio"]["bytes"]), dtype="float32")
175
+ if sr != SAMPLE_RATE:
176
+ raise ValueError(f"{row['id']}: expected 16 kHz, got {sr}")
177
+ words = [TimedWord(w["word"], w["start"], w["end"]) for w in row["words"] if w["word"].strip()]
178
+ yield Utterance(row["id"], audio, row["transcript"], words)
179
+
180
+
181
+ def normalize_text(s: str) -> str:
182
+ """Lowercase, drop punctuation except apostrophes, collapse spaces (LibriSpeech refs are already so)."""
183
+ s = re.sub(r"[^\w' ]+", " ", s.lower())
184
+ return " ".join(w.strip("'") for w in s.split() if w.strip("'"))
185
+
186
+
187
+ def english_normalizer() -> tuple[str, Callable[[str], str]]:
188
+ """Whisper's English normalizer (numbers, spellings, contractions) when the `whisper-normalizer`
189
+ package is installed, as on the Open ASR Leaderboard; otherwise `normalize_text`. Returns (name, fn)."""
190
+ try:
191
+ from whisper_normalizer.english import EnglishTextNormalizer
192
+ except ImportError:
193
+ return "basic", normalize_text
194
+ return "whisper-english", EnglishTextNormalizer()
195
+
196
+
197
+ def normalizer_for(language: str | None) -> tuple[str, Callable[[str], str]]:
198
+ """English: Whisper's English normalizer. Other languages: Whisper's basic normalizer (lowercase, drop
199
+ punctuation and bracketed text, keep diacritics), as the Open ASR Leaderboard's multilingual track."""
200
+ if language is None or language.lower() in ("english", "en"):
201
+ return english_normalizer()
202
+ try:
203
+ from whisper_normalizer.basic import BasicTextNormalizer
204
+ except ImportError:
205
+ return "basic", normalize_text
206
+ return "whisper-basic", BasicTextNormalizer()
207
+
208
+
209
+ # Extra evaluation sets: WER (and speed) only, no reference word timings.
210
+ ESB_REPO, ESB_REVISION = "hf-audio/esb-datasets-test-only-sorted", "de79e57adf9983993bcc72586d7313460825f5eb"
211
+ FLEURS_REPO, FLEURS_REVISION = "google/fleurs", "70bb2e84b976b7e960aa89f1c648e09c59f894dd"
212
+ EVAL_SETS = {
213
+ # name: (repo, revision, parquet prefix, id column, text column, language)
214
+ "ami": (ESB_REPO, ESB_REVISION, "ami/test-", "id", "text", "English"), # meetings, CC-BY-4.0
215
+ "earnings22": (ESB_REPO, ESB_REVISION, "earnings22/test-", "id", "text", "English"), # CC-BY-SA-4.0
216
+ "fleurs_nl": (
217
+ FLEURS_REPO,
218
+ FLEURS_REVISION,
219
+ "parquet-data/nl_nl/test-",
220
+ "id",
221
+ "raw_transcription",
222
+ "Dutch",
223
+ ),
224
+ }
225
+
226
+
227
+ def load_eval_set(name: str, n: int, seed: int = 0) -> tuple[list[Utterance], str]:
228
+ """A fixed random sample of `n` utterances (all if fewer) and the set's language. Utterances whose
229
+ reference is empty are skipped before sampling."""
230
+ import pyarrow.parquet as pq
231
+ import soundfile as sf
232
+ from huggingface_hub import HfApi, hf_hub_download
233
+
234
+ repo, rev, prefix, id_col, text_col, language = EVAL_SETS[name]
235
+ files = sorted(
236
+ f for f in HfApi().list_repo_files(repo, repo_type="dataset", revision=rev)
237
+ if f.startswith(prefix) and f.endswith(".parquet")
238
+ ) # fmt: skip
239
+ tables = [pq.read_table(hf_hub_download(repo, f, repo_type="dataset", revision=rev)) for f in files]
240
+ rows = [(t, i) for t in tables for i in range(t.num_rows) if t.column(text_col)[i].as_py().strip()]
241
+ rng = np.random.default_rng(seed)
242
+ pick = sorted(rng.choice(len(rows), size=min(n, len(rows)), replace=False).tolist())
243
+ utts = []
244
+ for k in pick:
245
+ t, i = rows[k]
246
+ audio, sr = sf.read(io.BytesIO(t.column("audio")[i].as_py()["bytes"]), dtype="float32")
247
+ if audio.ndim == 2:
248
+ audio = audio.mean(axis=1)
249
+ if sr != SAMPLE_RATE:
250
+ from rumblex.audio import load_audio
251
+
252
+ audio = load_audio(audio, sr=sr)
253
+ utts.append(Utterance(str(t.column(id_col)[i].as_py()), audio, t.column(text_col)[i].as_py(), []))
254
+ return utts, language
255
+
256
+
257
+ def wer(refs: Sequence[str], hyps: Sequence[str], norm: Callable[[str], str] = normalize_text) -> dict:
258
+ import jiwer
259
+
260
+ r, h = [norm(x) for x in refs], [norm(x) for x in hyps]
261
+ keep = [i for i, x in enumerate(r) if x.strip()] # jiwer rejects empty references
262
+ o = jiwer.process_words([r[i] for i in keep], [h[i] for i in keep])
263
+ return {
264
+ "wer": round(o.wer, 4),
265
+ "sub": o.substitutions,
266
+ "del": o.deletions,
267
+ "ins": o.insertions,
268
+ "ref_words": o.hits + o.substitutions + o.deletions,
269
+ }
270
+
271
+
272
+ @dataclass
273
+ class SystemOutput:
274
+ text: str
275
+ words: list[TimedWord]
276
+ timings: dict[str, float] = field(default_factory=dict) # seconds per stage
277
+
278
+
279
+ class System:
280
+ """One ASR + word-alignment system under test. `load` is timed separately from `run`."""
281
+
282
+ name = "system"
283
+
284
+ def load(self) -> None:
285
+ pass
286
+
287
+ def run(self, audio: np.ndarray, language: str | None) -> SystemOutput:
288
+ raise NotImplementedError
289
+
290
+ def info(self) -> dict:
291
+ return {}
292
+
293
+
294
+ class RumbleXSystem(System):
295
+ name = "rumblex"
296
+
297
+ def __init__(
298
+ self,
299
+ model: str = "Reson8/Rumble-2-Flash",
300
+ device: str | None = None,
301
+ compute_type: str | None = None,
302
+ batch_size: int = 8,
303
+ digits: bool = False,
304
+ max_chunk_s: float = 20.0,
305
+ ):
306
+ self.kw = {"model": model, "device": device, "compute_type": compute_type, "batch_size": batch_size}
307
+ self.digits = digits
308
+ self.max_chunk_s = max_chunk_s
309
+
310
+ def load(self) -> None:
311
+ from rumblex import load_model
312
+ from rumblex.aligner import Aligner
313
+
314
+ self.tr = load_model(**self.kw)
315
+ self.al = Aligner.from_transcriber(self.tr)
316
+
317
+ def run(self, audio: np.ndarray, language: str | None) -> SystemOutput:
318
+ t0 = _now()
319
+ res = self.tr.transcribe(
320
+ audio,
321
+ language=language,
322
+ digits=self.digits,
323
+ keep_encoder_states=True,
324
+ max_chunk_s=self.max_chunk_s,
325
+ )
326
+ t1 = _now()
327
+ self.al.align(res, audio)
328
+ t2 = _now()
329
+ words = [TimedWord(w.word, w.start, w.end) for s in res.segments for w in s.words]
330
+ text = " ".join(s.text for s in res.segments)
331
+ return SystemOutput(text, words, {"transcribe": t1 - t0, "align": t2 - t1})
332
+
333
+ def info(self) -> dict:
334
+ return {
335
+ **self.kw,
336
+ "dtype": str(self.tr.dtype),
337
+ "digits": self.digits,
338
+ "max_chunk_s": self.max_chunk_s,
339
+ }
340
+
341
+
342
+ class RumbleXVLLMSystem(RumbleXSystem):
343
+ """rumblex with decoding on a vLLM server (`rumblex.vllm_backend`); alignment as in `RumbleXSystem`."""
344
+
345
+ name = "rumblex-vllm"
346
+
347
+ def __init__(self, base_url: str = "http://localhost:8000/v1", concurrency: int = 32, **kw):
348
+ super().__init__(**kw)
349
+ self.base_url, self.concurrency = base_url, concurrency
350
+
351
+ def load(self) -> None:
352
+ from rumblex.aligner import Aligner, AlignOptions, CTCHead
353
+ from rumblex.vllm_backend import VLLMTranscriber
354
+
355
+ self.tr = VLLMTranscriber(
356
+ self.base_url,
357
+ self.kw["model"],
358
+ concurrency=self.concurrency,
359
+ batch_size=self.kw["batch_size"],
360
+ device=self.kw["device"],
361
+ dtype=self.kw["compute_type"],
362
+ )
363
+ head = CTCHead.from_pretrained(self.kw["model"], device=self.tr.device)
364
+ self.al = Aligner(head, self.tr.tokenizer, AlignOptions())
365
+
366
+ def info(self) -> dict:
367
+ return {
368
+ **super().info(),
369
+ "backend": "vllm",
370
+ "base_url": self.base_url,
371
+ "concurrency": self.concurrency,
372
+ }
373
+
374
+
375
+ class WhisperXSystem(System):
376
+ """WhisperX (faster-whisper + wav2vec2 alignment). Needs `pip install whisperx` (not a rumblex dependency)."""
377
+
378
+ name = "whisperx"
379
+
380
+ def __init__(
381
+ self,
382
+ model: str = "large-v3",
383
+ device: str = "cuda",
384
+ compute_type: str = "float16",
385
+ batch_size: int = 16,
386
+ ):
387
+ self.kw = {"model": model, "device": device, "compute_type": compute_type, "batch_size": batch_size}
388
+ self._align: dict = {}
389
+
390
+ def load(self) -> None:
391
+ import whisperx
392
+
393
+ self.wx = whisperx
394
+ self.model = whisperx.load_model(
395
+ self.kw["model"], self.kw["device"], compute_type=self.kw["compute_type"]
396
+ )
397
+
398
+ def run(self, audio: np.ndarray, language: str | None) -> SystemOutput:
399
+ lang = _iso(language)
400
+ t0 = _now()
401
+ res = self.model.transcribe(audio, batch_size=self.kw["batch_size"], language=lang)
402
+ t1 = _now()
403
+ code = res.get("language") or lang or "en"
404
+ if code not in self._align: # alignment model load is a one-off, kept out of the timing
405
+ self._align[code] = self.wx.load_align_model(language_code=code, device=self.kw["device"])
406
+ t2 = _now()
407
+ model_a, meta = self._align[code]
408
+ aligned = self.wx.align(
409
+ res["segments"], model_a, meta, audio, self.kw["device"], return_char_alignments=False
410
+ )
411
+ t3 = _now()
412
+ words = [
413
+ TimedWord(w["word"], float(w["start"]), float(w["end"]))
414
+ for s in aligned["segments"]
415
+ for w in s.get("words", [])
416
+ if "start" in w and "end" in w # WhisperX leaves some tokens (e.g. bare numbers) untimed
417
+ ]
418
+ text = " ".join(s["text"].strip() for s in res["segments"])
419
+ return SystemOutput(text, words, {"transcribe": t1 - t0, "align": t3 - t2})
420
+
421
+ def info(self) -> dict:
422
+ import importlib.metadata as md
423
+
424
+ return {**self.kw, "whisperx": md.version("whisperx")}
425
+
426
+
427
+ _ISO = {"english": "en", "dutch": "nl", "german": "de", "french": "fr", "spanish": "es"}
428
+
429
+
430
+ def _iso(language: str | None) -> str | None:
431
+ if language is None:
432
+ return None
433
+ return _ISO.get(language.lower(), language.lower())
434
+
435
+
436
+ def _now() -> float:
437
+ import time
438
+
439
+ try:
440
+ import torch
441
+
442
+ if torch.cuda.is_available():
443
+ torch.cuda.synchronize()
444
+ except ImportError:
445
+ pass
446
+ return time.perf_counter()
447
+
448
+
449
+ class GpuMemorySampler:
450
+ """Peak memory used on the GPU while running, sampled from nvidia-smi. It counts memory torch doesn't
451
+ see (e.g. CTranslate2's), so both systems are measured the same way. None without nvidia-smi."""
452
+
453
+ def __init__(self, interval: float = 0.2, index: int = 0):
454
+ self.interval, self.index, self.peak_mb = interval, index, None
455
+
456
+ def _read(self) -> int | None:
457
+ import subprocess
458
+
459
+ cmd = ["nvidia-smi", f"--id={self.index}", "--query-gpu=memory.used", "--format=csv,noheader,nounits"]
460
+ try:
461
+ out = subprocess.run(cmd, capture_output=True, text=True, timeout=5, check=True).stdout
462
+ return int(out.strip().splitlines()[0])
463
+ except Exception: # noqa: BLE001 - no GPU / no nvidia-smi: just no reading
464
+ return None
465
+
466
+ def __enter__(self):
467
+ import threading
468
+
469
+ self._stop = threading.Event()
470
+
471
+ def loop():
472
+ while not self._stop.is_set():
473
+ v = self._read()
474
+ if v is not None:
475
+ self.peak_mb = max(self.peak_mb or 0, v)
476
+ self._stop.wait(self.interval)
477
+
478
+ self._t = threading.Thread(target=loop, daemon=True)
479
+ self._t.start()
480
+ return self
481
+
482
+ def __exit__(self, *exc):
483
+ self._stop.set()
484
+ self._t.join()
485
+
486
+
487
+ def run_accuracy(
488
+ system: System,
489
+ utts: Sequence[Utterance],
490
+ language: str | None,
491
+ norm: Callable[[str], str] = normalize_text,
492
+ ) -> dict:
493
+ """One run per utterance: WER over all of them, and word timing errors against the references."""
494
+ hyps, score, secs = [], TimingScore(), 0.0
495
+ for u in utts:
496
+ out = system.run(u.audio, language)
497
+ secs += sum(out.timings.values())
498
+ hyps.append(out.text)
499
+ score.add(score_timings(u.words, out.words))
500
+ audio_s = sum(len(u.audio) for u in utts) / SAMPLE_RATE
501
+ return {
502
+ "utterances": len(utts),
503
+ "audio_seconds": round(audio_s, 1),
504
+ **wer([u.transcript for u in utts], hyps, norm),
505
+ "timing": score.summary(),
506
+ "rtfx_per_utterance": round(audio_s / secs, 1) if secs else None,
507
+ }
508
+
509
+
510
+ def run_speed(system: System, audio: np.ndarray, language: str | None) -> dict:
511
+ """One long recording end to end: stage times, real-time factor, peak GPU memory."""
512
+ with GpuMemorySampler() as mem:
513
+ t0 = _now()
514
+ out = system.run(audio, language)
515
+ total = _now() - t0
516
+ dur = len(audio) / SAMPLE_RATE
517
+ return {
518
+ "audio_seconds": round(dur, 1),
519
+ "total_seconds": round(total, 2),
520
+ "rtfx": round(dur / total, 1) if total else None,
521
+ "stages": {k: round(v, 2) for k, v in out.timings.items()},
522
+ "words": len(out.words),
523
+ "peak_gpu_mb": mem.peak_mb,
524
+ }
525
+
526
+
527
+ def long_audio_from_librispeech(utts: Sequence[Utterance], seconds: float, gap: float = 0.5) -> np.ndarray:
528
+ """Real speech for speed runs: utterances back to back with short gaps, cycled up to `seconds`."""
529
+ if not utts:
530
+ return np.zeros(0, np.float32)
531
+ target = int(seconds * SAMPLE_RATE)
532
+ g = np.zeros(int(gap * SAMPLE_RATE), np.float32)
533
+ pieces, total = [], 0
534
+ while total < target:
535
+ for u in utts:
536
+ pieces += [u.audio, g]
537
+ total += len(u.audio) + len(g)
538
+ if total >= target:
539
+ break
540
+ return np.concatenate(pieces)[:target]
541
+
542
+
543
+ def _pct(x) -> str:
544
+ return f"{100 * x:.1f}%" if isinstance(x, (int, float)) else "n/a"
545
+
546
+
547
+ def _unit(x, unit: str) -> str:
548
+ return "n/a" if x is None else f"{x}{unit}"
549
+
550
+
551
+ def markdown_table(results: dict) -> str:
552
+ rows = [
553
+ "| system | WER | start MAE | starts ≤ 100 ms | end MAE | ends ≤ 100 ms | long file speed | peak GPU |",
554
+ "|---|---|---|---|---|---|---|---|",
555
+ ]
556
+ for name, r in results.items():
557
+ a, s = r.get("accuracy", {}), r.get("speed", {})
558
+ t = a.get("timing", {})
559
+ rows.append(
560
+ f"| {name} | {_pct(a.get('wer'))} | {_unit(t.get('start_mae_ms'), ' ms')}"
561
+ f" | {_pct(t.get('start_within_100ms'))} | {_unit(t.get('end_mae_ms'), ' ms')}"
562
+ f" | {_pct(t.get('end_within_100ms'))} | {_unit(s.get('rtfx'), 'x')} | {_unit(s.get('peak_gpu_mb'), ' MB')} |"
563
+ )
564
+ return "\n".join(rows)