rumblex 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
rumblex/__init__.py ADDED
@@ -0,0 +1,72 @@
1
+ """rumblex: transcription, word timestamps, diarization and subtitles on Reson8/Rumble-2-Flash.
2
+
3
+ import rumblex
4
+ model = rumblex.load_model() # Reson8/Rumble-2-Flash, GPU if available
5
+ result = rumblex.transcribe("meeting.mp4", model) # segments + word timestamps
6
+ rumblex.write_result(result, "meeting.mp4", "out/", ["srt", "json"])
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from typing import TYPE_CHECKING
12
+
13
+ from rumblex.types import Segment, TranscriptResult, WordSegment
14
+ from rumblex.writers import write_result
15
+
16
+ if TYPE_CHECKING:
17
+ import torch
18
+
19
+ from rumblex.audio import AudioInput
20
+ from rumblex.transcriber import RumbleTranscriber
21
+
22
+ __version__ = "0.1.0"
23
+ __all__ = ["Segment", "TranscriptResult", "WordSegment", "load_model", "transcribe", "write_result"]
24
+
25
+
26
+ def load_model(
27
+ model: str = "Reson8/Rumble-2-Flash",
28
+ device: str | None = None,
29
+ compute_type: str | torch.dtype | None = None,
30
+ batch_size: int = 8,
31
+ ) -> RumbleTranscriber:
32
+ """Load the model (a Hugging Face repo id or a local folder). `device` defaults to cuda if available.
33
+ `compute_type`: float32 / float16 / bfloat16 (or fp32 / fp16 / bf16); default bfloat16 on Ampere and
34
+ newer GPUs, float32 otherwise. `batch_size` is the number of chunks decoded together."""
35
+ from rumblex.transcriber import RumbleTranscriber, resolve_dtype
36
+
37
+ dtype = resolve_dtype(compute_type)
38
+ return RumbleTranscriber(model, device=device, dtype=dtype, batch_size=batch_size)
39
+
40
+
41
+ def transcribe(
42
+ audio: AudioInput,
43
+ model: RumbleTranscriber | None = None,
44
+ *,
45
+ align: bool = True,
46
+ sr: int | None = None,
47
+ **kwargs,
48
+ ) -> TranscriptResult:
49
+ """Transcribe a file, or an array (16 kHz unless `sr` says otherwise). kwargs: language, prompt, digits,
50
+ filler, verbatim, max_chunk_s, chunks (precomputed SpeechChunks instead of VAD) and progress
51
+ (called with (done, total) chunks). With `align`, words get CTC timestamps (word ends refined on the
52
+ audio) and segments are cut at sentence ends.
53
+
54
+ Without `model`, a new model is loaded on every call; load it once with `load_model()` for more files."""
55
+ from rumblex.audio import load_audio
56
+
57
+ model = model or load_model()
58
+ wav = load_audio(audio, sr=sr) # decoded once, reused below
59
+ result = model.transcribe(wav, keep_encoder_states=align, **kwargs)
60
+ if align:
61
+ _aligner_for(model).align(result, wav)
62
+ result.metadata.pop("chunk_transcripts", None)
63
+ return result
64
+
65
+
66
+ def _aligner_for(model: RumbleTranscriber):
67
+ """One Aligner per model, kept on the model, so repeated calls don't reload the CTC head (357 MB)."""
68
+ from rumblex.aligner import Aligner
69
+
70
+ if getattr(model, "_rumblex_aligner", None) is None:
71
+ model._rumblex_aligner = Aligner.from_transcriber(model)
72
+ return model._rumblex_aligner
rumblex/__main__.py ADDED
@@ -0,0 +1,5 @@
1
+ import sys
2
+
3
+ from rumblex.cli import main
4
+
5
+ sys.exit(main())
rumblex/aligner.py ADDED
@@ -0,0 +1,536 @@
1
+ """Word timestamps from Rumble-2-Flash's auxiliary CTC head.
2
+
3
+ For each chunk, the encoder states captured during transcription go through the CTC head
4
+ (`ctc/ctc_head.safetensors`, one Linear 896 -> 99,631; class 99,630 is blank, the rest map 1:1 onto
5
+ the decoder tokenizer). The decoder's own text tokens are force-aligned to those log-probs with CTC
6
+ Viterbi, and tokens are grouped into words at the leading-space marker.
7
+
8
+ The head is very peaky (a token is a ~1-frame spike), so the alignment gives good onsets but no word
9
+ ends. A word ends where the next one starts (`END_RULES["next_start"]`), pulled back to the first
10
+ silence in the audio when the audio is available (`refine_ends`). Both were chosen on LibriSpeech
11
+ dev_clean and reported on test_clean in docs/alignment_eval.md.
12
+ If alignment fails for a chunk, words are spread over the chunk's VAD speech regions by length.
13
+
14
+ Words are split at spaces, so languages written without spaces (Chinese, Japanese) get one "word"
15
+ per chunk.
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ import json
21
+ import re
22
+ from collections.abc import Callable, Sequence
23
+ from dataclasses import dataclass
24
+ from itertools import pairwise
25
+ from pathlib import Path
26
+
27
+ import numpy as np
28
+ import torch
29
+
30
+ from rumblex.audio import SAMPLE_RATE
31
+ from rumblex.types import Segment, TranscriptResult, WordSegment
32
+
33
+ BLANK = 99_630
34
+ NUM_CLASSES = 99_631
35
+
36
+ # Encoder geometry (docs/model_inspection.md): 100 mel frames (1 s) per conv block -> 13 frames.
37
+ # Three stride-2 convs put frame k of a block at mel frame 8k, i.e. 80 ms apart, block-aligned.
38
+ BLOCK_SECONDS = 1.0
39
+ FRAMES_PER_BLOCK = 13
40
+ FRAME_STEP = 0.08
41
+
42
+
43
+ def frame_times(n: int, grid: str = "block") -> np.ndarray:
44
+ """Time in seconds (chunk-local) of each encoder frame's centre.
45
+ `block`: c + 0.08 k for frame k of 1 s block c (the conv geometry). `uniform`: i / 13."""
46
+ i = np.arange(n)
47
+ if grid == "block":
48
+ return (i // FRAMES_PER_BLOCK) * BLOCK_SECONDS + (i % FRAMES_PER_BLOCK) * FRAME_STEP
49
+ if grid == "uniform":
50
+ return i / FRAMES_PER_BLOCK
51
+ raise ValueError(f"unknown frame grid {grid!r}")
52
+
53
+
54
+ def _model_file(model: str, filename: str) -> str:
55
+ local = Path(model) / filename
56
+ if local.is_file():
57
+ return str(local)
58
+ from huggingface_hub import hf_hub_download
59
+
60
+ return hf_hub_download(model, filename)
61
+
62
+
63
+ class CTCHead:
64
+ """The checkpoint's CTC head, loaded from its safetensors file (no remote code)."""
65
+
66
+ def __init__(self, weight: torch.Tensor, bias: torch.Tensor, device: str | torch.device = "cpu"):
67
+ if weight.shape[0] != NUM_CLASSES:
68
+ raise ValueError(f"CTC head has {weight.shape[0]} classes, expected {NUM_CLASSES}")
69
+ self.device = torch.device(device)
70
+ self.weight = weight.to(self.device, torch.float32)
71
+ self.bias = bias.to(self.device, torch.float32)
72
+
73
+ @classmethod
74
+ def from_pretrained(
75
+ cls, model: str = "Reson8/Rumble-2-Flash", device: str | torch.device = "cpu"
76
+ ) -> CTCHead:
77
+ """`model` is a Hugging Face repo id or a local copy of the checkpoint (a folder with `ctc/`)."""
78
+ from safetensors.torch import load_file
79
+
80
+ vocab = json.loads(Path(_model_file(model, "ctc/ctc_vocab.json")).read_text(encoding="utf-8"))
81
+ if vocab["blank"] != BLANK or vocab["token_ids"] != list(range(BLANK)):
82
+ raise ValueError("ctc_vocab.json is not the identity mapping this aligner assumes")
83
+ sd = load_file(_model_file(model, "ctc/ctc_head.safetensors"))
84
+ return cls(sd["weight"], sd["bias"], device)
85
+
86
+ @torch.inference_mode()
87
+ def emissions(self, states: torch.Tensor, classes: Sequence[int]) -> np.ndarray:
88
+ """Log-probs [frames, len(classes)] of the given classes, normalised over all 99,631."""
89
+ h = states.to(self.device, torch.float32)
90
+ idx = torch.as_tensor(list(classes), device=self.device)
91
+ out = []
92
+ for i in range(0, h.shape[0], 512): # 512 x 99,631 floats = 200 MB at most
93
+ logits = torch.nn.functional.linear(h[i : i + 512], self.weight, self.bias)
94
+ out.append(logits[:, idx] - torch.logsumexp(logits, dim=-1, keepdim=True))
95
+ return torch.cat(out).cpu().numpy() if out else np.zeros((0, len(idx)), np.float32)
96
+
97
+
98
+ def ctc_viterbi(em: np.ndarray, n_tokens: int, same_as_prev: Sequence[bool]) -> np.ndarray | None:
99
+ """CTC forced alignment, vectorised over states.
100
+
101
+ `em`: [T, 1 + n_tokens] log-probs, column 0 = blank, column k+1 = target token k.
102
+ `same_as_prev[k]`: token k equals token k-1 (then a blank must separate them).
103
+ Returns the state path [T] over the 2n+1 extended states (even = blank, 2k+1 = token k),
104
+ or None if the tokens can't fit in T frames.
105
+ """
106
+ T, L = em.shape[0], n_tokens
107
+ S = 2 * L + 1
108
+ if T == 0 or L + sum(same_as_prev) > T:
109
+ return None
110
+ cols = np.zeros(S, dtype=np.int64)
111
+ cols[1::2] = np.arange(1, L + 1)
112
+ e = em[:, cols] # [T, S]
113
+ skip_ok = np.zeros(S, dtype=bool) # may jump s-2 -> s: token states whose token differs from the previous
114
+ skip_ok[3::2] = ~np.asarray(same_as_prev[1:], dtype=bool) if L > 1 else []
115
+ neg = -np.inf
116
+ dp = np.full(S, neg)
117
+ dp[0] = e[0, 0]
118
+ if S > 1:
119
+ dp[1] = e[0, 1]
120
+ back = np.zeros((T, S), dtype=np.int8)
121
+ for t in range(1, T):
122
+ stay = dp
123
+ step = np.concatenate(([neg], dp[:-1]))
124
+ skip = np.where(skip_ok, np.concatenate(([neg, neg], dp[:-2])), neg)
125
+ cand = np.stack([stay, step, skip])
126
+ arg = cand.argmax(0)
127
+ dp = cand[arg, np.arange(S)] + e[t]
128
+ back[t] = arg
129
+ s = S - 1 if S == 1 or dp[S - 1] >= dp[S - 2] else S - 2
130
+ if not np.isfinite(dp[s]):
131
+ return None
132
+ path = np.empty(T, dtype=np.int64)
133
+ for t in range(T - 1, -1, -1):
134
+ path[t] = s
135
+ s -= int(back[t, s]) # int(): numpy 2 would make s an int8 and overflow past 127 states
136
+ return path
137
+
138
+
139
+ def token_spans(path: np.ndarray, n_tokens: int) -> list[tuple[int, int]]:
140
+ """(first, last) frame of each token on a Viterbi path."""
141
+ spans = [[-1, -1] for _ in range(n_tokens)]
142
+ for t, s in enumerate(path):
143
+ if s % 2:
144
+ k = s // 2
145
+ if spans[k][0] < 0:
146
+ spans[k][0] = t
147
+ spans[k][1] = t
148
+ return [(a, b) for a, b in spans]
149
+
150
+
151
+ _HAS_ALNUM = re.compile(r"\w")
152
+
153
+
154
+ @dataclass
155
+ class AlignedWord:
156
+ text: str
157
+ first: int # first frame of its first token
158
+ last: int # last frame of its last token
159
+ score: float
160
+
161
+
162
+ def group_words(
163
+ token_ids: Sequence[int],
164
+ spans: Sequence[tuple[int, int]],
165
+ token_probs: Sequence[float],
166
+ tokens: Sequence[str],
167
+ decode: Callable[[list[int]], str],
168
+ ) -> list[AlignedWord]:
169
+ """Group BPE tokens into words at the leading-space marker (`Ġ`). Punctuation-only groups
170
+ (e.g. a lone ` -`) are attached to the previous word, or before the first word (an opening quote)."""
171
+ groups: list[list[int]] = []
172
+ for k, tok in enumerate(tokens):
173
+ if not groups or tok.startswith("Ġ"):
174
+ groups.append([k])
175
+ else:
176
+ groups[-1].append(k)
177
+ words: list[AlignedWord] = []
178
+ lead: list[int] = [] # punctuation-only groups before the first word
179
+ for g in groups:
180
+ text = decode([token_ids[k] for k in g]).strip()
181
+ if not text:
182
+ continue
183
+ score = float(np.mean([token_probs[k] for k in g]))
184
+ if not _HAS_ALNUM.search(text):
185
+ if words:
186
+ words[-1].text += text
187
+ else:
188
+ lead += g
189
+ continue
190
+ if lead and not words:
191
+ text = decode([token_ids[k] for k in lead]).strip() + text
192
+ words.append(AlignedWord(text, spans[g[0]][0], spans[g[-1]][1], score))
193
+ if lead and not words: # nothing but punctuation: keep it as one word
194
+ text = decode([token_ids[k] for k in lead]).strip()
195
+ score = float(np.mean([token_probs[k] for k in lead]))
196
+ words.append(AlignedWord(text, spans[lead[0]][0], spans[lead[-1]][1], score))
197
+ return words
198
+
199
+
200
+ # Each rule maps word starts and spike ends (seconds, [n]) plus the chunk's speech regions to word ends.
201
+
202
+
203
+ def _end_spike(starts, spike_end, regions):
204
+ return spike_end.copy()
205
+
206
+
207
+ def _end_next_start(starts, spike_end, regions):
208
+ ends = spike_end.copy()
209
+ ends[:-1] = np.maximum(ends[:-1], starts[1:])
210
+ return ends
211
+
212
+
213
+ def _end_midpoint(starts, spike_end, regions):
214
+ ends = spike_end.copy()
215
+ ends[:-1] = np.maximum(ends[:-1], (ends[:-1] + starts[1:]) / 2)
216
+ return ends
217
+
218
+
219
+ def _end_region(starts, spike_end, regions):
220
+ """Run on to the next word's start, but not past the end of the VAD speech region the word is in,
221
+ so a word before a pause doesn't swallow the pause. The last word runs to its region's end."""
222
+ nxt = np.append(starts[1:], np.inf)
223
+ region_end = np.full_like(spike_end, np.inf)
224
+ for r0, r1 in regions:
225
+ inside = (spike_end > r0) & (spike_end <= r1 + 1e-6)
226
+ region_end[inside] = r1
227
+ cap = np.minimum(nxt, region_end)
228
+ cap[~np.isfinite(cap)] = spike_end[~np.isfinite(cap)]
229
+ return np.maximum(spike_end, cap)
230
+
231
+
232
+ END_RULES: dict[str, Callable] = {
233
+ "spike": _end_spike,
234
+ "next_start": _end_next_start,
235
+ "midpoint": _end_midpoint,
236
+ "region": _end_region,
237
+ }
238
+
239
+
240
+ # Chosen on LibriSpeech dev_clean (docs/alignment_eval.md): 25 dB below the speech level, for 30 ms.
241
+ SILENCE_DB = 25.0
242
+ MIN_SILENCE = 0.03
243
+
244
+
245
+ @dataclass
246
+ class LoudnessRefinement:
247
+ """Audio for refining word times on loudness. `audio` covers the chunk, starting at global time
248
+ `audio_start`. Moving ends to the first silence helped on dev_clean; moving starts to sound onsets
249
+ did not, so `starts` is off by default."""
250
+
251
+ audio: np.ndarray
252
+ audio_start: float
253
+ silence_db: float = SILENCE_DB
254
+ min_silence: float = MIN_SILENCE
255
+ starts: bool = False
256
+ ends: bool = True
257
+
258
+
259
+ def word_times(
260
+ words: Sequence[AlignedWord],
261
+ n_frames: int,
262
+ chunk_start: float,
263
+ regions: Sequence[tuple[float, float]],
264
+ grid: str = "block",
265
+ end_rule: str = "next_start",
266
+ start_shift: float = 0.0,
267
+ loudness: LoudnessRefinement | None = None,
268
+ ) -> tuple[np.ndarray, np.ndarray]:
269
+ """Start/end seconds (global) for aligned words. A frame is FRAME_STEP wide around its centre.
270
+ Words never overlap: an end is clamped to the next word's start."""
271
+ times = frame_times(n_frames, grid) + chunk_start
272
+ spike_start = np.array([times[w.first] for w in words])
273
+ starts = np.maximum(spike_start - FRAME_STEP / 2 + start_shift, chunk_start)
274
+ spike_end = np.array([times[w.last] for w in words]) + FRAME_STEP / 2
275
+ lr = loudness
276
+ if lr is not None and lr.starts:
277
+ starts = refine_starts(
278
+ starts, spike_start, spike_end, lr.audio, lr.audio_start, lr.silence_db, lr.min_silence
279
+ )
280
+ ends = END_RULES[end_rule](starts, spike_end, list(regions))
281
+ ends[:-1] = np.minimum(ends[:-1], starts[1:])
282
+ if lr is not None and lr.ends:
283
+ ends = refine_ends(starts, spike_end, ends, lr.audio, lr.audio_start, lr.silence_db, lr.min_silence)
284
+ ends = np.maximum(ends, starts + 0.01)
285
+ return starts, ends
286
+
287
+
288
+ ENERGY_HOP = 0.01 # seconds per loudness frame
289
+
290
+
291
+ def frame_db(audio: np.ndarray, sr: int = SAMPLE_RATE, hop: float = ENERGY_HOP) -> np.ndarray:
292
+ """Loudness (dB of RMS) per non-overlapping `hop` frame."""
293
+ n = round(hop * sr)
294
+ k = len(audio) // n
295
+ if k == 0:
296
+ return np.zeros(0)
297
+ f = audio[: k * n].astype(np.float64).reshape(k, n)
298
+ return 10 * np.log10((f**2).mean(axis=1) + 1e-12)
299
+
300
+
301
+ def _silent_frames(audio: np.ndarray, silence_db: float) -> np.ndarray:
302
+ """Per 10 ms frame: is it `silence_db` below the chunk's speech level (95th percentile loudness)?"""
303
+ db = frame_db(audio)
304
+ return db < np.percentile(db, 95) - silence_db if db.size else np.zeros(0, bool)
305
+
306
+
307
+ def refine_ends(
308
+ starts: np.ndarray,
309
+ spike_end: np.ndarray,
310
+ ends: np.ndarray,
311
+ audio: np.ndarray,
312
+ audio_start: float,
313
+ silence_db: float = SILENCE_DB,
314
+ min_silence: float = MIN_SILENCE,
315
+ ) -> np.ndarray:
316
+ """Pull word ends back to where the audio goes silent.
317
+
318
+ `audio` covers the chunk and starts at global time `audio_start`. Between a word's last CTC spike and
319
+ its current end (usually the next word's start), the first run of at least `min_silence` seconds whose
320
+ loudness is `silence_db` below the chunk's speech level (95th percentile) ends the word. Without
321
+ such a run the end is kept, so this only ever moves ends earlier.
322
+ """
323
+ silent = _silent_frames(audio, silence_db)
324
+ if silent.size == 0:
325
+ return ends
326
+ need = max(1, round(min_silence / ENERGY_HOP))
327
+ # run[i] = number of consecutive silent frames starting at frame i
328
+ run = np.zeros(len(silent) + 1, dtype=np.int64)
329
+ for i in range(len(silent) - 1, -1, -1):
330
+ run[i] = run[i + 1] + 1 if silent[i] else 0
331
+ out = ends.copy()
332
+ for i in range(len(ends)):
333
+ a = max(0, int(np.floor((spike_end[i] - audio_start) / ENERGY_HOP)))
334
+ b = min(len(silent), int(np.ceil((ends[i] - audio_start) / ENERGY_HOP)))
335
+ if b - a < need:
336
+ continue
337
+ hits = np.nonzero(run[a:b] >= need)[0]
338
+ if hits.size:
339
+ out[i] = max(audio_start + (a + hits[0]) * ENERGY_HOP, starts[i] + 0.01)
340
+ return out
341
+
342
+
343
+ def refine_starts(
344
+ starts: np.ndarray,
345
+ spike_start: np.ndarray,
346
+ spike_end: np.ndarray,
347
+ audio: np.ndarray,
348
+ audio_start: float,
349
+ silence_db: float = SILENCE_DB,
350
+ min_silence: float = MIN_SILENCE,
351
+ ) -> np.ndarray:
352
+ """Move a word's start to where sound resumes after a silence, if there is one.
353
+
354
+ Searches between the previous word's last spike (or the chunk start) and the word's first spike for
355
+ the last run of at least `min_silence` seconds of silence (same definition as `refine_ends`); the
356
+ word then starts where that run ends. Words with no such silence before them are left alone.
357
+ """
358
+ silent = _silent_frames(audio, silence_db)
359
+ if silent.size == 0:
360
+ return starts
361
+ need = max(1, round(min_silence / ENERGY_HOP))
362
+ back = np.zeros(len(silent), dtype=np.int64) # back[i] = consecutive silent frames ending at frame i
363
+ for i in range(len(silent)):
364
+ back[i] = (back[i - 1] + 1 if i else 1) if silent[i] else 0
365
+ out = starts.copy()
366
+ for i in range(len(starts)):
367
+ lo = spike_end[i - 1] if i else audio_start
368
+ a = max(0, int(np.floor((lo - audio_start) / ENERGY_HOP)))
369
+ b = min(len(silent), int(np.ceil((spike_start[i] - audio_start) / ENERGY_HOP)))
370
+ hits = np.nonzero(back[a:b] >= need)[0]
371
+ if hits.size:
372
+ onset = audio_start + (a + hits[-1] + 1) * ENERGY_HOP
373
+ out[i] = min(onset, spike_start[i])
374
+ return out
375
+
376
+
377
+ @dataclass
378
+ class AlignOptions:
379
+ end_rule: str = "next_start" # chosen on dev_clean, see docs/alignment_eval.md
380
+ grid: str = "block"
381
+ start_shift: float = -0.022 # added to every start; median start error on dev_clean (block grid)
382
+ split_sentences: bool = True
383
+ # Loudness refinement, used when the audio is passed to align() (see LoudnessRefinement).
384
+ refine_ends: bool = True
385
+ refine_starts: bool = False
386
+ silence_db: float = SILENCE_DB
387
+ min_silence: float = MIN_SILENCE
388
+
389
+
390
+ class Aligner:
391
+ def __init__(self, head: CTCHead, tokenizer, opts: AlignOptions | None = None):
392
+ self.head = head
393
+ self.tokenizer = tokenizer
394
+ self.opts = opts or AlignOptions()
395
+ if self.opts.end_rule not in END_RULES:
396
+ raise ValueError(f"end_rule must be one of {list(END_RULES)}")
397
+
398
+ @classmethod
399
+ def from_transcriber(
400
+ cls, transcriber, opts: AlignOptions | None = None, model: str | None = None
401
+ ) -> Aligner:
402
+ head = CTCHead.from_pretrained(model or transcriber.model.name_or_path, device=transcriber.device)
403
+ return cls(head, transcriber.tokenizer, opts)
404
+
405
+ def align_tokens(
406
+ self,
407
+ token_ids: Sequence[int],
408
+ states: torch.Tensor,
409
+ chunk_start: float,
410
+ regions: Sequence[tuple[float, float]] = (),
411
+ audio: np.ndarray | None = None,
412
+ ) -> list[WordSegment] | None:
413
+ """Word timings (global seconds) for one chunk's tokens, or None if CTC alignment fails.
414
+ `regions` are the chunk's speech regions in global seconds."""
415
+ token_ids = list(token_ids)
416
+ if not token_ids or states is None:
417
+ return None
418
+ em = self.head.emissions(states, [BLANK, *token_ids])
419
+ same = [False] + [a == b for a, b in pairwise(token_ids)]
420
+ path = ctc_viterbi(em, len(token_ids), same)
421
+ if path is None:
422
+ return None
423
+ spans = token_spans(path, len(token_ids))
424
+ probs = [float(np.exp(em[a : b + 1, k + 1]).mean()) for k, (a, b) in enumerate(spans)]
425
+ tokens = self.tokenizer.convert_ids_to_tokens(token_ids)
426
+ words = group_words(token_ids, spans, probs, tokens, self._decode)
427
+ if not words:
428
+ return None
429
+ o = self.opts
430
+ loudness = None
431
+ if audio is not None and (o.refine_ends or o.refine_starts):
432
+ loudness = LoudnessRefinement(
433
+ audio, chunk_start, o.silence_db, o.min_silence, starts=o.refine_starts, ends=o.refine_ends
434
+ )
435
+ starts, ends = word_times(
436
+ words, len(em), chunk_start, regions, o.grid, o.end_rule, o.start_shift, loudness
437
+ )
438
+ return [
439
+ WordSegment(w.text, round(float(s), 3), round(float(e), 3), round(w.score, 3))
440
+ for w, s, e in zip(words, starts, ends)
441
+ ]
442
+
443
+ def align(self, result: TranscriptResult, audio: np.ndarray | None = None) -> TranscriptResult:
444
+ """Fill in words for a result from `RumbleTranscriber.transcribe` (needs keep_encoder_states=True).
445
+ Pass the 16 kHz `audio` that was transcribed to enable the loudness refinement of word ends.
446
+ Segments are re-cut at sentence ends when `split_sentences` is on."""
447
+ items = result.metadata.get("chunk_transcripts")
448
+ if items is None:
449
+ raise ValueError("result has no chunk_transcripts; transcribe with keep_encoder_states=True")
450
+ if any(it.text and it.encoder_states is None for it in items):
451
+ raise ValueError(
452
+ "chunk_transcripts have no encoder states; transcribe with keep_encoder_states=True"
453
+ )
454
+ segments: list[Segment] = []
455
+ n_fallback = 0
456
+ for it in items:
457
+ if not it.text:
458
+ continue
459
+ regions = [(r.start / SAMPLE_RATE, r.end / SAMPLE_RATE) for r in it.chunk.regions] or [
460
+ (it.chunk.start_s, it.chunk.end_s)
461
+ ]
462
+ chunk_audio = None if audio is None else it.chunk.slice(audio)
463
+ words = self.align_tokens(it.token_ids, it.encoder_states, it.chunk.start_s, regions, chunk_audio)
464
+ if words is None:
465
+ n_fallback += 1
466
+ words = proportional_words(it.text, regions)
467
+ clamp_to_chunk(words, it.chunk.start_s, it.chunk.end_s)
468
+ seg = Segment(words[0].start, words[-1].end, it.text, words, language=it.language)
469
+ segments.extend(split_sentences(seg) if self.opts.split_sentences else [seg])
470
+ result.segments = segments
471
+ result.metadata["alignment"] = {
472
+ **vars(self.opts),
473
+ "audio_refinement": audio is not None,
474
+ "fallback_chunks": n_fallback,
475
+ }
476
+ return result
477
+
478
+ def _decode(self, ids: list[int]) -> str:
479
+ return self.tokenizer.decode(ids, clean_up_tokenization_spaces=False)
480
+
481
+
482
+ def clamp_to_chunk(words: list[WordSegment], start: float, end: float) -> None:
483
+ """Keep word times inside the chunk's real audio. The silence appended before encoding
484
+ (transcriber.END_PAD) can otherwise put the last word's end a few ms past the end."""
485
+ for w in words:
486
+ w.end = round(min(w.end, end), 3)
487
+ w.start = round(min(max(w.start, start), max(w.end - 0.01, start)), 3)
488
+
489
+
490
+ def proportional_words(text: str, regions: Sequence[tuple[float, float]]) -> list[WordSegment]:
491
+ """Spread words over the speech regions in proportion to their length (score 0: not aligned)."""
492
+ words = text.split()
493
+ if not words:
494
+ return []
495
+ total = sum(e - s for s, e in regions)
496
+ weights = np.array([len(w) + 1 for w in words], dtype=float)
497
+ cum = np.concatenate(([0.0], np.cumsum(weights) / weights.sum() * total))
498
+
499
+ def at(x: float, start: bool) -> float:
500
+ """Speech-time offset -> wall time. A start exactly on a region boundary goes to the next region."""
501
+ for s, e in regions:
502
+ if x < e - s or (not start and x <= e - s):
503
+ return s + x
504
+ x -= e - s
505
+ return regions[-1][1]
506
+
507
+ return [
508
+ WordSegment(w, round(float(at(cum[i], True)), 3), round(float(at(cum[i + 1], False)), 3), 0.0)
509
+ for i, w in enumerate(words)
510
+ ]
511
+
512
+
513
+ _SENTENCE_END = re.compile(r"[.!?。?!]['\")\]]*$")
514
+ # Words ending in "." that rarely end a sentence (English and Dutch titles and common abbreviations).
515
+ _ABBREVIATIONS = {
516
+ "mr.", "mrs.", "ms.", "dr.", "prof.", "st.", "jr.", "sr.", "vs.", "etc.", "e.g.", "i.e.", "no.",
517
+ "dhr.", "mevr.", "mw.", "bijv.", "o.a.", "z.g.a.n.", "nr.", "ca.",
518
+ } # fmt: skip
519
+
520
+
521
+ def split_sentences(seg: Segment) -> list[Segment]:
522
+ """Cut a segment after words that end a sentence."""
523
+ if not seg.words:
524
+ return [seg]
525
+ out, cur = [], []
526
+ for w in seg.words:
527
+ cur.append(w)
528
+ if _SENTENCE_END.search(w.word) and w.word.lower() not in _ABBREVIATIONS:
529
+ out.append(cur)
530
+ cur = []
531
+ if cur:
532
+ out.append(cur)
533
+ return [
534
+ Segment(ws[0].start, ws[-1].end, " ".join(w.word for w in ws), ws, seg.speaker, seg.language)
535
+ for ws in out
536
+ ]