rumblex 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rumblex/__init__.py +72 -0
- rumblex/__main__.py +5 -0
- rumblex/aligner.py +536 -0
- rumblex/audio.py +104 -0
- rumblex/bench.py +564 -0
- rumblex/cli.py +274 -0
- rumblex/diarizer.py +181 -0
- rumblex/py.typed +0 -0
- rumblex/transcriber.py +319 -0
- rumblex/types.py +46 -0
- rumblex/vad.py +261 -0
- rumblex/vllm_backend.py +227 -0
- rumblex/writers.py +281 -0
- rumblex-0.1.0.dist-info/METADATA +230 -0
- rumblex-0.1.0.dist-info/RECORD +18 -0
- rumblex-0.1.0.dist-info/WHEEL +4 -0
- rumblex-0.1.0.dist-info/entry_points.txt +2 -0
- rumblex-0.1.0.dist-info/licenses/LICENSE +202 -0
rumblex/__init__.py
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""rumblex: transcription, word timestamps, diarization and subtitles on Reson8/Rumble-2-Flash.
|
|
2
|
+
|
|
3
|
+
import rumblex
|
|
4
|
+
model = rumblex.load_model() # Reson8/Rumble-2-Flash, GPU if available
|
|
5
|
+
result = rumblex.transcribe("meeting.mp4", model) # segments + word timestamps
|
|
6
|
+
rumblex.write_result(result, "meeting.mp4", "out/", ["srt", "json"])
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from typing import TYPE_CHECKING
|
|
12
|
+
|
|
13
|
+
from rumblex.types import Segment, TranscriptResult, WordSegment
|
|
14
|
+
from rumblex.writers import write_result
|
|
15
|
+
|
|
16
|
+
if TYPE_CHECKING:
|
|
17
|
+
import torch
|
|
18
|
+
|
|
19
|
+
from rumblex.audio import AudioInput
|
|
20
|
+
from rumblex.transcriber import RumbleTranscriber
|
|
21
|
+
|
|
22
|
+
__version__ = "0.1.0"
|
|
23
|
+
__all__ = ["Segment", "TranscriptResult", "WordSegment", "load_model", "transcribe", "write_result"]
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def load_model(
|
|
27
|
+
model: str = "Reson8/Rumble-2-Flash",
|
|
28
|
+
device: str | None = None,
|
|
29
|
+
compute_type: str | torch.dtype | None = None,
|
|
30
|
+
batch_size: int = 8,
|
|
31
|
+
) -> RumbleTranscriber:
|
|
32
|
+
"""Load the model (a Hugging Face repo id or a local folder). `device` defaults to cuda if available.
|
|
33
|
+
`compute_type`: float32 / float16 / bfloat16 (or fp32 / fp16 / bf16); default bfloat16 on Ampere and
|
|
34
|
+
newer GPUs, float32 otherwise. `batch_size` is the number of chunks decoded together."""
|
|
35
|
+
from rumblex.transcriber import RumbleTranscriber, resolve_dtype
|
|
36
|
+
|
|
37
|
+
dtype = resolve_dtype(compute_type)
|
|
38
|
+
return RumbleTranscriber(model, device=device, dtype=dtype, batch_size=batch_size)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def transcribe(
|
|
42
|
+
audio: AudioInput,
|
|
43
|
+
model: RumbleTranscriber | None = None,
|
|
44
|
+
*,
|
|
45
|
+
align: bool = True,
|
|
46
|
+
sr: int | None = None,
|
|
47
|
+
**kwargs,
|
|
48
|
+
) -> TranscriptResult:
|
|
49
|
+
"""Transcribe a file, or an array (16 kHz unless `sr` says otherwise). kwargs: language, prompt, digits,
|
|
50
|
+
filler, verbatim, max_chunk_s, chunks (precomputed SpeechChunks instead of VAD) and progress
|
|
51
|
+
(called with (done, total) chunks). With `align`, words get CTC timestamps (word ends refined on the
|
|
52
|
+
audio) and segments are cut at sentence ends.
|
|
53
|
+
|
|
54
|
+
Without `model`, a new model is loaded on every call; load it once with `load_model()` for more files."""
|
|
55
|
+
from rumblex.audio import load_audio
|
|
56
|
+
|
|
57
|
+
model = model or load_model()
|
|
58
|
+
wav = load_audio(audio, sr=sr) # decoded once, reused below
|
|
59
|
+
result = model.transcribe(wav, keep_encoder_states=align, **kwargs)
|
|
60
|
+
if align:
|
|
61
|
+
_aligner_for(model).align(result, wav)
|
|
62
|
+
result.metadata.pop("chunk_transcripts", None)
|
|
63
|
+
return result
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _aligner_for(model: RumbleTranscriber):
|
|
67
|
+
"""One Aligner per model, kept on the model, so repeated calls don't reload the CTC head (357 MB)."""
|
|
68
|
+
from rumblex.aligner import Aligner
|
|
69
|
+
|
|
70
|
+
if getattr(model, "_rumblex_aligner", None) is None:
|
|
71
|
+
model._rumblex_aligner = Aligner.from_transcriber(model)
|
|
72
|
+
return model._rumblex_aligner
|
rumblex/__main__.py
ADDED
rumblex/aligner.py
ADDED
|
@@ -0,0 +1,536 @@
|
|
|
1
|
+
"""Word timestamps from Rumble-2-Flash's auxiliary CTC head.
|
|
2
|
+
|
|
3
|
+
For each chunk, the encoder states captured during transcription go through the CTC head
|
|
4
|
+
(`ctc/ctc_head.safetensors`, one Linear 896 -> 99,631; class 99,630 is blank, the rest map 1:1 onto
|
|
5
|
+
the decoder tokenizer). The decoder's own text tokens are force-aligned to those log-probs with CTC
|
|
6
|
+
Viterbi, and tokens are grouped into words at the leading-space marker.
|
|
7
|
+
|
|
8
|
+
The head is very peaky (a token is a ~1-frame spike), so the alignment gives good onsets but no word
|
|
9
|
+
ends. A word ends where the next one starts (`END_RULES["next_start"]`), pulled back to the first
|
|
10
|
+
silence in the audio when the audio is available (`refine_ends`). Both were chosen on LibriSpeech
|
|
11
|
+
dev_clean and reported on test_clean in docs/alignment_eval.md.
|
|
12
|
+
If alignment fails for a chunk, words are spread over the chunk's VAD speech regions by length.
|
|
13
|
+
|
|
14
|
+
Words are split at spaces, so languages written without spaces (Chinese, Japanese) get one "word"
|
|
15
|
+
per chunk.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import json
|
|
21
|
+
import re
|
|
22
|
+
from collections.abc import Callable, Sequence
|
|
23
|
+
from dataclasses import dataclass
|
|
24
|
+
from itertools import pairwise
|
|
25
|
+
from pathlib import Path
|
|
26
|
+
|
|
27
|
+
import numpy as np
|
|
28
|
+
import torch
|
|
29
|
+
|
|
30
|
+
from rumblex.audio import SAMPLE_RATE
|
|
31
|
+
from rumblex.types import Segment, TranscriptResult, WordSegment
|
|
32
|
+
|
|
33
|
+
BLANK = 99_630
|
|
34
|
+
NUM_CLASSES = 99_631
|
|
35
|
+
|
|
36
|
+
# Encoder geometry (docs/model_inspection.md): 100 mel frames (1 s) per conv block -> 13 frames.
|
|
37
|
+
# Three stride-2 convs put frame k of a block at mel frame 8k, i.e. 80 ms apart, block-aligned.
|
|
38
|
+
BLOCK_SECONDS = 1.0
|
|
39
|
+
FRAMES_PER_BLOCK = 13
|
|
40
|
+
FRAME_STEP = 0.08
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def frame_times(n: int, grid: str = "block") -> np.ndarray:
|
|
44
|
+
"""Time in seconds (chunk-local) of each encoder frame's centre.
|
|
45
|
+
`block`: c + 0.08 k for frame k of 1 s block c (the conv geometry). `uniform`: i / 13."""
|
|
46
|
+
i = np.arange(n)
|
|
47
|
+
if grid == "block":
|
|
48
|
+
return (i // FRAMES_PER_BLOCK) * BLOCK_SECONDS + (i % FRAMES_PER_BLOCK) * FRAME_STEP
|
|
49
|
+
if grid == "uniform":
|
|
50
|
+
return i / FRAMES_PER_BLOCK
|
|
51
|
+
raise ValueError(f"unknown frame grid {grid!r}")
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _model_file(model: str, filename: str) -> str:
|
|
55
|
+
local = Path(model) / filename
|
|
56
|
+
if local.is_file():
|
|
57
|
+
return str(local)
|
|
58
|
+
from huggingface_hub import hf_hub_download
|
|
59
|
+
|
|
60
|
+
return hf_hub_download(model, filename)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class CTCHead:
|
|
64
|
+
"""The checkpoint's CTC head, loaded from its safetensors file (no remote code)."""
|
|
65
|
+
|
|
66
|
+
def __init__(self, weight: torch.Tensor, bias: torch.Tensor, device: str | torch.device = "cpu"):
|
|
67
|
+
if weight.shape[0] != NUM_CLASSES:
|
|
68
|
+
raise ValueError(f"CTC head has {weight.shape[0]} classes, expected {NUM_CLASSES}")
|
|
69
|
+
self.device = torch.device(device)
|
|
70
|
+
self.weight = weight.to(self.device, torch.float32)
|
|
71
|
+
self.bias = bias.to(self.device, torch.float32)
|
|
72
|
+
|
|
73
|
+
@classmethod
|
|
74
|
+
def from_pretrained(
|
|
75
|
+
cls, model: str = "Reson8/Rumble-2-Flash", device: str | torch.device = "cpu"
|
|
76
|
+
) -> CTCHead:
|
|
77
|
+
"""`model` is a Hugging Face repo id or a local copy of the checkpoint (a folder with `ctc/`)."""
|
|
78
|
+
from safetensors.torch import load_file
|
|
79
|
+
|
|
80
|
+
vocab = json.loads(Path(_model_file(model, "ctc/ctc_vocab.json")).read_text(encoding="utf-8"))
|
|
81
|
+
if vocab["blank"] != BLANK or vocab["token_ids"] != list(range(BLANK)):
|
|
82
|
+
raise ValueError("ctc_vocab.json is not the identity mapping this aligner assumes")
|
|
83
|
+
sd = load_file(_model_file(model, "ctc/ctc_head.safetensors"))
|
|
84
|
+
return cls(sd["weight"], sd["bias"], device)
|
|
85
|
+
|
|
86
|
+
@torch.inference_mode()
|
|
87
|
+
def emissions(self, states: torch.Tensor, classes: Sequence[int]) -> np.ndarray:
|
|
88
|
+
"""Log-probs [frames, len(classes)] of the given classes, normalised over all 99,631."""
|
|
89
|
+
h = states.to(self.device, torch.float32)
|
|
90
|
+
idx = torch.as_tensor(list(classes), device=self.device)
|
|
91
|
+
out = []
|
|
92
|
+
for i in range(0, h.shape[0], 512): # 512 x 99,631 floats = 200 MB at most
|
|
93
|
+
logits = torch.nn.functional.linear(h[i : i + 512], self.weight, self.bias)
|
|
94
|
+
out.append(logits[:, idx] - torch.logsumexp(logits, dim=-1, keepdim=True))
|
|
95
|
+
return torch.cat(out).cpu().numpy() if out else np.zeros((0, len(idx)), np.float32)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def ctc_viterbi(em: np.ndarray, n_tokens: int, same_as_prev: Sequence[bool]) -> np.ndarray | None:
|
|
99
|
+
"""CTC forced alignment, vectorised over states.
|
|
100
|
+
|
|
101
|
+
`em`: [T, 1 + n_tokens] log-probs, column 0 = blank, column k+1 = target token k.
|
|
102
|
+
`same_as_prev[k]`: token k equals token k-1 (then a blank must separate them).
|
|
103
|
+
Returns the state path [T] over the 2n+1 extended states (even = blank, 2k+1 = token k),
|
|
104
|
+
or None if the tokens can't fit in T frames.
|
|
105
|
+
"""
|
|
106
|
+
T, L = em.shape[0], n_tokens
|
|
107
|
+
S = 2 * L + 1
|
|
108
|
+
if T == 0 or L + sum(same_as_prev) > T:
|
|
109
|
+
return None
|
|
110
|
+
cols = np.zeros(S, dtype=np.int64)
|
|
111
|
+
cols[1::2] = np.arange(1, L + 1)
|
|
112
|
+
e = em[:, cols] # [T, S]
|
|
113
|
+
skip_ok = np.zeros(S, dtype=bool) # may jump s-2 -> s: token states whose token differs from the previous
|
|
114
|
+
skip_ok[3::2] = ~np.asarray(same_as_prev[1:], dtype=bool) if L > 1 else []
|
|
115
|
+
neg = -np.inf
|
|
116
|
+
dp = np.full(S, neg)
|
|
117
|
+
dp[0] = e[0, 0]
|
|
118
|
+
if S > 1:
|
|
119
|
+
dp[1] = e[0, 1]
|
|
120
|
+
back = np.zeros((T, S), dtype=np.int8)
|
|
121
|
+
for t in range(1, T):
|
|
122
|
+
stay = dp
|
|
123
|
+
step = np.concatenate(([neg], dp[:-1]))
|
|
124
|
+
skip = np.where(skip_ok, np.concatenate(([neg, neg], dp[:-2])), neg)
|
|
125
|
+
cand = np.stack([stay, step, skip])
|
|
126
|
+
arg = cand.argmax(0)
|
|
127
|
+
dp = cand[arg, np.arange(S)] + e[t]
|
|
128
|
+
back[t] = arg
|
|
129
|
+
s = S - 1 if S == 1 or dp[S - 1] >= dp[S - 2] else S - 2
|
|
130
|
+
if not np.isfinite(dp[s]):
|
|
131
|
+
return None
|
|
132
|
+
path = np.empty(T, dtype=np.int64)
|
|
133
|
+
for t in range(T - 1, -1, -1):
|
|
134
|
+
path[t] = s
|
|
135
|
+
s -= int(back[t, s]) # int(): numpy 2 would make s an int8 and overflow past 127 states
|
|
136
|
+
return path
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def token_spans(path: np.ndarray, n_tokens: int) -> list[tuple[int, int]]:
|
|
140
|
+
"""(first, last) frame of each token on a Viterbi path."""
|
|
141
|
+
spans = [[-1, -1] for _ in range(n_tokens)]
|
|
142
|
+
for t, s in enumerate(path):
|
|
143
|
+
if s % 2:
|
|
144
|
+
k = s // 2
|
|
145
|
+
if spans[k][0] < 0:
|
|
146
|
+
spans[k][0] = t
|
|
147
|
+
spans[k][1] = t
|
|
148
|
+
return [(a, b) for a, b in spans]
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
_HAS_ALNUM = re.compile(r"\w")
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
@dataclass
|
|
155
|
+
class AlignedWord:
|
|
156
|
+
text: str
|
|
157
|
+
first: int # first frame of its first token
|
|
158
|
+
last: int # last frame of its last token
|
|
159
|
+
score: float
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def group_words(
|
|
163
|
+
token_ids: Sequence[int],
|
|
164
|
+
spans: Sequence[tuple[int, int]],
|
|
165
|
+
token_probs: Sequence[float],
|
|
166
|
+
tokens: Sequence[str],
|
|
167
|
+
decode: Callable[[list[int]], str],
|
|
168
|
+
) -> list[AlignedWord]:
|
|
169
|
+
"""Group BPE tokens into words at the leading-space marker (`Ġ`). Punctuation-only groups
|
|
170
|
+
(e.g. a lone ` -`) are attached to the previous word, or before the first word (an opening quote)."""
|
|
171
|
+
groups: list[list[int]] = []
|
|
172
|
+
for k, tok in enumerate(tokens):
|
|
173
|
+
if not groups or tok.startswith("Ġ"):
|
|
174
|
+
groups.append([k])
|
|
175
|
+
else:
|
|
176
|
+
groups[-1].append(k)
|
|
177
|
+
words: list[AlignedWord] = []
|
|
178
|
+
lead: list[int] = [] # punctuation-only groups before the first word
|
|
179
|
+
for g in groups:
|
|
180
|
+
text = decode([token_ids[k] for k in g]).strip()
|
|
181
|
+
if not text:
|
|
182
|
+
continue
|
|
183
|
+
score = float(np.mean([token_probs[k] for k in g]))
|
|
184
|
+
if not _HAS_ALNUM.search(text):
|
|
185
|
+
if words:
|
|
186
|
+
words[-1].text += text
|
|
187
|
+
else:
|
|
188
|
+
lead += g
|
|
189
|
+
continue
|
|
190
|
+
if lead and not words:
|
|
191
|
+
text = decode([token_ids[k] for k in lead]).strip() + text
|
|
192
|
+
words.append(AlignedWord(text, spans[g[0]][0], spans[g[-1]][1], score))
|
|
193
|
+
if lead and not words: # nothing but punctuation: keep it as one word
|
|
194
|
+
text = decode([token_ids[k] for k in lead]).strip()
|
|
195
|
+
score = float(np.mean([token_probs[k] for k in lead]))
|
|
196
|
+
words.append(AlignedWord(text, spans[lead[0]][0], spans[lead[-1]][1], score))
|
|
197
|
+
return words
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
# Each rule maps word starts and spike ends (seconds, [n]) plus the chunk's speech regions to word ends.
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _end_spike(starts, spike_end, regions):
|
|
204
|
+
return spike_end.copy()
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def _end_next_start(starts, spike_end, regions):
|
|
208
|
+
ends = spike_end.copy()
|
|
209
|
+
ends[:-1] = np.maximum(ends[:-1], starts[1:])
|
|
210
|
+
return ends
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _end_midpoint(starts, spike_end, regions):
|
|
214
|
+
ends = spike_end.copy()
|
|
215
|
+
ends[:-1] = np.maximum(ends[:-1], (ends[:-1] + starts[1:]) / 2)
|
|
216
|
+
return ends
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def _end_region(starts, spike_end, regions):
|
|
220
|
+
"""Run on to the next word's start, but not past the end of the VAD speech region the word is in,
|
|
221
|
+
so a word before a pause doesn't swallow the pause. The last word runs to its region's end."""
|
|
222
|
+
nxt = np.append(starts[1:], np.inf)
|
|
223
|
+
region_end = np.full_like(spike_end, np.inf)
|
|
224
|
+
for r0, r1 in regions:
|
|
225
|
+
inside = (spike_end > r0) & (spike_end <= r1 + 1e-6)
|
|
226
|
+
region_end[inside] = r1
|
|
227
|
+
cap = np.minimum(nxt, region_end)
|
|
228
|
+
cap[~np.isfinite(cap)] = spike_end[~np.isfinite(cap)]
|
|
229
|
+
return np.maximum(spike_end, cap)
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
END_RULES: dict[str, Callable] = {
|
|
233
|
+
"spike": _end_spike,
|
|
234
|
+
"next_start": _end_next_start,
|
|
235
|
+
"midpoint": _end_midpoint,
|
|
236
|
+
"region": _end_region,
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
# Chosen on LibriSpeech dev_clean (docs/alignment_eval.md): 25 dB below the speech level, for 30 ms.
|
|
241
|
+
SILENCE_DB = 25.0
|
|
242
|
+
MIN_SILENCE = 0.03
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
@dataclass
|
|
246
|
+
class LoudnessRefinement:
|
|
247
|
+
"""Audio for refining word times on loudness. `audio` covers the chunk, starting at global time
|
|
248
|
+
`audio_start`. Moving ends to the first silence helped on dev_clean; moving starts to sound onsets
|
|
249
|
+
did not, so `starts` is off by default."""
|
|
250
|
+
|
|
251
|
+
audio: np.ndarray
|
|
252
|
+
audio_start: float
|
|
253
|
+
silence_db: float = SILENCE_DB
|
|
254
|
+
min_silence: float = MIN_SILENCE
|
|
255
|
+
starts: bool = False
|
|
256
|
+
ends: bool = True
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def word_times(
|
|
260
|
+
words: Sequence[AlignedWord],
|
|
261
|
+
n_frames: int,
|
|
262
|
+
chunk_start: float,
|
|
263
|
+
regions: Sequence[tuple[float, float]],
|
|
264
|
+
grid: str = "block",
|
|
265
|
+
end_rule: str = "next_start",
|
|
266
|
+
start_shift: float = 0.0,
|
|
267
|
+
loudness: LoudnessRefinement | None = None,
|
|
268
|
+
) -> tuple[np.ndarray, np.ndarray]:
|
|
269
|
+
"""Start/end seconds (global) for aligned words. A frame is FRAME_STEP wide around its centre.
|
|
270
|
+
Words never overlap: an end is clamped to the next word's start."""
|
|
271
|
+
times = frame_times(n_frames, grid) + chunk_start
|
|
272
|
+
spike_start = np.array([times[w.first] for w in words])
|
|
273
|
+
starts = np.maximum(spike_start - FRAME_STEP / 2 + start_shift, chunk_start)
|
|
274
|
+
spike_end = np.array([times[w.last] for w in words]) + FRAME_STEP / 2
|
|
275
|
+
lr = loudness
|
|
276
|
+
if lr is not None and lr.starts:
|
|
277
|
+
starts = refine_starts(
|
|
278
|
+
starts, spike_start, spike_end, lr.audio, lr.audio_start, lr.silence_db, lr.min_silence
|
|
279
|
+
)
|
|
280
|
+
ends = END_RULES[end_rule](starts, spike_end, list(regions))
|
|
281
|
+
ends[:-1] = np.minimum(ends[:-1], starts[1:])
|
|
282
|
+
if lr is not None and lr.ends:
|
|
283
|
+
ends = refine_ends(starts, spike_end, ends, lr.audio, lr.audio_start, lr.silence_db, lr.min_silence)
|
|
284
|
+
ends = np.maximum(ends, starts + 0.01)
|
|
285
|
+
return starts, ends
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
ENERGY_HOP = 0.01 # seconds per loudness frame
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def frame_db(audio: np.ndarray, sr: int = SAMPLE_RATE, hop: float = ENERGY_HOP) -> np.ndarray:
|
|
292
|
+
"""Loudness (dB of RMS) per non-overlapping `hop` frame."""
|
|
293
|
+
n = round(hop * sr)
|
|
294
|
+
k = len(audio) // n
|
|
295
|
+
if k == 0:
|
|
296
|
+
return np.zeros(0)
|
|
297
|
+
f = audio[: k * n].astype(np.float64).reshape(k, n)
|
|
298
|
+
return 10 * np.log10((f**2).mean(axis=1) + 1e-12)
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def _silent_frames(audio: np.ndarray, silence_db: float) -> np.ndarray:
|
|
302
|
+
"""Per 10 ms frame: is it `silence_db` below the chunk's speech level (95th percentile loudness)?"""
|
|
303
|
+
db = frame_db(audio)
|
|
304
|
+
return db < np.percentile(db, 95) - silence_db if db.size else np.zeros(0, bool)
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def refine_ends(
|
|
308
|
+
starts: np.ndarray,
|
|
309
|
+
spike_end: np.ndarray,
|
|
310
|
+
ends: np.ndarray,
|
|
311
|
+
audio: np.ndarray,
|
|
312
|
+
audio_start: float,
|
|
313
|
+
silence_db: float = SILENCE_DB,
|
|
314
|
+
min_silence: float = MIN_SILENCE,
|
|
315
|
+
) -> np.ndarray:
|
|
316
|
+
"""Pull word ends back to where the audio goes silent.
|
|
317
|
+
|
|
318
|
+
`audio` covers the chunk and starts at global time `audio_start`. Between a word's last CTC spike and
|
|
319
|
+
its current end (usually the next word's start), the first run of at least `min_silence` seconds whose
|
|
320
|
+
loudness is `silence_db` below the chunk's speech level (95th percentile) ends the word. Without
|
|
321
|
+
such a run the end is kept, so this only ever moves ends earlier.
|
|
322
|
+
"""
|
|
323
|
+
silent = _silent_frames(audio, silence_db)
|
|
324
|
+
if silent.size == 0:
|
|
325
|
+
return ends
|
|
326
|
+
need = max(1, round(min_silence / ENERGY_HOP))
|
|
327
|
+
# run[i] = number of consecutive silent frames starting at frame i
|
|
328
|
+
run = np.zeros(len(silent) + 1, dtype=np.int64)
|
|
329
|
+
for i in range(len(silent) - 1, -1, -1):
|
|
330
|
+
run[i] = run[i + 1] + 1 if silent[i] else 0
|
|
331
|
+
out = ends.copy()
|
|
332
|
+
for i in range(len(ends)):
|
|
333
|
+
a = max(0, int(np.floor((spike_end[i] - audio_start) / ENERGY_HOP)))
|
|
334
|
+
b = min(len(silent), int(np.ceil((ends[i] - audio_start) / ENERGY_HOP)))
|
|
335
|
+
if b - a < need:
|
|
336
|
+
continue
|
|
337
|
+
hits = np.nonzero(run[a:b] >= need)[0]
|
|
338
|
+
if hits.size:
|
|
339
|
+
out[i] = max(audio_start + (a + hits[0]) * ENERGY_HOP, starts[i] + 0.01)
|
|
340
|
+
return out
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
def refine_starts(
|
|
344
|
+
starts: np.ndarray,
|
|
345
|
+
spike_start: np.ndarray,
|
|
346
|
+
spike_end: np.ndarray,
|
|
347
|
+
audio: np.ndarray,
|
|
348
|
+
audio_start: float,
|
|
349
|
+
silence_db: float = SILENCE_DB,
|
|
350
|
+
min_silence: float = MIN_SILENCE,
|
|
351
|
+
) -> np.ndarray:
|
|
352
|
+
"""Move a word's start to where sound resumes after a silence, if there is one.
|
|
353
|
+
|
|
354
|
+
Searches between the previous word's last spike (or the chunk start) and the word's first spike for
|
|
355
|
+
the last run of at least `min_silence` seconds of silence (same definition as `refine_ends`); the
|
|
356
|
+
word then starts where that run ends. Words with no such silence before them are left alone.
|
|
357
|
+
"""
|
|
358
|
+
silent = _silent_frames(audio, silence_db)
|
|
359
|
+
if silent.size == 0:
|
|
360
|
+
return starts
|
|
361
|
+
need = max(1, round(min_silence / ENERGY_HOP))
|
|
362
|
+
back = np.zeros(len(silent), dtype=np.int64) # back[i] = consecutive silent frames ending at frame i
|
|
363
|
+
for i in range(len(silent)):
|
|
364
|
+
back[i] = (back[i - 1] + 1 if i else 1) if silent[i] else 0
|
|
365
|
+
out = starts.copy()
|
|
366
|
+
for i in range(len(starts)):
|
|
367
|
+
lo = spike_end[i - 1] if i else audio_start
|
|
368
|
+
a = max(0, int(np.floor((lo - audio_start) / ENERGY_HOP)))
|
|
369
|
+
b = min(len(silent), int(np.ceil((spike_start[i] - audio_start) / ENERGY_HOP)))
|
|
370
|
+
hits = np.nonzero(back[a:b] >= need)[0]
|
|
371
|
+
if hits.size:
|
|
372
|
+
onset = audio_start + (a + hits[-1] + 1) * ENERGY_HOP
|
|
373
|
+
out[i] = min(onset, spike_start[i])
|
|
374
|
+
return out
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
@dataclass
|
|
378
|
+
class AlignOptions:
|
|
379
|
+
end_rule: str = "next_start" # chosen on dev_clean, see docs/alignment_eval.md
|
|
380
|
+
grid: str = "block"
|
|
381
|
+
start_shift: float = -0.022 # added to every start; median start error on dev_clean (block grid)
|
|
382
|
+
split_sentences: bool = True
|
|
383
|
+
# Loudness refinement, used when the audio is passed to align() (see LoudnessRefinement).
|
|
384
|
+
refine_ends: bool = True
|
|
385
|
+
refine_starts: bool = False
|
|
386
|
+
silence_db: float = SILENCE_DB
|
|
387
|
+
min_silence: float = MIN_SILENCE
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
class Aligner:
|
|
391
|
+
def __init__(self, head: CTCHead, tokenizer, opts: AlignOptions | None = None):
|
|
392
|
+
self.head = head
|
|
393
|
+
self.tokenizer = tokenizer
|
|
394
|
+
self.opts = opts or AlignOptions()
|
|
395
|
+
if self.opts.end_rule not in END_RULES:
|
|
396
|
+
raise ValueError(f"end_rule must be one of {list(END_RULES)}")
|
|
397
|
+
|
|
398
|
+
@classmethod
|
|
399
|
+
def from_transcriber(
|
|
400
|
+
cls, transcriber, opts: AlignOptions | None = None, model: str | None = None
|
|
401
|
+
) -> Aligner:
|
|
402
|
+
head = CTCHead.from_pretrained(model or transcriber.model.name_or_path, device=transcriber.device)
|
|
403
|
+
return cls(head, transcriber.tokenizer, opts)
|
|
404
|
+
|
|
405
|
+
def align_tokens(
|
|
406
|
+
self,
|
|
407
|
+
token_ids: Sequence[int],
|
|
408
|
+
states: torch.Tensor,
|
|
409
|
+
chunk_start: float,
|
|
410
|
+
regions: Sequence[tuple[float, float]] = (),
|
|
411
|
+
audio: np.ndarray | None = None,
|
|
412
|
+
) -> list[WordSegment] | None:
|
|
413
|
+
"""Word timings (global seconds) for one chunk's tokens, or None if CTC alignment fails.
|
|
414
|
+
`regions` are the chunk's speech regions in global seconds."""
|
|
415
|
+
token_ids = list(token_ids)
|
|
416
|
+
if not token_ids or states is None:
|
|
417
|
+
return None
|
|
418
|
+
em = self.head.emissions(states, [BLANK, *token_ids])
|
|
419
|
+
same = [False] + [a == b for a, b in pairwise(token_ids)]
|
|
420
|
+
path = ctc_viterbi(em, len(token_ids), same)
|
|
421
|
+
if path is None:
|
|
422
|
+
return None
|
|
423
|
+
spans = token_spans(path, len(token_ids))
|
|
424
|
+
probs = [float(np.exp(em[a : b + 1, k + 1]).mean()) for k, (a, b) in enumerate(spans)]
|
|
425
|
+
tokens = self.tokenizer.convert_ids_to_tokens(token_ids)
|
|
426
|
+
words = group_words(token_ids, spans, probs, tokens, self._decode)
|
|
427
|
+
if not words:
|
|
428
|
+
return None
|
|
429
|
+
o = self.opts
|
|
430
|
+
loudness = None
|
|
431
|
+
if audio is not None and (o.refine_ends or o.refine_starts):
|
|
432
|
+
loudness = LoudnessRefinement(
|
|
433
|
+
audio, chunk_start, o.silence_db, o.min_silence, starts=o.refine_starts, ends=o.refine_ends
|
|
434
|
+
)
|
|
435
|
+
starts, ends = word_times(
|
|
436
|
+
words, len(em), chunk_start, regions, o.grid, o.end_rule, o.start_shift, loudness
|
|
437
|
+
)
|
|
438
|
+
return [
|
|
439
|
+
WordSegment(w.text, round(float(s), 3), round(float(e), 3), round(w.score, 3))
|
|
440
|
+
for w, s, e in zip(words, starts, ends)
|
|
441
|
+
]
|
|
442
|
+
|
|
443
|
+
def align(self, result: TranscriptResult, audio: np.ndarray | None = None) -> TranscriptResult:
|
|
444
|
+
"""Fill in words for a result from `RumbleTranscriber.transcribe` (needs keep_encoder_states=True).
|
|
445
|
+
Pass the 16 kHz `audio` that was transcribed to enable the loudness refinement of word ends.
|
|
446
|
+
Segments are re-cut at sentence ends when `split_sentences` is on."""
|
|
447
|
+
items = result.metadata.get("chunk_transcripts")
|
|
448
|
+
if items is None:
|
|
449
|
+
raise ValueError("result has no chunk_transcripts; transcribe with keep_encoder_states=True")
|
|
450
|
+
if any(it.text and it.encoder_states is None for it in items):
|
|
451
|
+
raise ValueError(
|
|
452
|
+
"chunk_transcripts have no encoder states; transcribe with keep_encoder_states=True"
|
|
453
|
+
)
|
|
454
|
+
segments: list[Segment] = []
|
|
455
|
+
n_fallback = 0
|
|
456
|
+
for it in items:
|
|
457
|
+
if not it.text:
|
|
458
|
+
continue
|
|
459
|
+
regions = [(r.start / SAMPLE_RATE, r.end / SAMPLE_RATE) for r in it.chunk.regions] or [
|
|
460
|
+
(it.chunk.start_s, it.chunk.end_s)
|
|
461
|
+
]
|
|
462
|
+
chunk_audio = None if audio is None else it.chunk.slice(audio)
|
|
463
|
+
words = self.align_tokens(it.token_ids, it.encoder_states, it.chunk.start_s, regions, chunk_audio)
|
|
464
|
+
if words is None:
|
|
465
|
+
n_fallback += 1
|
|
466
|
+
words = proportional_words(it.text, regions)
|
|
467
|
+
clamp_to_chunk(words, it.chunk.start_s, it.chunk.end_s)
|
|
468
|
+
seg = Segment(words[0].start, words[-1].end, it.text, words, language=it.language)
|
|
469
|
+
segments.extend(split_sentences(seg) if self.opts.split_sentences else [seg])
|
|
470
|
+
result.segments = segments
|
|
471
|
+
result.metadata["alignment"] = {
|
|
472
|
+
**vars(self.opts),
|
|
473
|
+
"audio_refinement": audio is not None,
|
|
474
|
+
"fallback_chunks": n_fallback,
|
|
475
|
+
}
|
|
476
|
+
return result
|
|
477
|
+
|
|
478
|
+
def _decode(self, ids: list[int]) -> str:
|
|
479
|
+
return self.tokenizer.decode(ids, clean_up_tokenization_spaces=False)
|
|
480
|
+
|
|
481
|
+
|
|
482
|
+
def clamp_to_chunk(words: list[WordSegment], start: float, end: float) -> None:
|
|
483
|
+
"""Keep word times inside the chunk's real audio. The silence appended before encoding
|
|
484
|
+
(transcriber.END_PAD) can otherwise put the last word's end a few ms past the end."""
|
|
485
|
+
for w in words:
|
|
486
|
+
w.end = round(min(w.end, end), 3)
|
|
487
|
+
w.start = round(min(max(w.start, start), max(w.end - 0.01, start)), 3)
|
|
488
|
+
|
|
489
|
+
|
|
490
|
+
def proportional_words(text: str, regions: Sequence[tuple[float, float]]) -> list[WordSegment]:
|
|
491
|
+
"""Spread words over the speech regions in proportion to their length (score 0: not aligned)."""
|
|
492
|
+
words = text.split()
|
|
493
|
+
if not words:
|
|
494
|
+
return []
|
|
495
|
+
total = sum(e - s for s, e in regions)
|
|
496
|
+
weights = np.array([len(w) + 1 for w in words], dtype=float)
|
|
497
|
+
cum = np.concatenate(([0.0], np.cumsum(weights) / weights.sum() * total))
|
|
498
|
+
|
|
499
|
+
def at(x: float, start: bool) -> float:
|
|
500
|
+
"""Speech-time offset -> wall time. A start exactly on a region boundary goes to the next region."""
|
|
501
|
+
for s, e in regions:
|
|
502
|
+
if x < e - s or (not start and x <= e - s):
|
|
503
|
+
return s + x
|
|
504
|
+
x -= e - s
|
|
505
|
+
return regions[-1][1]
|
|
506
|
+
|
|
507
|
+
return [
|
|
508
|
+
WordSegment(w, round(float(at(cum[i], True)), 3), round(float(at(cum[i + 1], False)), 3), 0.0)
|
|
509
|
+
for i, w in enumerate(words)
|
|
510
|
+
]
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
_SENTENCE_END = re.compile(r"[.!?。?!]['\")\]]*$")
|
|
514
|
+
# Words ending in "." that rarely end a sentence (English and Dutch titles and common abbreviations).
|
|
515
|
+
_ABBREVIATIONS = {
|
|
516
|
+
"mr.", "mrs.", "ms.", "dr.", "prof.", "st.", "jr.", "sr.", "vs.", "etc.", "e.g.", "i.e.", "no.",
|
|
517
|
+
"dhr.", "mevr.", "mw.", "bijv.", "o.a.", "z.g.a.n.", "nr.", "ca.",
|
|
518
|
+
} # fmt: skip
|
|
519
|
+
|
|
520
|
+
|
|
521
|
+
def split_sentences(seg: Segment) -> list[Segment]:
|
|
522
|
+
"""Cut a segment after words that end a sentence."""
|
|
523
|
+
if not seg.words:
|
|
524
|
+
return [seg]
|
|
525
|
+
out, cur = [], []
|
|
526
|
+
for w in seg.words:
|
|
527
|
+
cur.append(w)
|
|
528
|
+
if _SENTENCE_END.search(w.word) and w.word.lower() not in _ABBREVIATIONS:
|
|
529
|
+
out.append(cur)
|
|
530
|
+
cur = []
|
|
531
|
+
if cur:
|
|
532
|
+
out.append(cur)
|
|
533
|
+
return [
|
|
534
|
+
Segment(ws[0].start, ws[-1].end, " ".join(w.word for w in ws), ws, seg.speaker, seg.language)
|
|
535
|
+
for ws in out
|
|
536
|
+
]
|