kinescript-chatterbox 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kinescript_chatterbox/__init__.py +143 -0
- kinescript_chatterbox/__main__.py +14 -0
- kinescript_chatterbox/align.py +183 -0
- kinescript_chatterbox-0.1.0.dist-info/METADATA +118 -0
- kinescript_chatterbox-0.1.0.dist-info/RECORD +9 -0
- kinescript_chatterbox-0.1.0.dist-info/WHEEL +5 -0
- kinescript_chatterbox-0.1.0.dist-info/entry_points.txt +2 -0
- kinescript_chatterbox-0.1.0.dist-info/licenses/LICENSE +25 -0
- kinescript_chatterbox-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
"""Chatterbox voices for `kine voice`: Resemble AI's open models speak a sound's text, Turbo with tags such as
|
|
2
|
+
[laugh], and a wav2vec2 alignment times each word (Chatterbox gives no word times of its own).
|
|
3
|
+
|
|
4
|
+
kine voice calls synth(req) through the "kinescript.voice" entry point, or this package runs as
|
|
5
|
+
`python -m kinescript_chatterbox` for KINE_VOICE_CMD_CHATTERBOX, when Chatterbox lives in another Python.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import os
|
|
10
|
+
import re
|
|
11
|
+
import wave
|
|
12
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
from .align import TAG, align
|
|
16
|
+
|
|
17
|
+
MODELS = ("turbo", "standard") # turbo acts on tags; standard takes exaggeration and cfg_weight instead
|
|
18
|
+
TURBO_TAGS: set[str] = set() # filled from Turbo's tokenizer when it loads
|
|
19
|
+
OPTIONS = {"turbo": {"temperature", "top_p", "top_k", "repetition_penalty"},
|
|
20
|
+
"standard": {"exaggeration", "cfg_weight", "temperature", "min_p", "top_p", "repetition_penalty"}}
|
|
21
|
+
CHUNK = 300 # characters per generate call: Chatterbox reads long text better a few sentences at a time
|
|
22
|
+
_models: dict = {}
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _device() -> str:
|
|
26
|
+
"""cuda when this torch was built for the GPU, else cpu. CHATTERBOX_DEVICE picks one."""
|
|
27
|
+
if os.environ.get("CHATTERBOX_DEVICE"):
|
|
28
|
+
return os.environ["CHATTERBOX_DEVICE"]
|
|
29
|
+
import torch
|
|
30
|
+
if torch.cuda.is_available():
|
|
31
|
+
major, minor = torch.cuda.get_device_capability()
|
|
32
|
+
if f"sm_{major}{minor}" in torch.cuda.get_arch_list():
|
|
33
|
+
return "cuda"
|
|
34
|
+
return "cpu"
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _model(kind: str):
|
|
38
|
+
"""The model and its own voice, loaded once per process."""
|
|
39
|
+
if kind not in _models:
|
|
40
|
+
if kind == "turbo":
|
|
41
|
+
from chatterbox.tts_turbo import ChatterboxTurboTTS as Model
|
|
42
|
+
else:
|
|
43
|
+
from chatterbox.tts import ChatterboxTTS as Model
|
|
44
|
+
m = Model.from_pretrained(device=_device())
|
|
45
|
+
if kind == "turbo":
|
|
46
|
+
TURBO_TAGS.update(t for t in m.tokenizer.get_added_vocab() if TAG.fullmatch(t))
|
|
47
|
+
# chatterbox-tts 0.1.7 scales the clip by a NumPy float64, which makes float64 under NumPy 2 (Python
|
|
48
|
+
# 3.13 and later), and its tokenizer then fails on a clip to copy; keep the clip float32
|
|
49
|
+
norm = m.norm_loudness
|
|
50
|
+
m.norm_loudness = lambda wav, sr, **kw: norm(wav, sr, **kw).astype("float32", copy=False)
|
|
51
|
+
_models[kind] = (m, m.conds)
|
|
52
|
+
return _models[kind]
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _forms(word: str) -> set[str]:
|
|
56
|
+
return {word} | {word[:-len(s)] for s in ("ing", "es", "s") if word.endswith(s) and len(word) - len(s) >= 3}
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def turbo_tag(tag: str) -> str | None:
|
|
60
|
+
"""The Turbo tag for one written as another backend writes it ([laughs] -> [laugh], [whispers] ->
|
|
61
|
+
[whispering], [clears throat] -> [clear throat]), or None."""
|
|
62
|
+
t = tag.lower()
|
|
63
|
+
if t in TURBO_TAGS:
|
|
64
|
+
return t
|
|
65
|
+
words = t[1:-1].split()
|
|
66
|
+
for known in sorted(TURBO_TAGS):
|
|
67
|
+
kw = known[1:-1].split()
|
|
68
|
+
if len(kw) == len(words) and all(_forms(a) & _forms(b) for a, b in zip(words, kw)):
|
|
69
|
+
return known
|
|
70
|
+
return None
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def chunks(text: str, size: int = CHUNK) -> list[str]:
|
|
74
|
+
"""The text in pieces of whole sentences, each at most about `size` characters."""
|
|
75
|
+
out = []
|
|
76
|
+
for sentence in re.split(r"(?<=[.!?])\s+", text.strip()):
|
|
77
|
+
if out and len(out[-1]) + 1 + len(sentence) <= size:
|
|
78
|
+
out[-1] += " " + sentence
|
|
79
|
+
elif sentence:
|
|
80
|
+
out.append(sentence)
|
|
81
|
+
return out
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _version(kind: str) -> str:
|
|
85
|
+
def v(name):
|
|
86
|
+
try:
|
|
87
|
+
return version(name)
|
|
88
|
+
except PackageNotFoundError:
|
|
89
|
+
return "?"
|
|
90
|
+
return f"chatterbox-tts {v('chatterbox-tts')} {kind}, kinescript-chatterbox {v('kinescript-chatterbox')}"
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def synth(req: dict) -> dict:
|
|
94
|
+
"""Speak req["tagged"] (Turbo) or req["text"] (standard) into the WAV req["out"], in the voice of the clip
|
|
95
|
+
req["speaker"] names (5 s or more, relative to req["base"]) or the model's own voice, and time each word."""
|
|
96
|
+
import numpy as np
|
|
97
|
+
import torch
|
|
98
|
+
opts = dict(req.get("options") or {})
|
|
99
|
+
kind = str(opts.pop("model", "turbo"))
|
|
100
|
+
if kind not in MODELS:
|
|
101
|
+
raise ValueError(f"Chatterbox model '{kind}': use one of {', '.join(MODELS)}")
|
|
102
|
+
seed = int(opts.pop("seed", 0))
|
|
103
|
+
m, own = _model(kind)
|
|
104
|
+
ignored = [f"options.{k}" for k in opts if k not in OPTIONS[kind]]
|
|
105
|
+
if float(req.get("rate") or 1) != 1:
|
|
106
|
+
ignored.append(f"rate {req['rate']}: Chatterbox speaks at one speed")
|
|
107
|
+
text = (req.get("tagged") or req["text"]) if kind == "turbo" else req["text"]
|
|
108
|
+
if kind == "turbo": # a tag Turbo does not know would be read out as a word
|
|
109
|
+
ignored += [t for t in TAG.findall(text) if not turbo_tag(t)]
|
|
110
|
+
text = re.sub(r"\s+", " ", TAG.sub(lambda t: turbo_tag(t.group(0)) or "", text)).strip()
|
|
111
|
+
speaker = req.get("speaker")
|
|
112
|
+
if speaker:
|
|
113
|
+
ref = Path(speaker)
|
|
114
|
+
ref = ref if ref.is_absolute() else Path(req.get("base") or ".") / ref
|
|
115
|
+
if not ref.exists():
|
|
116
|
+
raise FileNotFoundError(f"Chatterbox clones the voice of a clip of 5 s or more; {ref} does not exist")
|
|
117
|
+
m.prepare_conditionals(str(ref), **({"exaggeration": opts["exaggeration"]} if "exaggeration" in opts else {}))
|
|
118
|
+
else:
|
|
119
|
+
m.conds = own
|
|
120
|
+
kw = {k: v for k, v in opts.items() if k in OPTIONS[kind]}
|
|
121
|
+
audio, words, untimed, at = [], [], [], 0
|
|
122
|
+
for k, piece in enumerate(chunks(text)):
|
|
123
|
+
torch.manual_seed(seed + k) # the same text says the same thing on the next run
|
|
124
|
+
wav = m.generate(piece, **kw).squeeze(0).numpy().astype(np.float32)
|
|
125
|
+
got = align(wav, m.sr, piece, device=_device(), offset=at / m.sr)
|
|
126
|
+
said = " ".join(TAG.sub(" ", piece).split())
|
|
127
|
+
if not got and said: # the sound is too short to hold the text: it was cut off
|
|
128
|
+
untimed.append(said)
|
|
129
|
+
words += got
|
|
130
|
+
audio.append(wav)
|
|
131
|
+
at += len(wav)
|
|
132
|
+
out = np.concatenate(audio) if audio else np.zeros(0, np.float32)
|
|
133
|
+
with wave.open(req["out"], "wb") as f:
|
|
134
|
+
f.setnchannels(1)
|
|
135
|
+
f.setsampwidth(2)
|
|
136
|
+
f.setframerate(m.sr)
|
|
137
|
+
f.writeframes((np.clip(out, -1, 1) * 32767).astype("<i2").tobytes())
|
|
138
|
+
got = {"words": words, "speaker": speaker or f"{kind} built-in", "version": _version(kind),
|
|
139
|
+
"seconds": round(at / m.sr, 3), "tags": kind == "turbo", "ignored": ignored}
|
|
140
|
+
if untimed:
|
|
141
|
+
got["note"] = ("too short for its text, so these words have no times (try another seed): "
|
|
142
|
+
+ " / ".join(untimed))
|
|
143
|
+
return got
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"""For KINE_VOICE_CMD_CHATTERBOX: a JSON list of requests on stdin, the list of results on stdout.
|
|
2
|
+
|
|
3
|
+
KINE_VOICE_CMD_CHATTERBOX='["/path/to/python", "-m", "kinescript_chatterbox"]' kine voice scene.json
|
|
4
|
+
"""
|
|
5
|
+
import contextlib
|
|
6
|
+
import json
|
|
7
|
+
import sys
|
|
8
|
+
|
|
9
|
+
from . import synth
|
|
10
|
+
|
|
11
|
+
reqs = json.loads(sys.stdin.read())
|
|
12
|
+
with contextlib.redirect_stdout(sys.stderr): # Chatterbox's own messages stay out of the answer
|
|
13
|
+
results = [synth(r) for r in reqs]
|
|
14
|
+
sys.stdout.write(json.dumps(results))
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
"""Word times for speech whose words are known: wav2vec2 gives each 20 ms of sound the odds of every letter, and a
|
|
2
|
+
CTC forced alignment finds the path through the text's letters that fits those odds best.
|
|
3
|
+
|
|
4
|
+
A number is aligned as it is read ("250" as TWO HUNDRED FIFTY, "2026" as TWENTY TWENTY SIX), so its words claim
|
|
5
|
+
their own sound. A tag ([laugh]), and a word with nothing left to align, become a wildcard that any sound can
|
|
6
|
+
fill, so a laugh does not stretch the words around it.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import re
|
|
11
|
+
|
|
12
|
+
import numpy as np
|
|
13
|
+
|
|
14
|
+
MODEL = "facebook/wav2vec2-base-960h"
|
|
15
|
+
SR = 16000
|
|
16
|
+
HOP = 320 # input samples per output frame: 20 ms
|
|
17
|
+
TAG = re.compile(r"(?<!\[)\[(?!\[)[^\[\]()]+\](?") # as kinescript.voice.TAG
|
|
18
|
+
WILD, MAYBE = -1, -2 # a wildcard that must take a frame (a word with no letters), and one that may take none (a tag)
|
|
19
|
+
PENALTY = 1.0 # what a wildcard frame costs (nats), so the text's own letters win wherever they fit
|
|
20
|
+
_model: dict = {}
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
ONES = ("zero one two three four five six seven eight nine ten eleven twelve thirteen fourteen fifteen sixteen "
|
|
24
|
+
"seventeen eighteen nineteen").split()
|
|
25
|
+
TENS = "_ _ twenty thirty forty fifty sixty seventy eighty ninety".split()
|
|
26
|
+
SYMBOLS = {"%": "percent", "&": "and", "+": "plus", "$": "dollars"}
|
|
27
|
+
ORDINALS = {"one": "first", "two": "second", "three": "third", "five": "fifth", "eight": "eighth", "nine": "ninth",
|
|
28
|
+
"twelve": "twelfth"}
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def letters(word: str) -> str:
|
|
32
|
+
return re.sub(r"[^A-Z']", "", word.upper().replace("’", "'"))
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _say(n: int) -> list[str]:
|
|
36
|
+
if n < 20:
|
|
37
|
+
return [ONES[n]]
|
|
38
|
+
if n < 100:
|
|
39
|
+
return [TENS[n // 10]] + ([ONES[n % 10]] if n % 10 else [])
|
|
40
|
+
if n < 1000:
|
|
41
|
+
return [ONES[n // 100], "hundred"] + (_say(n % 100) if n % 100 else [])
|
|
42
|
+
size, name = next((s, w) for s, w in ((10 ** 9, "billion"), (10 ** 6, "million"), (1000, "thousand")) if n >= s)
|
|
43
|
+
return _say(n // size) + [name] + (_say(n % size) if n % size else [])
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def number(digits: str) -> list[str]:
|
|
47
|
+
"""How a number is read: 250 two hundred fifty, 3.5 three point five, 1.2.0 one point two point zero, 2026 (a
|
|
48
|
+
year) twenty twenty six."""
|
|
49
|
+
whole, *rest = digits.replace(",", "").split(".")
|
|
50
|
+
n = int(whole)
|
|
51
|
+
if not rest and len(whole) == 4 and 1100 <= n <= 2099 and n % 100 and not 2000 <= n <= 2009:
|
|
52
|
+
return _say(n // 100) + (["oh"] if n % 100 < 10 else []) + _say(n % 100)
|
|
53
|
+
if len(rest) == 1:
|
|
54
|
+
return _say(n) + ["point"] + [ONES[int(d)] for d in rest[0]]
|
|
55
|
+
return _say(n) + [w for part in rest for w in ["point"] + _say(int(part))]
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def ordinal(word: str) -> str:
|
|
59
|
+
"""first, twentieth, hundredth: the ordinal of a number word."""
|
|
60
|
+
if word in ORDINALS:
|
|
61
|
+
return ORDINALS[word]
|
|
62
|
+
return word[:-1] + "ieth" if word.endswith("y") else word + "th"
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def spoken(word: str) -> list[str]:
|
|
66
|
+
"""The words a written word is read as, in capitals: "250" -> [TWO, HUNDRED, FIFTY], "Render." -> [RENDER]."""
|
|
67
|
+
if not re.search(r"\d", word):
|
|
68
|
+
return [letters(word)]
|
|
69
|
+
out = []
|
|
70
|
+
for part in re.findall(r"\d+(?:[.,]\d+)*|[A-Za-z']+|[%&+$]", word):
|
|
71
|
+
if part.lower() in ("st", "nd", "rd", "th") and out and out[-1] in ONES + TENS + ["hundred", "thousand"]:
|
|
72
|
+
out[-1] = ordinal(out[-1]) # 21st: twenty first
|
|
73
|
+
else:
|
|
74
|
+
out += number(part) if part[0].isdigit() else [SYMBOLS.get(part, part)]
|
|
75
|
+
if "$" in word: # $5: five dollars
|
|
76
|
+
out = [w for w in out if w != "dollars"] + ["dollars"]
|
|
77
|
+
return [letters(w) for w in out]
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def units(tagged: str, vocab: dict) -> tuple[list[str], list[tuple[int, int | None]]]:
|
|
81
|
+
"""The words of the text as written, and what is aligned: (vocabulary id, or WILD/MAYBE; the word's index, or
|
|
82
|
+
None for a tag or a word gap)."""
|
|
83
|
+
words, seq, gap = [], [], vocab["|"]
|
|
84
|
+
for piece in re.split(f"({TAG.pattern})", tagged):
|
|
85
|
+
if TAG.fullmatch(piece):
|
|
86
|
+
seq.append((MAYBE, None))
|
|
87
|
+
continue
|
|
88
|
+
for w in piece.split():
|
|
89
|
+
if not re.search(r"\w", w): # punctuation on its own ("—"): nothing is said
|
|
90
|
+
continue
|
|
91
|
+
if seq and seq[-1][0] != MAYBE:
|
|
92
|
+
seq.append((gap, None))
|
|
93
|
+
ids = []
|
|
94
|
+
for k, part in enumerate(spoken(w)):
|
|
95
|
+
ids += ([gap] if k else []) + [vocab[c] for c in part if c in vocab]
|
|
96
|
+
# nothing left to align: any sound
|
|
97
|
+
seq += [(i, len(words)) for i in ids] if set(ids) - {gap} else [(WILD, len(words))]
|
|
98
|
+
words.append(w)
|
|
99
|
+
return words, seq
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def ctc_path(logp: np.ndarray, seq: list[tuple[int, int | None]], blank: int) -> np.ndarray:
|
|
103
|
+
"""The best CTC path through seq for frames of log odds logp (frames x vocabulary): the index into seq that
|
|
104
|
+
each frame says, or -1 for a blank."""
|
|
105
|
+
T = logp.shape[0]
|
|
106
|
+
labels = [blank]
|
|
107
|
+
for lab, _ in seq:
|
|
108
|
+
labels += [lab, blank]
|
|
109
|
+
S = len(labels)
|
|
110
|
+
speech = np.delete(logp, blank, axis=1).max(axis=1) - PENALTY # a wildcard scores as the likeliest letter
|
|
111
|
+
lab = np.array(labels)
|
|
112
|
+
emit = np.where(lab[None, :] < 0, speech[:, None], logp[:, np.clip(lab, 0, None)])
|
|
113
|
+
# a step of 2 skips the blank between two different letters; around a tag it skips the tag itself
|
|
114
|
+
skip = np.zeros(S, bool)
|
|
115
|
+
for s in range(2, S):
|
|
116
|
+
if lab[s] != blank and lab[s] != lab[s - 2]:
|
|
117
|
+
skip[s] = True
|
|
118
|
+
if lab[s] == blank and lab[s - 1] == MAYBE:
|
|
119
|
+
skip[s] = True
|
|
120
|
+
neg = -np.inf
|
|
121
|
+
score = np.full(S, neg)
|
|
122
|
+
score[0], score[1] = emit[0, 0], emit[0, 1]
|
|
123
|
+
if S > 3 and lab[1] == MAYBE:
|
|
124
|
+
score[2] = emit[0, 2]
|
|
125
|
+
back = np.zeros((T, S), np.int8)
|
|
126
|
+
for t in range(1, T):
|
|
127
|
+
one = np.concatenate([[neg], score[:-1]])
|
|
128
|
+
two = np.where(skip, np.concatenate([[neg, neg], score[:-2]]), neg)
|
|
129
|
+
stack = np.stack([score, one, two])
|
|
130
|
+
back[t] = stack.argmax(axis=0)
|
|
131
|
+
score = stack.max(axis=0) + emit[t]
|
|
132
|
+
ends = [S - 1, S - 2] + ([S - 3, S - 4] if S > 4 and lab[S - 2] == MAYBE else [])
|
|
133
|
+
s = max(ends, key=lambda e: score[e])
|
|
134
|
+
if not np.isfinite(score[s]):
|
|
135
|
+
raise ValueError("the sound is too short for its text")
|
|
136
|
+
path = np.empty(T, int)
|
|
137
|
+
for t in range(T - 1, -1, -1):
|
|
138
|
+
path[t] = s
|
|
139
|
+
s -= int(back[t, s]) # int8 minus a Python int stays int8, which overflows past 127 states
|
|
140
|
+
return np.where(path % 2 == 1, (path - 1) // 2, -1)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def times(path: np.ndarray, seq: list, words: list[str], offset: float = 0.0) -> list[dict]:
|
|
144
|
+
"""[{"word", "start", "end"}] from the frames each word's letters took; a word that took none (the alignment
|
|
145
|
+
could not place it) takes the end of the word before."""
|
|
146
|
+
owner = np.array([seq[i][1] if i >= 0 and seq[i][1] is not None else -1 for i in path])
|
|
147
|
+
out, last = [], offset
|
|
148
|
+
for k, w in enumerate(words):
|
|
149
|
+
frames = np.flatnonzero(owner == k)
|
|
150
|
+
if len(frames):
|
|
151
|
+
start, end = offset + frames[0] * HOP / SR, offset + (frames[-1] + 1) * HOP / SR
|
|
152
|
+
else:
|
|
153
|
+
start = end = last
|
|
154
|
+
out.append({"word": w, "start": round(start, 3), "end": round(end, 3)})
|
|
155
|
+
last = end
|
|
156
|
+
return out
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _load(device: str):
|
|
160
|
+
if device not in _model:
|
|
161
|
+
from transformers import Wav2Vec2CTCTokenizer, Wav2Vec2ForCTC
|
|
162
|
+
model = Wav2Vec2ForCTC.from_pretrained(MODEL).to(device).eval()
|
|
163
|
+
_model[device] = (model, Wav2Vec2CTCTokenizer.from_pretrained(MODEL).get_vocab())
|
|
164
|
+
return _model[device]
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def align(audio: np.ndarray, sr: int, tagged: str, device: str = "cpu", offset: float = 0.0) -> list[dict]:
|
|
168
|
+
"""The words of `tagged` (tags left out) with the times `audio` (mono floats at sr) says them."""
|
|
169
|
+
import librosa
|
|
170
|
+
import torch
|
|
171
|
+
model, vocab = _load(device)
|
|
172
|
+
words, seq = units(tagged, vocab)
|
|
173
|
+
if not words:
|
|
174
|
+
return []
|
|
175
|
+
x = librosa.resample(np.asarray(audio, np.float32), orig_sr=sr, target_sr=SR) if sr != SR else audio
|
|
176
|
+
x = (x - x.mean()) / np.sqrt(x.var() + 1e-7)
|
|
177
|
+
with torch.inference_mode():
|
|
178
|
+
logits = model(torch.from_numpy(x.astype(np.float32))[None].to(device)).logits[0]
|
|
179
|
+
logp = torch.log_softmax(logits.float(), dim=-1).cpu().numpy()
|
|
180
|
+
try:
|
|
181
|
+
return times(ctc_path(logp, seq, vocab["<pad>"]), seq, words, offset)
|
|
182
|
+
except ValueError: # too short to hold every letter: no honest times
|
|
183
|
+
return []
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: kinescript-chatterbox
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Chatterbox voices for kine voice: laughs and sighs from tags, cloned voices, and the time of each word
|
|
5
|
+
Author: Dr. Know How
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/drknowhow/kinescript/tree/main/plugins/kinescript-chatterbox
|
|
8
|
+
Project-URL: Documentation, https://drknowhow.github.io/kinescript/
|
|
9
|
+
Project-URL: Source, https://github.com/drknowhow/kinescript/tree/main/plugins/kinescript-chatterbox
|
|
10
|
+
Project-URL: Issues, https://github.com/drknowhow/kinescript/issues
|
|
11
|
+
Project-URL: Changelog, https://github.com/drknowhow/kinescript/blob/main/CHANGELOG.md
|
|
12
|
+
Keywords: kinescript,chatterbox,text to speech,tts,voice-over,forced alignment,voice cloning
|
|
13
|
+
Classifier: Development Status :: 4 - Beta
|
|
14
|
+
Classifier: Environment :: Console
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
24
|
+
Classifier: Topic :: Multimedia :: Sound/Audio :: Speech
|
|
25
|
+
Classifier: Topic :: Multimedia :: Video
|
|
26
|
+
Requires-Python: >=3.10
|
|
27
|
+
Description-Content-Type: text/markdown
|
|
28
|
+
License-File: LICENSE
|
|
29
|
+
Requires-Dist: chatterbox-tts>=0.1.7
|
|
30
|
+
Requires-Dist: setuptools<82
|
|
31
|
+
Dynamic: license-file
|
|
32
|
+
|
|
33
|
+
# kinescript-chatterbox
|
|
34
|
+
|
|
35
|
+
[Chatterbox](https://github.com/resemble-ai/chatterbox) voices for `kine voice`, from Resemble AI's open models. It
|
|
36
|
+
speaks the `text` of a scene's sounds into WAV files on your own machine and writes the time of each word, so
|
|
37
|
+
layers can start on a word and captions follow the voice. Chatterbox Turbo acts on tags such as `[laugh]` and
|
|
38
|
+
`[sigh]`, and either model can copy the voice of a short clip.
|
|
39
|
+
|
|
40
|
+
```json
|
|
41
|
+
{"id": "vo1", "role": "voice", "backend": "chatterbox",
|
|
42
|
+
"text": "Well [sigh] that render took a while. [[go]][laugh] But look at it."}
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
kine voice scene.json # writes voice/vo1.wav and voice/vo1.words.json
|
|
47
|
+
kine check scene.json
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
## Install
|
|
51
|
+
|
|
52
|
+
It needs Kinescript 1.3 or newer, and Chatterbox needs PyTorch. Into the same environment as Kinescript:
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
pip install kinescript-chatterbox
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
From a checkout of [Kinescript](https://github.com/drknowhow/kinescript), `pip install ./plugins/kinescript-chatterbox`
|
|
59
|
+
does the same.
|
|
60
|
+
|
|
61
|
+
On Python 3.10 to 3.13, Chatterbox pins PyTorch 2.6, whose CUDA builds stop before the RTX 50 series, so on such a
|
|
62
|
+
card it runs on the CPU. On Python 3.14 it takes PyTorch 2.9 or newer; install a CUDA build first to use the GPU:
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
pip install torch torchaudio --index-url https://download.pytorch.org/whl/cu128
|
|
66
|
+
pip install kinescript-chatterbox
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
To keep it out of Kinescript's environment, install it into another Python and point `KINE_VOICE_CMD_CHATTERBOX`
|
|
70
|
+
at it; `kine voice` sends it the lines as JSON on stdin and never through a shell:
|
|
71
|
+
|
|
72
|
+
```bash
|
|
73
|
+
python -m venv ~/chatterbox-env
|
|
74
|
+
~/chatterbox-env/bin/pip install kinescript-chatterbox
|
|
75
|
+
export KINE_VOICE_CMD_CHATTERBOX='["'$HOME'/chatterbox-env/bin/python", "-m", "kinescript_chatterbox"]'
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
The first run downloads Chatterbox Turbo (4.0 GB) and wav2vec2 (380 MB) from Hugging Face. The standard model is a
|
|
79
|
+
separate download, made the first time a sound asks for it.
|
|
80
|
+
|
|
81
|
+
## Tags
|
|
82
|
+
|
|
83
|
+
Turbo (in chatterbox-tts 0.1.7) knows 19 tags:
|
|
84
|
+
|
|
85
|
+
- sounds: `[laugh]`, `[chuckle]`, `[sigh]`, `[gasp]`, `[groan]`, `[cough]`, `[clear throat]`, `[sniff]`, `[shush]`
|
|
86
|
+
- delivery: `[whispering]`, `[crying]`, `[angry]`, `[happy]`, `[sarcastic]`, `[surprised]`, `[fear]`, `[dramatic]`,
|
|
87
|
+
`[narration]`, `[advertisement]`
|
|
88
|
+
|
|
89
|
+
A tag written the way ElevenLabs writes it, such as `[laughs]`, `[whispers]` or `[clears throat]`, is read as
|
|
90
|
+
Turbo's `[laugh]`, `[whispering]` or `[clear throat]`, so one scene works with both. A tag Turbo does not know is left
|
|
91
|
+
out, so it is not read aloud, and listed under `ignored` in what `kine voice` prints.
|
|
92
|
+
|
|
93
|
+
## Voices and settings
|
|
94
|
+
|
|
95
|
+
`speaker` is a WAV clip of 5 s or more, relative to the scene file, whose voice Chatterbox copies. Without one it
|
|
96
|
+
speaks in the model's own voice. `rate` is not supported; Chatterbox speaks at one speed, and `kine voice` lists a
|
|
97
|
+
rate other than 1 under `ignored`.
|
|
98
|
+
|
|
99
|
+
`options`:
|
|
100
|
+
|
|
101
|
+
- `model`: `turbo` (the default) acts on tags. `standard` is the larger original model, which has no tags but takes
|
|
102
|
+
`exaggeration` (0.5 by default; higher is more emotional) and `cfg_weight` (lower is slower and more deliberate).
|
|
103
|
+
- `temperature`, `top_p`, `top_k` (Turbo), `min_p` (standard), `repetition_penalty`: sampling settings.
|
|
104
|
+
- `seed`: 0 by default. The same text, voice and seed speak the same way again on the same machine.
|
|
105
|
+
|
|
106
|
+
Chatterbox puts Resemble AI's inaudible Perth watermark in everything it speaks.
|
|
107
|
+
|
|
108
|
+
## Word times
|
|
109
|
+
|
|
110
|
+
Chatterbox gives no word times, so the plugin finds them: wav2vec2 (`facebook/wav2vec2-base-960h`) hears the letters
|
|
111
|
+
in each 20 ms of the sound, and a forced alignment fits the text's letters to them. Numbers are aligned as they are
|
|
112
|
+
read ("250" as two hundred fifty, "2026" as twenty twenty six, "21st" as twenty first). A tag is matched to whatever
|
|
113
|
+
sound fills its place, so a laugh does not stretch the words around it. Word times come for English only.
|
|
114
|
+
|
|
115
|
+
## Device
|
|
116
|
+
|
|
117
|
+
It runs on the GPU when the installed PyTorch was built for it, and on the CPU otherwise. `CHATTERBOX_DEVICE=cpu`
|
|
118
|
+
or `CHATTERBOX_DEVICE=cuda` picks one.
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
kinescript_chatterbox/__init__.py,sha256=UMfSDlA1tQu6iq5zSG7hUEE5LROwC1jMupveTPqrfX0,6496
|
|
2
|
+
kinescript_chatterbox/__main__.py,sha256=WEo8d9L-lNb5m2yE1alz_U5t5C-bmofGtcDBTbSQ-7I,482
|
|
3
|
+
kinescript_chatterbox/align.py,sha256=pGsloxv0-53h8yHXMFl29FD1IzGgYpfflJutrR_05zo,8318
|
|
4
|
+
kinescript_chatterbox-0.1.0.dist-info/licenses/LICENSE,sha256=1dTEpVbx7pmCEJy776QUX7yDNLQKZ2csOgwWSqXa_o8,1260
|
|
5
|
+
kinescript_chatterbox-0.1.0.dist-info/METADATA,sha256=gn4E0p33vU6rwVIfjzycUgR29YlBDewzddNF58XOQ4Q,5557
|
|
6
|
+
kinescript_chatterbox-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
7
|
+
kinescript_chatterbox-0.1.0.dist-info/entry_points.txt,sha256=Si4KREZ7aWeTxkTFuSNMOVlBYhOBPaGov9MA8OB13xU,60
|
|
8
|
+
kinescript_chatterbox-0.1.0.dist-info/top_level.txt,sha256=aFObPDSuuZgMBGjBVbzJLP90s2Br0DE0ScELlt9vDAQ,22
|
|
9
|
+
kinescript_chatterbox-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Dr. Know How
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
22
|
+
|
|
23
|
+
The fonts in kinescript/fonts/ (Space Grotesk, IBM Plex Mono) are not covered by
|
|
24
|
+
this license. They are distributed under the SIL Open Font License 1.1; see the
|
|
25
|
+
-OFL.txt files next to them.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
kinescript_chatterbox
|