kinescript-chatterbox 0.1.0__tar.gz

kinescript_chatterbox-0.1.0/LICENSE

@@ -0,0 +1,25 @@
+MIT License
+
+Copyright (c) 2026 Dr. Know How
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+The fonts in kinescript/fonts/ (Space Grotesk, IBM Plex Mono) are not covered by
+this license. They are distributed under the SIL Open Font License 1.1; see the
+-OFL.txt files next to them.

kinescript_chatterbox-0.1.0/PKG-INFO

@@ -0,0 +1,118 @@
+Metadata-Version: 2.4
+Name: kinescript-chatterbox
+Version: 0.1.0
+Summary: Chatterbox voices for kine voice: laughs and sighs from tags, cloned voices, and the time of each word
+Author: Dr. Know How
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/drknowhow/kinescript/tree/main/plugins/kinescript-chatterbox
+Project-URL: Documentation, https://drknowhow.github.io/kinescript/
+Project-URL: Source, https://github.com/drknowhow/kinescript/tree/main/plugins/kinescript-chatterbox
+Project-URL: Issues, https://github.com/drknowhow/kinescript/issues
+Project-URL: Changelog, https://github.com/drknowhow/kinescript/blob/main/CHANGELOG.md
+Keywords: kinescript,chatterbox,text to speech,tts,voice-over,forced alignment,voice cloning
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Multimedia :: Sound/Audio :: Speech
+Classifier: Topic :: Multimedia :: Video
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: chatterbox-tts>=0.1.7
+Requires-Dist: setuptools<82
+Dynamic: license-file
+
+# kinescript-chatterbox
+
+[Chatterbox](https://github.com/resemble-ai/chatterbox) voices for `kine voice`, from Resemble AI's open models. It
+speaks the `text` of a scene's sounds into WAV files on your own machine and writes the time of each word, so
+layers can start on a word and captions follow the voice. Chatterbox Turbo acts on tags such as `[laugh]` and
+`[sigh]`, and either model can copy the voice of a short clip.
+
+```json
+{"id": "vo1", "role": "voice", "backend": "chatterbox",
+ "text": "Well [sigh] that render took a while. [[go]][laugh] But look at it."}
+```
+
+```bash
+kine voice scene.json     # writes voice/vo1.wav and voice/vo1.words.json
+kine check scene.json
+```
+
+## Install
+
+It needs Kinescript 1.3 or newer, and Chatterbox needs PyTorch. Into the same environment as Kinescript:
+
+```bash
+pip install kinescript-chatterbox
+```
+
+From a checkout of [Kinescript](https://github.com/drknowhow/kinescript), `pip install ./plugins/kinescript-chatterbox`
+does the same.
+
+On Python 3.10 to 3.13, Chatterbox pins PyTorch 2.6, whose CUDA builds stop before the RTX 50 series, so on such a
+card it runs on the CPU. On Python 3.14 it takes PyTorch 2.9 or newer; install a CUDA build first to use the GPU:
+
+```bash
+pip install torch torchaudio --index-url https://download.pytorch.org/whl/cu128
+pip install kinescript-chatterbox
+```
+
+To keep it out of Kinescript's environment, install it into another Python and point `KINE_VOICE_CMD_CHATTERBOX`
+at it; `kine voice` sends it the lines as JSON on stdin and never through a shell:
+
+```bash
+python -m venv ~/chatterbox-env
+~/chatterbox-env/bin/pip install kinescript-chatterbox
+export KINE_VOICE_CMD_CHATTERBOX='["'$HOME'/chatterbox-env/bin/python", "-m", "kinescript_chatterbox"]'
+```
+
+The first run downloads Chatterbox Turbo (4.0 GB) and wav2vec2 (380 MB) from Hugging Face. The standard model is a
+separate download, made the first time a sound asks for it.
+
+## Tags
+
+Turbo (in chatterbox-tts 0.1.7) knows 19 tags:
+
+- sounds: `[laugh]`, `[chuckle]`, `[sigh]`, `[gasp]`, `[groan]`, `[cough]`, `[clear throat]`, `[sniff]`, `[shush]`
+- delivery: `[whispering]`, `[crying]`, `[angry]`, `[happy]`, `[sarcastic]`, `[surprised]`, `[fear]`, `[dramatic]`,
+  `[narration]`, `[advertisement]`
+
+A tag written the way ElevenLabs writes it, such as `[laughs]`, `[whispers]` or `[clears throat]`, is read as
+Turbo's `[laugh]`, `[whispering]` or `[clear throat]`, so one scene works with both. A tag Turbo does not know is left
+out, so it is not read aloud, and listed under `ignored` in what `kine voice` prints.
+
+## Voices and settings
+
+`speaker` is a WAV clip of 5 s or more, relative to the scene file, whose voice Chatterbox copies. Without one it
+speaks in the model's own voice. `rate` is not supported; Chatterbox speaks at one speed, and `kine voice` lists a
+rate other than 1 under `ignored`.
+
+`options`:
+
+- `model`: `turbo` (the default) acts on tags. `standard` is the larger original model, which has no tags but takes
+  `exaggeration` (0.5 by default; higher is more emotional) and `cfg_weight` (lower is slower and more deliberate).
+- `temperature`, `top_p`, `top_k` (Turbo), `min_p` (standard), `repetition_penalty`: sampling settings.
+- `seed`: 0 by default. The same text, voice and seed speak the same way again on the same machine.
+
+Chatterbox puts Resemble AI's inaudible Perth watermark in everything it speaks.
+
+## Word times
+
+Chatterbox gives no word times, so the plugin finds them: wav2vec2 (`facebook/wav2vec2-base-960h`) hears the letters
+in each 20 ms of the sound, and a forced alignment fits the text's letters to them. Numbers are aligned as they are
+read ("250" as two hundred fifty, "2026" as twenty twenty six, "21st" as twenty first). A tag is matched to whatever
+sound fills its place, so a laugh does not stretch the words around it. Word times come for English only.
+
+## Device
+
+It runs on the GPU when the installed PyTorch was built for it, and on the CPU otherwise. `CHATTERBOX_DEVICE=cpu`
+or `CHATTERBOX_DEVICE=cuda` picks one.

kinescript_chatterbox-0.1.0/README.md

@@ -0,0 +1,86 @@
+# kinescript-chatterbox
+
+[Chatterbox](https://github.com/resemble-ai/chatterbox) voices for `kine voice`, from Resemble AI's open models. It
+speaks the `text` of a scene's sounds into WAV files on your own machine and writes the time of each word, so
+layers can start on a word and captions follow the voice. Chatterbox Turbo acts on tags such as `[laugh]` and
+`[sigh]`, and either model can copy the voice of a short clip.
+
+```json
+{"id": "vo1", "role": "voice", "backend": "chatterbox",
+ "text": "Well [sigh] that render took a while. [[go]][laugh] But look at it."}
+```
+
+```bash
+kine voice scene.json     # writes voice/vo1.wav and voice/vo1.words.json
+kine check scene.json
+```
+
+## Install
+
+It needs Kinescript 1.3 or newer, and Chatterbox needs PyTorch. Into the same environment as Kinescript:
+
+```bash
+pip install kinescript-chatterbox
+```
+
+From a checkout of [Kinescript](https://github.com/drknowhow/kinescript), `pip install ./plugins/kinescript-chatterbox`
+does the same.
+
+On Python 3.10 to 3.13, Chatterbox pins PyTorch 2.6, whose CUDA builds stop before the RTX 50 series, so on such a
+card it runs on the CPU. On Python 3.14 it takes PyTorch 2.9 or newer; install a CUDA build first to use the GPU:
+
+```bash
+pip install torch torchaudio --index-url https://download.pytorch.org/whl/cu128
+pip install kinescript-chatterbox
+```
+
+To keep it out of Kinescript's environment, install it into another Python and point `KINE_VOICE_CMD_CHATTERBOX`
+at it; `kine voice` sends it the lines as JSON on stdin and never through a shell:
+
+```bash
+python -m venv ~/chatterbox-env
+~/chatterbox-env/bin/pip install kinescript-chatterbox
+export KINE_VOICE_CMD_CHATTERBOX='["'$HOME'/chatterbox-env/bin/python", "-m", "kinescript_chatterbox"]'
+```
+
+The first run downloads Chatterbox Turbo (4.0 GB) and wav2vec2 (380 MB) from Hugging Face. The standard model is a
+separate download, made the first time a sound asks for it.
+
+## Tags
+
+Turbo (in chatterbox-tts 0.1.7) knows 19 tags:
+
+- sounds: `[laugh]`, `[chuckle]`, `[sigh]`, `[gasp]`, `[groan]`, `[cough]`, `[clear throat]`, `[sniff]`, `[shush]`
+- delivery: `[whispering]`, `[crying]`, `[angry]`, `[happy]`, `[sarcastic]`, `[surprised]`, `[fear]`, `[dramatic]`,
+  `[narration]`, `[advertisement]`
+
+A tag written the way ElevenLabs writes it, such as `[laughs]`, `[whispers]` or `[clears throat]`, is read as
+Turbo's `[laugh]`, `[whispering]` or `[clear throat]`, so one scene works with both. A tag Turbo does not know is left
+out, so it is not read aloud, and listed under `ignored` in what `kine voice` prints.
+
+## Voices and settings
+
+`speaker` is a WAV clip of 5 s or more, relative to the scene file, whose voice Chatterbox copies. Without one it
+speaks in the model's own voice. `rate` is not supported; Chatterbox speaks at one speed, and `kine voice` lists a
+rate other than 1 under `ignored`.
+
+`options`:
+
+- `model`: `turbo` (the default) acts on tags. `standard` is the larger original model, which has no tags but takes
+  `exaggeration` (0.5 by default; higher is more emotional) and `cfg_weight` (lower is slower and more deliberate).
+- `temperature`, `top_p`, `top_k` (Turbo), `min_p` (standard), `repetition_penalty`: sampling settings.
+- `seed`: 0 by default. The same text, voice and seed speak the same way again on the same machine.
+
+Chatterbox puts Resemble AI's inaudible Perth watermark in everything it speaks.
+
+## Word times
+
+Chatterbox gives no word times, so the plugin finds them: wav2vec2 (`facebook/wav2vec2-base-960h`) hears the letters
+in each 20 ms of the sound, and a forced alignment fits the text's letters to them. Numbers are aligned as they are
+read ("250" as two hundred fifty, "2026" as twenty twenty six, "21st" as twenty first). A tag is matched to whatever
+sound fills its place, so a laugh does not stretch the words around it. Word times come for English only.
+
+## Device
+
+It runs on the GPU when the installed PyTorch was built for it, and on the CPU otherwise. `CHATTERBOX_DEVICE=cpu`
+or `CHATTERBOX_DEVICE=cuda` picks one.

kinescript_chatterbox-0.1.0/kinescript_chatterbox/__init__.py

@@ -0,0 +1,143 @@
+"""Chatterbox voices for `kine voice`: Resemble AI's open models speak a sound's text, Turbo with tags such as
+[laugh], and a wav2vec2 alignment times each word (Chatterbox gives no word times of its own).
+
+kine voice calls synth(req) through the "kinescript.voice" entry point, or this package runs as
+`python -m kinescript_chatterbox` for KINE_VOICE_CMD_CHATTERBOX, when Chatterbox lives in another Python.
+"""
+from __future__ import annotations
+
+import os
+import re
+import wave
+from importlib.metadata import PackageNotFoundError, version
+from pathlib import Path
+
+from .align import TAG, align
+
+MODELS = ("turbo", "standard")  # turbo acts on tags; standard takes exaggeration and cfg_weight instead
+TURBO_TAGS: set[str] = set()  # filled from Turbo's tokenizer when it loads
+OPTIONS = {"turbo": {"temperature", "top_p", "top_k", "repetition_penalty"},
+           "standard": {"exaggeration", "cfg_weight", "temperature", "min_p", "top_p", "repetition_penalty"}}
+CHUNK = 300  # characters per generate call: Chatterbox reads long text better a few sentences at a time
+_models: dict = {}
+
+
+def _device() -> str:
+    """cuda when this torch was built for the GPU, else cpu. CHATTERBOX_DEVICE picks one."""
+    if os.environ.get("CHATTERBOX_DEVICE"):
+        return os.environ["CHATTERBOX_DEVICE"]
+    import torch
+    if torch.cuda.is_available():
+        major, minor = torch.cuda.get_device_capability()
+        if f"sm_{major}{minor}" in torch.cuda.get_arch_list():
+            return "cuda"
+    return "cpu"
+
+
+def _model(kind: str):
+    """The model and its own voice, loaded once per process."""
+    if kind not in _models:
+        if kind == "turbo":
+            from chatterbox.tts_turbo import ChatterboxTurboTTS as Model
+        else:
+            from chatterbox.tts import ChatterboxTTS as Model
+        m = Model.from_pretrained(device=_device())
+        if kind == "turbo":
+            TURBO_TAGS.update(t for t in m.tokenizer.get_added_vocab() if TAG.fullmatch(t))
+            # chatterbox-tts 0.1.7 scales the clip by a NumPy float64, which makes float64 under NumPy 2 (Python
+            # 3.13 and later), and its tokenizer then fails on a clip to copy; keep the clip float32
+            norm = m.norm_loudness
+            m.norm_loudness = lambda wav, sr, **kw: norm(wav, sr, **kw).astype("float32", copy=False)
+        _models[kind] = (m, m.conds)
+    return _models[kind]
+
+
+def _forms(word: str) -> set[str]:
+    return {word} | {word[:-len(s)] for s in ("ing", "es", "s") if word.endswith(s) and len(word) - len(s) >= 3}
+
+
+def turbo_tag(tag: str) -> str | None:
+    """The Turbo tag for one written as another backend writes it ([laughs] -> [laugh], [whispers] ->
+    [whispering], [clears throat] -> [clear throat]), or None."""
+    t = tag.lower()
+    if t in TURBO_TAGS:
+        return t
+    words = t[1:-1].split()
+    for known in sorted(TURBO_TAGS):
+        kw = known[1:-1].split()
+        if len(kw) == len(words) and all(_forms(a) & _forms(b) for a, b in zip(words, kw)):
+            return known
+    return None
+
+
+def chunks(text: str, size: int = CHUNK) -> list[str]:
+    """The text in pieces of whole sentences, each at most about `size` characters."""
+    out = []
+    for sentence in re.split(r"(?<=[.!?])\s+", text.strip()):
+        if out and len(out[-1]) + 1 + len(sentence) <= size:
+            out[-1] += " " + sentence
+        elif sentence:
+            out.append(sentence)
+    return out
+
+
+def _version(kind: str) -> str:
+    def v(name):
+        try:
+            return version(name)
+        except PackageNotFoundError:
+            return "?"
+    return f"chatterbox-tts {v('chatterbox-tts')} {kind}, kinescript-chatterbox {v('kinescript-chatterbox')}"
+
+
+def synth(req: dict) -> dict:
+    """Speak req["tagged"] (Turbo) or req["text"] (standard) into the WAV req["out"], in the voice of the clip
+    req["speaker"] names (5 s or more, relative to req["base"]) or the model's own voice, and time each word."""
+    import numpy as np
+    import torch
+    opts = dict(req.get("options") or {})
+    kind = str(opts.pop("model", "turbo"))
+    if kind not in MODELS:
+        raise ValueError(f"Chatterbox model '{kind}': use one of {', '.join(MODELS)}")
+    seed = int(opts.pop("seed", 0))
+    m, own = _model(kind)
+    ignored = [f"options.{k}" for k in opts if k not in OPTIONS[kind]]
+    if float(req.get("rate") or 1) != 1:
+        ignored.append(f"rate {req['rate']}: Chatterbox speaks at one speed")
+    text = (req.get("tagged") or req["text"]) if kind == "turbo" else req["text"]
+    if kind == "turbo":  # a tag Turbo does not know would be read out as a word
+        ignored += [t for t in TAG.findall(text) if not turbo_tag(t)]
+        text = re.sub(r"\s+", " ", TAG.sub(lambda t: turbo_tag(t.group(0)) or "", text)).strip()
+    speaker = req.get("speaker")
+    if speaker:
+        ref = Path(speaker)
+        ref = ref if ref.is_absolute() else Path(req.get("base") or ".") / ref
+        if not ref.exists():
+            raise FileNotFoundError(f"Chatterbox clones the voice of a clip of 5 s or more; {ref} does not exist")
+        m.prepare_conditionals(str(ref), **({"exaggeration": opts["exaggeration"]} if "exaggeration" in opts else {}))
+    else:
+        m.conds = own
+    kw = {k: v for k, v in opts.items() if k in OPTIONS[kind]}
+    audio, words, untimed, at = [], [], [], 0
+    for k, piece in enumerate(chunks(text)):
+        torch.manual_seed(seed + k)  # the same text says the same thing on the next run
+        wav = m.generate(piece, **kw).squeeze(0).numpy().astype(np.float32)
+        got = align(wav, m.sr, piece, device=_device(), offset=at / m.sr)
+        said = " ".join(TAG.sub(" ", piece).split())
+        if not got and said:  # the sound is too short to hold the text: it was cut off
+            untimed.append(said)
+        words += got
+        audio.append(wav)
+        at += len(wav)
+    out = np.concatenate(audio) if audio else np.zeros(0, np.float32)
+    with wave.open(req["out"], "wb") as f:
+        f.setnchannels(1)
+        f.setsampwidth(2)
+        f.setframerate(m.sr)
+        f.writeframes((np.clip(out, -1, 1) * 32767).astype("<i2").tobytes())
+    got = {"words": words, "speaker": speaker or f"{kind} built-in", "version": _version(kind),
+           "seconds": round(at / m.sr, 3), "tags": kind == "turbo", "ignored": ignored}
+    if untimed:
+        got["note"] = ("too short for its text, so these words have no times (try another seed): "
+                       + " / ".join(untimed))
+    return got

kinescript_chatterbox-0.1.0/kinescript_chatterbox/__main__.py

@@ -0,0 +1,14 @@
+"""For KINE_VOICE_CMD_CHATTERBOX: a JSON list of requests on stdin, the list of results on stdout.
+
+    KINE_VOICE_CMD_CHATTERBOX='["/path/to/python", "-m", "kinescript_chatterbox"]' kine voice scene.json
+"""
+import contextlib
+import json
+import sys
+
+from . import synth
+
+reqs = json.loads(sys.stdin.read())
+with contextlib.redirect_stdout(sys.stderr):  # Chatterbox's own messages stay out of the answer
+    results = [synth(r) for r in reqs]
+sys.stdout.write(json.dumps(results))

kinescript_chatterbox-0.1.0/kinescript_chatterbox/align.py

@@ -0,0 +1,183 @@
+"""Word times for speech whose words are known: wav2vec2 gives each 20 ms of sound the odds of every letter, and a
+CTC forced alignment finds the path through the text's letters that fits those odds best.
+
+A number is aligned as it is read ("250" as TWO HUNDRED FIFTY, "2026" as TWENTY TWENTY SIX), so its words claim
+their own sound. A tag ([laugh]), and a word with nothing left to align, become a wildcard that any sound can
+fill, so a laugh does not stretch the words around it.
+"""
+from __future__ import annotations
+
+import re
+
+import numpy as np
+
+MODEL = "facebook/wav2vec2-base-960h"
+SR = 16000
+HOP = 320  # input samples per output frame: 20 ms
+TAG = re.compile(r"(?<!\[)\[(?!\[)[^\[\]()]+\](?![\](])")  # as kinescript.voice.TAG
+WILD, MAYBE = -1, -2  # a wildcard that must take a frame (a word with no letters), and one that may take none (a tag)
+PENALTY = 1.0  # what a wildcard frame costs (nats), so the text's own letters win wherever they fit
+_model: dict = {}
+
+
+ONES = ("zero one two three four five six seven eight nine ten eleven twelve thirteen fourteen fifteen sixteen "
+        "seventeen eighteen nineteen").split()
+TENS = "_ _ twenty thirty forty fifty sixty seventy eighty ninety".split()
+SYMBOLS = {"%": "percent", "&": "and", "+": "plus", "$": "dollars"}
+ORDINALS = {"one": "first", "two": "second", "three": "third", "five": "fifth", "eight": "eighth", "nine": "ninth",
+            "twelve": "twelfth"}
+
+
+def letters(word: str) -> str:
+    return re.sub(r"[^A-Z']", "", word.upper().replace("’", "'"))
+
+
+def _say(n: int) -> list[str]:
+    if n < 20:
+        return [ONES[n]]
+    if n < 100:
+        return [TENS[n // 10]] + ([ONES[n % 10]] if n % 10 else [])
+    if n < 1000:
+        return [ONES[n // 100], "hundred"] + (_say(n % 100) if n % 100 else [])
+    size, name = next((s, w) for s, w in ((10 ** 9, "billion"), (10 ** 6, "million"), (1000, "thousand")) if n >= s)
+    return _say(n // size) + [name] + (_say(n % size) if n % size else [])
+
+
+def number(digits: str) -> list[str]:
+    """How a number is read: 250 two hundred fifty, 3.5 three point five, 1.2.0 one point two point zero, 2026 (a
+    year) twenty twenty six."""
+    whole, *rest = digits.replace(",", "").split(".")
+    n = int(whole)
+    if not rest and len(whole) == 4 and 1100 <= n <= 2099 and n % 100 and not 2000 <= n <= 2009:
+        return _say(n // 100) + (["oh"] if n % 100 < 10 else []) + _say(n % 100)
+    if len(rest) == 1:
+        return _say(n) + ["point"] + [ONES[int(d)] for d in rest[0]]
+    return _say(n) + [w for part in rest for w in ["point"] + _say(int(part))]
+
+
+def ordinal(word: str) -> str:
+    """first, twentieth, hundredth: the ordinal of a number word."""
+    if word in ORDINALS:
+        return ORDINALS[word]
+    return word[:-1] + "ieth" if word.endswith("y") else word + "th"
+
+
+def spoken(word: str) -> list[str]:
+    """The words a written word is read as, in capitals: "250" -> [TWO, HUNDRED, FIFTY], "Render." -> [RENDER]."""
+    if not re.search(r"\d", word):
+        return [letters(word)]
+    out = []
+    for part in re.findall(r"\d+(?:[.,]\d+)*|[A-Za-z']+|[%&+$]", word):
+        if part.lower() in ("st", "nd", "rd", "th") and out and out[-1] in ONES + TENS + ["hundred", "thousand"]:
+            out[-1] = ordinal(out[-1])  # 21st: twenty first
+        else:
+            out += number(part) if part[0].isdigit() else [SYMBOLS.get(part, part)]
+    if "$" in word:  # $5: five dollars
+        out = [w for w in out if w != "dollars"] + ["dollars"]
+    return [letters(w) for w in out]
+
+
+def units(tagged: str, vocab: dict) -> tuple[list[str], list[tuple[int, int | None]]]:
+    """The words of the text as written, and what is aligned: (vocabulary id, or WILD/MAYBE; the word's index, or
+    None for a tag or a word gap)."""
+    words, seq, gap = [], [], vocab["|"]
+    for piece in re.split(f"({TAG.pattern})", tagged):
+        if TAG.fullmatch(piece):
+            seq.append((MAYBE, None))
+            continue
+        for w in piece.split():
+            if not re.search(r"\w", w):  # punctuation on its own ("—"): nothing is said
+                continue
+            if seq and seq[-1][0] != MAYBE:
+                seq.append((gap, None))
+            ids = []
+            for k, part in enumerate(spoken(w)):
+                ids += ([gap] if k else []) + [vocab[c] for c in part if c in vocab]
+            # nothing left to align: any sound
+            seq += [(i, len(words)) for i in ids] if set(ids) - {gap} else [(WILD, len(words))]
+            words.append(w)
+    return words, seq
+
+
+def ctc_path(logp: np.ndarray, seq: list[tuple[int, int | None]], blank: int) -> np.ndarray:
+    """The best CTC path through seq for frames of log odds logp (frames x vocabulary): the index into seq that
+    each frame says, or -1 for a blank."""
+    T = logp.shape[0]
+    labels = [blank]
+    for lab, _ in seq:
+        labels += [lab, blank]
+    S = len(labels)
+    speech = np.delete(logp, blank, axis=1).max(axis=1) - PENALTY  # a wildcard scores as the likeliest letter
+    lab = np.array(labels)
+    emit = np.where(lab[None, :] < 0, speech[:, None], logp[:, np.clip(lab, 0, None)])
+    # a step of 2 skips the blank between two different letters; around a tag it skips the tag itself
+    skip = np.zeros(S, bool)
+    for s in range(2, S):
+        if lab[s] != blank and lab[s] != lab[s - 2]:
+            skip[s] = True
+        if lab[s] == blank and lab[s - 1] == MAYBE:
+            skip[s] = True
+    neg = -np.inf
+    score = np.full(S, neg)
+    score[0], score[1] = emit[0, 0], emit[0, 1]
+    if S > 3 and lab[1] == MAYBE:
+        score[2] = emit[0, 2]
+    back = np.zeros((T, S), np.int8)
+    for t in range(1, T):
+        one = np.concatenate([[neg], score[:-1]])
+        two = np.where(skip, np.concatenate([[neg, neg], score[:-2]]), neg)
+        stack = np.stack([score, one, two])
+        back[t] = stack.argmax(axis=0)
+        score = stack.max(axis=0) + emit[t]
+    ends = [S - 1, S - 2] + ([S - 3, S - 4] if S > 4 and lab[S - 2] == MAYBE else [])
+    s = max(ends, key=lambda e: score[e])
+    if not np.isfinite(score[s]):
+        raise ValueError("the sound is too short for its text")
+    path = np.empty(T, int)
+    for t in range(T - 1, -1, -1):
+        path[t] = s
+        s -= int(back[t, s])  # int8 minus a Python int stays int8, which overflows past 127 states
+    return np.where(path % 2 == 1, (path - 1) // 2, -1)
+
+
+def times(path: np.ndarray, seq: list, words: list[str], offset: float = 0.0) -> list[dict]:
+    """[{"word", "start", "end"}] from the frames each word's letters took; a word that took none (the alignment
+    could not place it) takes the end of the word before."""
+    owner = np.array([seq[i][1] if i >= 0 and seq[i][1] is not None else -1 for i in path])
+    out, last = [], offset
+    for k, w in enumerate(words):
+        frames = np.flatnonzero(owner == k)
+        if len(frames):
+            start, end = offset + frames[0] * HOP / SR, offset + (frames[-1] + 1) * HOP / SR
+        else:
+            start = end = last
+        out.append({"word": w, "start": round(start, 3), "end": round(end, 3)})
+        last = end
+    return out
+
+
+def _load(device: str):
+    if device not in _model:
+        from transformers import Wav2Vec2CTCTokenizer, Wav2Vec2ForCTC
+        model = Wav2Vec2ForCTC.from_pretrained(MODEL).to(device).eval()
+        _model[device] = (model, Wav2Vec2CTCTokenizer.from_pretrained(MODEL).get_vocab())
+    return _model[device]
+
+
+def align(audio: np.ndarray, sr: int, tagged: str, device: str = "cpu", offset: float = 0.0) -> list[dict]:
+    """The words of `tagged` (tags left out) with the times `audio` (mono floats at sr) says them."""
+    import librosa
+    import torch
+    model, vocab = _load(device)
+    words, seq = units(tagged, vocab)
+    if not words:
+        return []
+    x = librosa.resample(np.asarray(audio, np.float32), orig_sr=sr, target_sr=SR) if sr != SR else audio
+    x = (x - x.mean()) / np.sqrt(x.var() + 1e-7)
+    with torch.inference_mode():
+        logits = model(torch.from_numpy(x.astype(np.float32))[None].to(device)).logits[0]
+    logp = torch.log_softmax(logits.float(), dim=-1).cpu().numpy()
+    try:
+        return times(ctc_path(logp, seq, vocab["<pad>"]), seq, words, offset)
+    except ValueError:  # too short to hold every letter: no honest times
+        return []

kinescript_chatterbox-0.1.0/kinescript_chatterbox.egg-info/PKG-INFO

@@ -0,0 +1,118 @@
+Metadata-Version: 2.4
+Name: kinescript-chatterbox
+Version: 0.1.0
+Summary: Chatterbox voices for kine voice: laughs and sighs from tags, cloned voices, and the time of each word
+Author: Dr. Know How
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/drknowhow/kinescript/tree/main/plugins/kinescript-chatterbox
+Project-URL: Documentation, https://drknowhow.github.io/kinescript/
+Project-URL: Source, https://github.com/drknowhow/kinescript/tree/main/plugins/kinescript-chatterbox
+Project-URL: Issues, https://github.com/drknowhow/kinescript/issues
+Project-URL: Changelog, https://github.com/drknowhow/kinescript/blob/main/CHANGELOG.md
+Keywords: kinescript,chatterbox,text to speech,tts,voice-over,forced alignment,voice cloning
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Multimedia :: Sound/Audio :: Speech
+Classifier: Topic :: Multimedia :: Video
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: chatterbox-tts>=0.1.7
+Requires-Dist: setuptools<82
+Dynamic: license-file
+
+# kinescript-chatterbox
+
+[Chatterbox](https://github.com/resemble-ai/chatterbox) voices for `kine voice`, from Resemble AI's open models. It
+speaks the `text` of a scene's sounds into WAV files on your own machine and writes the time of each word, so
+layers can start on a word and captions follow the voice. Chatterbox Turbo acts on tags such as `[laugh]` and
+`[sigh]`, and either model can copy the voice of a short clip.
+
+```json
+{"id": "vo1", "role": "voice", "backend": "chatterbox",
+ "text": "Well [sigh] that render took a while. [[go]][laugh] But look at it."}
+```
+
+```bash
+kine voice scene.json     # writes voice/vo1.wav and voice/vo1.words.json
+kine check scene.json
+```
+
+## Install
+
+It needs Kinescript 1.3 or newer, and Chatterbox needs PyTorch. Into the same environment as Kinescript:
+
+```bash
+pip install kinescript-chatterbox
+```
+
+From a checkout of [Kinescript](https://github.com/drknowhow/kinescript), `pip install ./plugins/kinescript-chatterbox`
+does the same.
+
+On Python 3.10 to 3.13, Chatterbox pins PyTorch 2.6, whose CUDA builds stop before the RTX 50 series, so on such a
+card it runs on the CPU. On Python 3.14 it takes PyTorch 2.9 or newer; install a CUDA build first to use the GPU:
+
+```bash
+pip install torch torchaudio --index-url https://download.pytorch.org/whl/cu128
+pip install kinescript-chatterbox
+```
+
+To keep it out of Kinescript's environment, install it into another Python and point `KINE_VOICE_CMD_CHATTERBOX`
+at it; `kine voice` sends it the lines as JSON on stdin and never through a shell:
+
+```bash
+python -m venv ~/chatterbox-env
+~/chatterbox-env/bin/pip install kinescript-chatterbox
+export KINE_VOICE_CMD_CHATTERBOX='["'$HOME'/chatterbox-env/bin/python", "-m", "kinescript_chatterbox"]'
+```
+
+The first run downloads Chatterbox Turbo (4.0 GB) and wav2vec2 (380 MB) from Hugging Face. The standard model is a
+separate download, made the first time a sound asks for it.
+
+## Tags
+
+Turbo (in chatterbox-tts 0.1.7) knows 19 tags:
+
+- sounds: `[laugh]`, `[chuckle]`, `[sigh]`, `[gasp]`, `[groan]`, `[cough]`, `[clear throat]`, `[sniff]`, `[shush]`
+- delivery: `[whispering]`, `[crying]`, `[angry]`, `[happy]`, `[sarcastic]`, `[surprised]`, `[fear]`, `[dramatic]`,
+  `[narration]`, `[advertisement]`
+
+A tag written the way ElevenLabs writes it, such as `[laughs]`, `[whispers]` or `[clears throat]`, is read as
+Turbo's `[laugh]`, `[whispering]` or `[clear throat]`, so one scene works with both. A tag Turbo does not know is left
+out, so it is not read aloud, and listed under `ignored` in what `kine voice` prints.
+
+## Voices and settings
+
+`speaker` is a WAV clip of 5 s or more, relative to the scene file, whose voice Chatterbox copies. Without one it
+speaks in the model's own voice. `rate` is not supported; Chatterbox speaks at one speed, and `kine voice` lists a
+rate other than 1 under `ignored`.
+
+`options`:
+
+- `model`: `turbo` (the default) acts on tags. `standard` is the larger original model, which has no tags but takes
+  `exaggeration` (0.5 by default; higher is more emotional) and `cfg_weight` (lower is slower and more deliberate).
+- `temperature`, `top_p`, `top_k` (Turbo), `min_p` (standard), `repetition_penalty`: sampling settings.
+- `seed`: 0 by default. The same text, voice and seed speak the same way again on the same machine.
+
+Chatterbox puts Resemble AI's inaudible Perth watermark in everything it speaks.
+
+## Word times
+
+Chatterbox gives no word times, so the plugin finds them: wav2vec2 (`facebook/wav2vec2-base-960h`) hears the letters
+in each 20 ms of the sound, and a forced alignment fits the text's letters to them. Numbers are aligned as they are
+read ("250" as two hundred fifty, "2026" as twenty twenty six, "21st" as twenty first). A tag is matched to whatever
+sound fills its place, so a laugh does not stretch the words around it. Word times come for English only.
+
+## Device
+
+It runs on the GPU when the installed PyTorch was built for it, and on the CPU otherwise. `CHATTERBOX_DEVICE=cpu`
+or `CHATTERBOX_DEVICE=cuda` picks one.

kinescript_chatterbox-0.1.0/kinescript_chatterbox.egg-info/SOURCES.txt

@@ -0,0 +1,12 @@
+LICENSE
+README.md
+pyproject.toml
+kinescript_chatterbox/__init__.py
+kinescript_chatterbox/__main__.py
+kinescript_chatterbox/align.py
+kinescript_chatterbox.egg-info/PKG-INFO
+kinescript_chatterbox.egg-info/SOURCES.txt
+kinescript_chatterbox.egg-info/dependency_links.txt
+kinescript_chatterbox.egg-info/entry_points.txt
+kinescript_chatterbox.egg-info/requires.txt
+kinescript_chatterbox.egg-info/top_level.txt

kinescript_chatterbox-0.1.0/kinescript_chatterbox.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

kinescript_chatterbox-0.1.0/kinescript_chatterbox.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[kinescript.voice]
+chatterbox = kinescript_chatterbox:synth

kinescript_chatterbox-0.1.0/kinescript_chatterbox.egg-info/requires.txt

@@ -0,0 +1,2 @@
+chatterbox-tts>=0.1.7
+setuptools<82

kinescript_chatterbox-0.1.0/kinescript_chatterbox.egg-info/top_level.txt

@@ -0,0 +1 @@
+kinescript_chatterbox

kinescript_chatterbox-0.1.0/pyproject.toml

@@ -0,0 +1,46 @@
+[build-system]
+requires = ["setuptools>=77"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "kinescript-chatterbox"
+version = "0.1.0"
+description = "Chatterbox voices for kine voice: laughs and sighs from tags, cloned voices, and the time of each word"
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [{ name = "Dr. Know How" }]
+requires-python = ">=3.10"
+# Chatterbox's watermarker (resemble-perth) imports pkg_resources, which setuptools 82 removed.
+dependencies = ["chatterbox-tts>=0.1.7", "setuptools<82"]
+keywords = ["kinescript", "chatterbox", "text to speech", "tts", "voice-over", "forced alignment", "voice cloning"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Environment :: Console",
+    "Intended Audience :: Developers",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+    "Topic :: Multimedia :: Sound/Audio :: Speech",
+    "Topic :: Multimedia :: Video",
+]
+
+# Same rule as the main package: trusted publishing from drknowhow/kinescript marks a link verified only under
+# https://github.com/drknowhow/kinescript or https://drknowhow.github.io/kinescript.
+[project.urls]
+Homepage = "https://github.com/drknowhow/kinescript/tree/main/plugins/kinescript-chatterbox"
+Documentation = "https://drknowhow.github.io/kinescript/"
+Source = "https://github.com/drknowhow/kinescript/tree/main/plugins/kinescript-chatterbox"
+Issues = "https://github.com/drknowhow/kinescript/issues"
+Changelog = "https://github.com/drknowhow/kinescript/blob/main/CHANGELOG.md"
+
+[project.entry-points."kinescript.voice"]
+chatterbox = "kinescript_chatterbox:synth"
+
+[tool.setuptools]
+packages = ["kinescript_chatterbox"]

kinescript_chatterbox-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+