altatts 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- altatts/__init__.py +106 -0
- altatts/audio.py +324 -0
- altatts/benchmark.py +165 -0
- altatts/client.py +94 -0
- altatts/config.py +456 -0
- altatts/datasets/__init__.py +24 -0
- altatts/datasets/adapters.py +339 -0
- altatts/datasets/cli.py +77 -0
- altatts/datasets/combine.py +803 -0
- altatts/datasets/registry.py +120 -0
- altatts/datasets/spec.py +53 -0
- altatts/datasets/stats.py +241 -0
- altatts/deploy.py +235 -0
- altatts/env/__init__.py +2 -0
- altatts/env/checks.py +703 -0
- altatts/env/compat.py +194 -0
- altatts/env/compat_table.json +28 -0
- altatts/env/distributed.py +130 -0
- altatts/env/doctor.py +256 -0
- altatts/env/fix.py +465 -0
- altatts/env/wheels.py +301 -0
- altatts/export.py +186 -0
- altatts/inference/__init__.py +13 -0
- altatts/inference/checkpoint.py +115 -0
- altatts/inference/evaluate.py +316 -0
- altatts/inference/streaming.py +174 -0
- altatts/inference/synthesize.py +148 -0
- altatts/inference/synthesizer.py +431 -0
- altatts/inference/voices.py +120 -0
- altatts/integrations/__init__.py +21 -0
- altatts/integrations/callcenter.py +220 -0
- altatts/languages/__init__.py +261 -0
- altatts/languages/english.py +79 -0
- altatts/languages/french.py +99 -0
- altatts/languages/kinyarwanda.py +114 -0
- altatts/languages/swahili.py +86 -0
- altatts/lexicon.py +298 -0
- altatts/main_cli.py +86 -0
- altatts/metrics.py +300 -0
- altatts/model/__init__.py +19 -0
- altatts/model/latent.py +129 -0
- altatts/model/losses.py +64 -0
- altatts/model/meanflow.py +305 -0
- altatts/model/modules.py +184 -0
- altatts/model/monotonic_align.py +108 -0
- altatts/model/style.py +96 -0
- altatts/model/text_encoder.py +177 -0
- altatts/model/vits.py +327 -0
- altatts/model/waveform.py +159 -0
- altatts/recipes/__init__.py +43 -0
- altatts/recipes/multilingual_callcenter.json +219 -0
- altatts/serving.py +507 -0
- altatts/speaker/__init__.py +2 -0
- altatts/speaker/enroll.py +249 -0
- altatts/speaker/enroll_cli.py +74 -0
- altatts/tags.py +338 -0
- altatts/text.py +547 -0
- altatts/training/__init__.py +25 -0
- altatts/training/anonymize.py +459 -0
- altatts/training/data.py +585 -0
- altatts/training/launch.py +164 -0
- altatts/training/logging.py +381 -0
- altatts/training/prepare.py +595 -0
- altatts/training/segment.py +439 -0
- altatts/training/train.py +324 -0
- altatts/training/trainer.py +779 -0
- altatts/utils.py +219 -0
- altatts/voices_cli.py +91 -0
- altatts-2.0.0.dist-info/METADATA +315 -0
- altatts-2.0.0.dist-info/RECORD +73 -0
- altatts-2.0.0.dist-info/WHEEL +5 -0
- altatts-2.0.0.dist-info/entry_points.txt +15 -0
- altatts-2.0.0.dist-info/top_level.txt +1 -0
altatts/__init__.py
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
"""
|
|
2
|
+
ALTATTS -- Accurate Lifelike TTS: natural, expressive, multilingual
|
|
3
|
+
text-to-speech with zero-shot voice cloning, built for Kinyarwanda and
|
|
4
|
+
reusable for any language.
|
|
5
|
+
======================================================================
|
|
6
|
+
|
|
7
|
+
The speaking half of the ALTA stack (ALTASR listens, ALTATTS talks).
|
|
8
|
+
|
|
9
|
+
``altatts.model`` the network: a multi-speaker VITS backbone + ALTATTS 2
|
|
10
|
+
additions -- zero-shot style (speaker) encoder,
|
|
11
|
+
emotion/intensity conditioning, per-character language
|
|
12
|
+
ids, and a MeanFlow latent prior (one/few-step
|
|
13
|
+
generative prior for natural prosody).
|
|
14
|
+
``altatts.text`` multilingual front end: code-switched Kinyarwanda /
|
|
15
|
+
``altatts.languages`` English / French / Swahili normalization, number
|
|
16
|
+
``altatts.tags`` verbalization per language, control tags for pauses,
|
|
17
|
+
non-verbal sounds and emotions, character tokenizer.
|
|
18
|
+
``altatts.training`` data preparation (any corpus shape), the GAN trainer
|
|
19
|
+
(CPU / multi-GPU / multi-node), fine-tuning, and the
|
|
20
|
+
privacy-preserving default-voice builder.
|
|
21
|
+
``altatts.speaker`` voice enrollment from a few seconds of audio.
|
|
22
|
+
``altatts.inference`` :class:`Synthesizer`, :class:`StreamingSynthesizer`,
|
|
23
|
+
evaluation with a WER gate.
|
|
24
|
+
``altatts.integrations`` call-center glue (G.711 framing, Twilio, VoiceAgent).
|
|
25
|
+
``altatts.env`` ``altatts doctor`` -- environment diagnosis & repair.
|
|
26
|
+
|
|
27
|
+
Quick use::
|
|
28
|
+
|
|
29
|
+
from altatts import Synthesizer
|
|
30
|
+
tts = Synthesizer.from_pretrained("runs/kinyarwanda")
|
|
31
|
+
wav, sr = tts.tts("Muraho! Amakuru yawe?", voice="female")
|
|
32
|
+
wav, sr = tts.tts("Twishimiye [pause:0.4] kubafasha!", voice="male_young",
|
|
33
|
+
emotion="happy", intensity=0.7)
|
|
34
|
+
tts.enroll_speaker("agent_claire", ["claire1.wav", "claire2.wav"])
|
|
35
|
+
wav, sr = tts.tts("Murakoze guhamagara.", voice="agent_claire")
|
|
36
|
+
|
|
37
|
+
Heavy dependencies (torch) are imported lazily: ``import altatts`` plus the
|
|
38
|
+
config/text/metrics layers work on any machine.
|
|
39
|
+
"""
|
|
40
|
+
from __future__ import annotations
|
|
41
|
+
|
|
42
|
+
__version__ = "2.0.0"
|
|
43
|
+
|
|
44
|
+
# Torch-free foundations -- always importable.
|
|
45
|
+
from .config import Config, PRESETS, PRESET_NAMES, load_config, preset # noqa: F401
|
|
46
|
+
from .lexicon import Lexicon, tag_entities, transliterate_english, is_foreign_word # noqa: E501
|
|
47
|
+
from .text import (CharTokenizer, normalize_text, strip_punctuation, # noqa: F401
|
|
48
|
+
expand_numbers, verbalize_number, split_sentences, # noqa: F401
|
|
49
|
+
plan_chunks, plain_text, # noqa: F401
|
|
50
|
+
build_tokenizer, load_tokenizer) # noqa: F401
|
|
51
|
+
from .tags import parse_tags, canonicalize, SPECIAL_TOKENS, DEFAULT_EMOTIONS # noqa: F401
|
|
52
|
+
from .languages import (available_languages, register_language, # noqa: F401
|
|
53
|
+
LanguageSpec, detect_word_language) # noqa: F401
|
|
54
|
+
from .metrics import (mcd, f0_metrics, median_f0, gender_from_f0, # noqa: F401
|
|
55
|
+
wer, cer, corpus_wer_cer, corpus_tts_metrics, rtf) # noqa: F401
|
|
56
|
+
|
|
57
|
+
# Torch-backed public API, resolved lazily (PEP 562).
|
|
58
|
+
_LAZY = {
|
|
59
|
+
"Synthesizer": "altatts.inference.synthesizer",
|
|
60
|
+
"StreamingSynthesizer": "altatts.inference.streaming",
|
|
61
|
+
"stream_to_wav": "altatts.inference.streaming",
|
|
62
|
+
"load_checkpoint_folder": "altatts.inference.checkpoint",
|
|
63
|
+
"load_checkpoint_bundle": "altatts.inference.checkpoint",
|
|
64
|
+
"VoiceBank": "altatts.inference.voices",
|
|
65
|
+
"build_model": "altatts.model.vits",
|
|
66
|
+
"SynthesizerTrn": "altatts.model.vits",
|
|
67
|
+
"Trainer": "altatts.training.trainer",
|
|
68
|
+
"TTSDataset": "altatts.training.data",
|
|
69
|
+
"load_records": "altatts.training.data",
|
|
70
|
+
"SpeakerRegistry": "altatts.training.data",
|
|
71
|
+
"prepare_metadata": "altatts.training.prepare",
|
|
72
|
+
"build_default_voices": "altatts.training.anonymize",
|
|
73
|
+
"embed_speaker_audio": "altatts.speaker.enroll",
|
|
74
|
+
"AudioDecodeError": "altatts.audio",
|
|
75
|
+
"VoiceAgent": "altatts.integrations.callcenter",
|
|
76
|
+
"telephony_frames": "altatts.integrations.callcenter",
|
|
77
|
+
"streaming_telephony_frames": "altatts.integrations.callcenter",
|
|
78
|
+
"twilio_media_messages": "altatts.integrations.callcenter",
|
|
79
|
+
"prerender_prompts": "altatts.integrations.callcenter",
|
|
80
|
+
"enroll_agent": "altatts.integrations.callcenter",
|
|
81
|
+
"export_onnx": "altatts.export",
|
|
82
|
+
"TTSClient": "altatts.client",
|
|
83
|
+
"serve": "altatts.serving",
|
|
84
|
+
"package": "altatts.deploy",
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def __getattr__(name):
|
|
89
|
+
target = _LAZY.get(name)
|
|
90
|
+
if target is None:
|
|
91
|
+
raise AttributeError(f"module 'altatts' has no attribute {name!r}")
|
|
92
|
+
import importlib
|
|
93
|
+
try:
|
|
94
|
+
module = importlib.import_module(target)
|
|
95
|
+
except ImportError as exc:
|
|
96
|
+
raise ImportError(
|
|
97
|
+
f"altatts.{name} requires PyTorch. Install the backend with "
|
|
98
|
+
f"`pip install torch torchaudio` -- or run `altatts doctor --fix` "
|
|
99
|
+
f"({exc})") from exc
|
|
100
|
+
value = getattr(module, name)
|
|
101
|
+
globals()[name] = value
|
|
102
|
+
return value
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def __dir__():
|
|
106
|
+
return sorted(list(globals()) + list(_LAZY))
|
altatts/audio.py
ADDED
|
@@ -0,0 +1,324 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Audio layer: decoding, spectrograms, resampling, telephony output.
|
|
3
|
+
|
|
4
|
+
Decoding mirrors ALTASR's robustness policy: try soundfile, then torchaudio,
|
|
5
|
+
then PyAV, and raise ``AudioDecodeError`` carrying every backend's reason if
|
|
6
|
+
all fail (training substitutes a neighbour and warns, exactly like ALTASR).
|
|
7
|
+
|
|
8
|
+
Spectrograms
|
|
9
|
+
------------
|
|
10
|
+
The model consumes/produces audio at ONE native rate (``AudioConfig.sample_rate``,
|
|
11
|
+
default 22050). Two feature views are computed with torch (GPU-friendly, exact
|
|
12
|
+
across train/infer):
|
|
13
|
+
|
|
14
|
+
* ``spectrogram_torch`` -- linear magnitude STFT (posterior encoder input)
|
|
15
|
+
* ``mel_spectrogram_torch`` -- log-mel (reconstruction loss + evaluation)
|
|
16
|
+
|
|
17
|
+
The mel filterbank is Slaney-style (area-normalised triangles), matching the
|
|
18
|
+
librosa convention the TTS literature standardises on, implemented here in
|
|
19
|
+
numpy so librosa is not a dependency.
|
|
20
|
+
|
|
21
|
+
Sample-rate flexibility
|
|
22
|
+
-----------------------
|
|
23
|
+
``resample`` converts the synthesized waveform to any caller-requested rate
|
|
24
|
+
(8000 for telephony, 16000 to feed ALTASR, 44100/48000 for media). ``to_pcm16``
|
|
25
|
+
/ ``to_mulaw`` / ``to_alaw`` produce the wire formats used by call-center
|
|
26
|
+
stacks (Twilio media streams, Asterisk/FreeSWITCH, SIP trunks).
|
|
27
|
+
"""
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
import io
|
|
31
|
+
import math
|
|
32
|
+
import os
|
|
33
|
+
from typing import Dict, Tuple
|
|
34
|
+
|
|
35
|
+
import numpy as np
|
|
36
|
+
|
|
37
|
+
try: # torch is required for the model layers, but the
|
|
38
|
+
import torch # numpy helpers below stay importable without it.
|
|
39
|
+
except Exception: # pragma: no cover
|
|
40
|
+
torch = None
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class AudioDecodeError(RuntimeError):
|
|
44
|
+
"""Raised when every decoding backend fails; carries per-backend reasons."""
|
|
45
|
+
|
|
46
|
+
def __init__(self, path: str, reasons: Dict[str, str]):
|
|
47
|
+
self.path, self.reasons = path, reasons
|
|
48
|
+
detail = "; ".join(f"{k}: {v}" for k, v in reasons.items())
|
|
49
|
+
super().__init__(f"could not decode {path!r} ({detail})")
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
# ---------------------------------------------------------------------------- #
|
|
53
|
+
# Decoding (numpy in, mono float32 out)
|
|
54
|
+
# ---------------------------------------------------------------------------- #
|
|
55
|
+
def load_audio(path: str, target_sr: int) -> np.ndarray:
|
|
56
|
+
"""Decode any common audio file to mono float32 at ``target_sr``."""
|
|
57
|
+
reasons: Dict[str, str] = {}
|
|
58
|
+
|
|
59
|
+
try:
|
|
60
|
+
import soundfile as sf
|
|
61
|
+
wav, sr = sf.read(path, dtype="float32", always_2d=True)
|
|
62
|
+
wav = wav.mean(axis=1)
|
|
63
|
+
return _resample_np(wav, sr, target_sr)
|
|
64
|
+
except Exception as e: # noqa: BLE001
|
|
65
|
+
reasons["soundfile"] = str(e)
|
|
66
|
+
|
|
67
|
+
if torch is not None:
|
|
68
|
+
try:
|
|
69
|
+
import torchaudio
|
|
70
|
+
wav, sr = torchaudio.load(path)
|
|
71
|
+
wav = wav.mean(dim=0).numpy().astype(np.float32)
|
|
72
|
+
return _resample_np(wav, sr, target_sr)
|
|
73
|
+
except Exception as e: # noqa: BLE001
|
|
74
|
+
reasons["torchaudio"] = str(e)
|
|
75
|
+
|
|
76
|
+
try:
|
|
77
|
+
import av # PyAV: pip install "altatts[av]"
|
|
78
|
+
with av.open(path) as container:
|
|
79
|
+
stream = container.streams.audio[0]
|
|
80
|
+
resampler = av.AudioResampler(format="fltp", layout="mono",
|
|
81
|
+
rate=target_sr)
|
|
82
|
+
chunks = []
|
|
83
|
+
for frame in container.decode(stream):
|
|
84
|
+
for rf in resampler.resample(frame):
|
|
85
|
+
chunks.append(rf.to_ndarray().reshape(-1))
|
|
86
|
+
if chunks:
|
|
87
|
+
return np.concatenate(chunks).astype(np.float32)
|
|
88
|
+
reasons["av"] = "no audio frames"
|
|
89
|
+
except Exception as e: # noqa: BLE001
|
|
90
|
+
reasons["av"] = str(e)
|
|
91
|
+
|
|
92
|
+
# Final fallback: the ffmpeg binary itself. Decodes anything ffmpeg can
|
|
93
|
+
# (webm/opus/m4a/aac/extension-less files) with zero Python deps.
|
|
94
|
+
if _ffmpeg_bin():
|
|
95
|
+
try:
|
|
96
|
+
import subprocess
|
|
97
|
+
proc = subprocess.run(
|
|
98
|
+
[_ffmpeg_bin(), "-v", "error", "-i", path,
|
|
99
|
+
"-f", "f32le", "-acodec", "pcm_f32le",
|
|
100
|
+
"-ac", "1", "-ar", str(target_sr), "pipe:1"],
|
|
101
|
+
stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=120)
|
|
102
|
+
if proc.returncode == 0 and proc.stdout:
|
|
103
|
+
return np.frombuffer(proc.stdout, dtype=np.float32).copy()
|
|
104
|
+
reasons["ffmpeg"] = (proc.stderr.decode("utf-8", "replace")
|
|
105
|
+
.strip()[:200] or "no output")
|
|
106
|
+
except Exception as e: # noqa: BLE001
|
|
107
|
+
reasons["ffmpeg"] = str(e)
|
|
108
|
+
else:
|
|
109
|
+
reasons["ffmpeg"] = "binary not found on PATH"
|
|
110
|
+
|
|
111
|
+
reasons["hint"] = ("install a decoder for this format: pip install av "
|
|
112
|
+
"(recommended, handles webm/opus/m4a/mp3), or "
|
|
113
|
+
"apt-get install ffmpeg, or pip install torchcodec")
|
|
114
|
+
raise AudioDecodeError(path, reasons)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
_FFMPEG_BIN = None
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _ffmpeg_bin():
|
|
121
|
+
global _FFMPEG_BIN
|
|
122
|
+
if _FFMPEG_BIN is None:
|
|
123
|
+
import shutil
|
|
124
|
+
_FFMPEG_BIN = shutil.which("ffmpeg") or ""
|
|
125
|
+
return _FFMPEG_BIN
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _resample_np(wav: np.ndarray, sr: int, target_sr: int) -> np.ndarray:
|
|
129
|
+
if sr == target_sr:
|
|
130
|
+
return np.ascontiguousarray(wav, dtype=np.float32)
|
|
131
|
+
if torch is not None:
|
|
132
|
+
import torchaudio.functional as AF
|
|
133
|
+
t = torch.from_numpy(np.ascontiguousarray(wav, dtype=np.float32))
|
|
134
|
+
return AF.resample(t, sr, target_sr).numpy()
|
|
135
|
+
# numpy fallback: polyphase-free linear interpolation (metrics-grade only)
|
|
136
|
+
n_out = int(round(len(wav) * target_sr / sr))
|
|
137
|
+
x_old = np.linspace(0.0, 1.0, num=len(wav), endpoint=False)
|
|
138
|
+
x_new = np.linspace(0.0, 1.0, num=n_out, endpoint=False)
|
|
139
|
+
return np.interp(x_new, x_old, wav).astype(np.float32)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def trim_silence(wav: np.ndarray, sr: int, top_db: float = 35.0,
|
|
143
|
+
frame_ms: float = 25.0, pad_ms: float = 60.0) -> np.ndarray:
|
|
144
|
+
"""Energy-based leading/trailing silence trim with a small kept pad."""
|
|
145
|
+
frame = max(1, int(sr * frame_ms / 1000))
|
|
146
|
+
if len(wav) < 2 * frame:
|
|
147
|
+
return wav
|
|
148
|
+
n = len(wav) // frame
|
|
149
|
+
rms = np.sqrt(np.mean(wav[: n * frame].reshape(n, frame) ** 2, axis=1) + 1e-12)
|
|
150
|
+
thresh = rms.max() * (10.0 ** (-top_db / 20.0))
|
|
151
|
+
keep = np.where(rms > thresh)[0]
|
|
152
|
+
if len(keep) == 0:
|
|
153
|
+
return wav
|
|
154
|
+
pad = int(sr * pad_ms / 1000)
|
|
155
|
+
start = max(0, keep[0] * frame - pad)
|
|
156
|
+
end = min(len(wav), (keep[-1] + 1) * frame + pad)
|
|
157
|
+
return wav[start:end]
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def peak_normalize(wav: np.ndarray, peak: float = 0.95) -> np.ndarray:
|
|
161
|
+
m = float(np.max(np.abs(wav)) + 1e-9)
|
|
162
|
+
return (wav * (peak / m)).astype(np.float32) if m > 0 else wav
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
# ---------------------------------------------------------------------------- #
|
|
166
|
+
# Mel filterbank (Slaney / librosa convention), numpy
|
|
167
|
+
# ---------------------------------------------------------------------------- #
|
|
168
|
+
def _hz_to_mel(f):
|
|
169
|
+
f = np.asanyarray(f, dtype=np.float64)
|
|
170
|
+
mel = 3.0 * f / 200.0
|
|
171
|
+
log_step = math.log(6.4) / 27.0
|
|
172
|
+
above = f >= 1000.0
|
|
173
|
+
mel = np.where(above, 15.0 + np.log(np.maximum(f, 1e-10) / 1000.0) / log_step, mel)
|
|
174
|
+
return mel
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def _mel_to_hz(m):
|
|
178
|
+
m = np.asanyarray(m, dtype=np.float64)
|
|
179
|
+
f = 200.0 * m / 3.0
|
|
180
|
+
log_step = math.log(6.4) / 27.0
|
|
181
|
+
above = m >= 15.0
|
|
182
|
+
return np.where(above, 1000.0 * np.exp(log_step * (m - 15.0)), f)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def mel_filterbank(sr: int, n_fft: int, n_mels: int,
|
|
186
|
+
fmin: float = 0.0, fmax: float = 0.0) -> np.ndarray:
|
|
187
|
+
"""(n_mels, n_fft//2+1) Slaney-normalised triangular filterbank."""
|
|
188
|
+
fmax = fmax or sr / 2.0
|
|
189
|
+
fft_freqs = np.linspace(0.0, sr / 2.0, n_fft // 2 + 1)
|
|
190
|
+
mel_pts = np.linspace(_hz_to_mel(fmin), _hz_to_mel(fmax), n_mels + 2)
|
|
191
|
+
hz_pts = _mel_to_hz(mel_pts)
|
|
192
|
+
fb = np.zeros((n_mels, len(fft_freqs)), dtype=np.float64)
|
|
193
|
+
fdiff = np.diff(hz_pts)
|
|
194
|
+
ramps = hz_pts.reshape(-1, 1) - fft_freqs.reshape(1, -1)
|
|
195
|
+
for i in range(n_mels):
|
|
196
|
+
lower = -ramps[i] / fdiff[i]
|
|
197
|
+
upper = ramps[i + 2] / fdiff[i + 1]
|
|
198
|
+
fb[i] = np.maximum(0.0, np.minimum(lower, upper))
|
|
199
|
+
enorm = 2.0 / (hz_pts[2: n_mels + 2] - hz_pts[:n_mels])
|
|
200
|
+
fb *= enorm[:, None]
|
|
201
|
+
return fb.astype(np.float32)
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
# ---------------------------------------------------------------------------- #
|
|
205
|
+
# Torch spectrograms (train / inference exactness)
|
|
206
|
+
# ---------------------------------------------------------------------------- #
|
|
207
|
+
_hann_cache: Dict[Tuple, "torch.Tensor"] = {}
|
|
208
|
+
_mel_cache: Dict[Tuple, "torch.Tensor"] = {}
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def spectrogram_torch(y: "torch.Tensor", n_fft: int, hop_length: int,
|
|
212
|
+
win_length: int) -> "torch.Tensor":
|
|
213
|
+
"""Linear magnitude spectrogram, (B, n_fft//2+1, T). y: (B, samples)."""
|
|
214
|
+
key = (win_length, y.device, y.dtype)
|
|
215
|
+
win = _hann_cache.get(key)
|
|
216
|
+
if win is None:
|
|
217
|
+
win = torch.hann_window(win_length, device=y.device, dtype=y.dtype)
|
|
218
|
+
_hann_cache[key] = win
|
|
219
|
+
pad = (n_fft - hop_length) // 2
|
|
220
|
+
y = torch.nn.functional.pad(y.unsqueeze(1), (pad, pad), mode="reflect").squeeze(1)
|
|
221
|
+
spec = torch.stft(y, n_fft, hop_length=hop_length, win_length=win_length,
|
|
222
|
+
window=win, center=False, pad_mode="reflect",
|
|
223
|
+
normalized=False, onesided=True, return_complex=True)
|
|
224
|
+
return torch.sqrt(spec.real.pow(2) + spec.imag.pow(2) + 1e-6)
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def spec_to_mel_torch(spec: "torch.Tensor", sr: int, n_fft: int, n_mels: int,
|
|
228
|
+
fmin: float, fmax: float) -> "torch.Tensor":
|
|
229
|
+
key = (sr, n_fft, n_mels, fmin, fmax, spec.device, spec.dtype)
|
|
230
|
+
fb = _mel_cache.get(key)
|
|
231
|
+
if fb is None:
|
|
232
|
+
fb = torch.from_numpy(mel_filterbank(sr, n_fft, n_mels, fmin, fmax))
|
|
233
|
+
fb = fb.to(device=spec.device, dtype=spec.dtype)
|
|
234
|
+
_mel_cache[key] = fb
|
|
235
|
+
mel = torch.matmul(fb, spec)
|
|
236
|
+
return torch.log(torch.clamp(mel, min=1e-5))
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def mel_spectrogram_torch(y: "torch.Tensor", sr: int, n_fft: int,
|
|
240
|
+
hop_length: int, win_length: int, n_mels: int,
|
|
241
|
+
fmin: float = 0.0, fmax: float = 0.0) -> "torch.Tensor":
|
|
242
|
+
spec = spectrogram_torch(y, n_fft, hop_length, win_length)
|
|
243
|
+
return spec_to_mel_torch(spec, sr, n_fft, n_mels, fmin, fmax)
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
# ---------------------------------------------------------------------------- #
|
|
247
|
+
# Output-rate conversion & telephony wire formats
|
|
248
|
+
# ---------------------------------------------------------------------------- #
|
|
249
|
+
def resample(wav: np.ndarray, sr: int, target_sr: int) -> np.ndarray:
|
|
250
|
+
"""High-quality resampling of a synthesized waveform to any rate."""
|
|
251
|
+
return _resample_np(wav, sr, target_sr)
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def to_pcm16(wav: np.ndarray) -> bytes:
|
|
255
|
+
wav = np.clip(wav, -1.0, 1.0)
|
|
256
|
+
return (wav * 32767.0).astype("<i2").tobytes()
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def to_mulaw(wav: np.ndarray) -> bytes:
|
|
260
|
+
"""G.711 mu-law bytes (8-bit) -- pair with an 8000 Hz waveform."""
|
|
261
|
+
mu = 255.0
|
|
262
|
+
wav = np.clip(wav, -1.0, 1.0)
|
|
263
|
+
mag = np.log1p(mu * np.abs(wav)) / np.log1p(mu)
|
|
264
|
+
comp = np.sign(wav) * mag
|
|
265
|
+
return ((comp + 1.0) / 2.0 * mu + 0.5).astype(np.uint8).tobytes()
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def to_alaw(wav: np.ndarray) -> bytes:
|
|
269
|
+
"""G.711 A-law bytes (8-bit) -- pair with an 8000 Hz waveform."""
|
|
270
|
+
A = 87.6
|
|
271
|
+
wav = np.clip(wav, -1.0, 1.0)
|
|
272
|
+
ax = np.abs(wav)
|
|
273
|
+
small = ax < (1.0 / A)
|
|
274
|
+
comp = np.where(small, A * ax / (1.0 + math.log(A)),
|
|
275
|
+
(1.0 + np.log(np.maximum(A * ax, 1e-12))) / (1.0 + math.log(A)))
|
|
276
|
+
y = np.sign(wav) * comp
|
|
277
|
+
pcm = (np.clip(y, -1.0, 1.0) * 32767.0).astype(np.int16)
|
|
278
|
+
out = np.empty(len(pcm), dtype=np.uint8)
|
|
279
|
+
for i, s in enumerate(pcm): # scalar loop: clarity > speed
|
|
280
|
+
sign = 0x80 if s >= 0 else 0
|
|
281
|
+
s = min(abs(int(s)), 32767) >> 3 # 13-bit magnitude
|
|
282
|
+
if s >= 32:
|
|
283
|
+
exp = int(math.floor(math.log2(s))) - 4
|
|
284
|
+
mant = (s >> exp) & 0x0F
|
|
285
|
+
byte = (exp << 4) | mant
|
|
286
|
+
else:
|
|
287
|
+
byte = s >> 1
|
|
288
|
+
out[i] = (byte ^ 0x55) | sign
|
|
289
|
+
return out.tobytes()
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def _wav_pcm16_bytes(wav: np.ndarray, sr: int) -> bytes:
|
|
293
|
+
"""RIFF/WAV via the stdlib (16-bit PCM) -- no soundfile needed."""
|
|
294
|
+
import wave
|
|
295
|
+
buf = io.BytesIO()
|
|
296
|
+
with wave.open(buf, "wb") as w:
|
|
297
|
+
w.setnchannels(1)
|
|
298
|
+
w.setsampwidth(2)
|
|
299
|
+
w.setframerate(sr)
|
|
300
|
+
w.writeframes(to_pcm16(wav))
|
|
301
|
+
return buf.getvalue()
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def save_wav(path: str, wav: np.ndarray, sr: int) -> None:
|
|
305
|
+
parent = os.path.dirname(os.path.abspath(path))
|
|
306
|
+
os.makedirs(parent, exist_ok=True)
|
|
307
|
+
try:
|
|
308
|
+
import soundfile as sf
|
|
309
|
+
sf.write(path, np.clip(wav, -1.0, 1.0), sr)
|
|
310
|
+
except ImportError:
|
|
311
|
+
with open(path, "wb") as f:
|
|
312
|
+
f.write(_wav_pcm16_bytes(wav, sr))
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def wav_bytes(wav: np.ndarray, sr: int) -> bytes:
|
|
316
|
+
"""In-memory RIFF/WAV (16-bit PCM) -- convenient for web/IVR responses."""
|
|
317
|
+
try:
|
|
318
|
+
import soundfile as sf
|
|
319
|
+
buf = io.BytesIO()
|
|
320
|
+
sf.write(buf, np.clip(wav, -1.0, 1.0), sr, format="WAV",
|
|
321
|
+
subtype="PCM_16")
|
|
322
|
+
return buf.getvalue()
|
|
323
|
+
except ImportError:
|
|
324
|
+
return _wav_pcm16_bytes(wav, sr)
|
altatts/benchmark.py
ADDED
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
"""``altatts-benchmark`` -- compare ALTATTS checkpoints (and optionally the
|
|
2
|
+
Meta MMS-TTS Kinyarwanda baseline) on the same test set.
|
|
3
|
+
|
|
4
|
+
altatts-benchmark \
|
|
5
|
+
--models runs/kinyarwanda-small runs/kinyarwanda-base \
|
|
6
|
+
--metadata data/tts_test.json --audio-root data/ \
|
|
7
|
+
--asr-model path/to/altasr --out benchmark.md
|
|
8
|
+
|
|
9
|
+
Adds Meta's MMS-TTS (facebook/mms-tts-kin) as a baseline row with
|
|
10
|
+
``--include-mms`` (requires ``pip install altatts[mms]``).
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import argparse
|
|
16
|
+
import json
|
|
17
|
+
import os
|
|
18
|
+
import time
|
|
19
|
+
from typing import Dict, List, Optional
|
|
20
|
+
|
|
21
|
+
import numpy as np
|
|
22
|
+
|
|
23
|
+
from .utils import quiet_warnings, clear_terminal
|
|
24
|
+
|
|
25
|
+
_METRIC_COLS = ("mcd_db", "f0_rmse_hz", "vuv_error", "pmos", "rtf",
|
|
26
|
+
"gender_match", "spk_sim", "asr_wer", "asr_cer")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _mms_eval(metadata: List[str], audio_root: str, asr_model: Optional[str],
|
|
30
|
+
max_utterances: Optional[int], device: str) -> Optional[Dict]:
|
|
31
|
+
"""Evaluate facebook/mms-tts-kin on the same protocol (baseline row)."""
|
|
32
|
+
try:
|
|
33
|
+
import torch
|
|
34
|
+
from transformers import VitsModel, AutoTokenizer
|
|
35
|
+
except ImportError:
|
|
36
|
+
print("[benchmark] transformers not installed -- skipping MMS "
|
|
37
|
+
"baseline. Install with: pip install altatts[mms]")
|
|
38
|
+
return None
|
|
39
|
+
|
|
40
|
+
from .config import preset
|
|
41
|
+
from .audio import load_audio, AudioDecodeError, resample
|
|
42
|
+
from .metrics import (mcd, f0_metrics, predicted_mos, corpus_tts_metrics)
|
|
43
|
+
from .text import normalize_text, strip_punctuation
|
|
44
|
+
from .training.data import load_records, resolve_audio_path
|
|
45
|
+
|
|
46
|
+
cfg = preset("base")
|
|
47
|
+
records = load_records(metadata, cfg)
|
|
48
|
+
if max_utterances:
|
|
49
|
+
records = records[:max_utterances]
|
|
50
|
+
|
|
51
|
+
dev = torch.device("cuda" if (device in ("auto", "cuda")
|
|
52
|
+
and torch.cuda.is_available()) else "cpu")
|
|
53
|
+
model = VitsModel.from_pretrained("facebook/mms-tts-kin").to(dev).eval()
|
|
54
|
+
tok = AutoTokenizer.from_pretrained("facebook/mms-tts-kin")
|
|
55
|
+
mms_sr = int(model.config.sampling_rate)
|
|
56
|
+
|
|
57
|
+
transcriber = None
|
|
58
|
+
if asr_model:
|
|
59
|
+
try:
|
|
60
|
+
from altasr import Transcriber
|
|
61
|
+
transcriber = Transcriber.from_pretrained(asr_model, device=device)
|
|
62
|
+
except ImportError:
|
|
63
|
+
pass
|
|
64
|
+
|
|
65
|
+
pairs = []
|
|
66
|
+
for rec in records:
|
|
67
|
+
path = resolve_audio_path(audio_root, rec["audio_path"])
|
|
68
|
+
try:
|
|
69
|
+
ref = load_audio(path, target_sr=mms_sr)
|
|
70
|
+
except (AudioDecodeError, Exception):
|
|
71
|
+
continue
|
|
72
|
+
t0 = time.perf_counter()
|
|
73
|
+
with torch.inference_mode():
|
|
74
|
+
from .text import plain_text
|
|
75
|
+
inp = tok(plain_text(rec["norm_text"]), return_tensors="pt").to(dev)
|
|
76
|
+
syn = model(**inp).waveform.squeeze().float().cpu().numpy()
|
|
77
|
+
dt = time.perf_counter() - t0
|
|
78
|
+
m = {"mcd_db": mcd(ref, syn, mms_sr), "rtf": dt / (len(syn) / mms_sr),
|
|
79
|
+
"pmos": predicted_mos(syn, mms_sr)}
|
|
80
|
+
m.update(f0_metrics(ref, syn, mms_sr))
|
|
81
|
+
if transcriber is not None:
|
|
82
|
+
try:
|
|
83
|
+
hyp = transcriber.transcribe(resample(syn, mms_sr, 16000),
|
|
84
|
+
sample_rate=16000)
|
|
85
|
+
if isinstance(hyp, dict):
|
|
86
|
+
hyp = hyp.get("text", "")
|
|
87
|
+
tc = cfg.text
|
|
88
|
+
m["ref_text"] = strip_punctuation(normalize_text(
|
|
89
|
+
rec["text"], tc.keep_punctuation, tc.punctuation,
|
|
90
|
+
tc.expand_numbers))
|
|
91
|
+
m["hyp_text"] = strip_punctuation(str(hyp).lower())
|
|
92
|
+
except Exception: # noqa: BLE001
|
|
93
|
+
pass
|
|
94
|
+
pairs.append(m)
|
|
95
|
+
return corpus_tts_metrics(pairs) if pairs else None
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _markdown(rows: Dict[str, Dict]) -> str:
|
|
99
|
+
cols = [c for c in _METRIC_COLS
|
|
100
|
+
if any(c in s for s in rows.values())]
|
|
101
|
+
lines = ["| model | " + " | ".join(cols) + " |",
|
|
102
|
+
"|---" * (len(cols) + 1) + "|"]
|
|
103
|
+
for name, s in rows.items():
|
|
104
|
+
cells = [f"{s[c]:.3f}" if c in s else "--" for c in cols]
|
|
105
|
+
lines.append(f"| {name} | " + " | ".join(cells) + " |")
|
|
106
|
+
lines.append("")
|
|
107
|
+
lines.append("Lower is better for mcd_db, f0_rmse_hz, vuv_error, rtf, "
|
|
108
|
+
"asr_wer, asr_cer. Higher is better for pmos (1-5), "
|
|
109
|
+
"gender_match (0-1) and spk_sim (cosine, 0-1).")
|
|
110
|
+
return "\n".join(lines)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def main(argv: Optional[List[str]] = None) -> None:
|
|
114
|
+
quiet_warnings()
|
|
115
|
+
clear_terminal()
|
|
116
|
+
p = argparse.ArgumentParser(description="Benchmark ALTATTS checkpoints")
|
|
117
|
+
p.add_argument("--models", required=True, nargs="+",
|
|
118
|
+
help="Checkpoint folders to compare")
|
|
119
|
+
p.add_argument("--metadata", required=True, nargs="+")
|
|
120
|
+
p.add_argument("--audio-root", default="")
|
|
121
|
+
p.add_argument("--asr-model", default=None,
|
|
122
|
+
help="ALTASR checkpoint for WER/CER")
|
|
123
|
+
p.add_argument("--include-mms", action="store_true",
|
|
124
|
+
help="Add facebook/mms-tts-kin as a baseline row")
|
|
125
|
+
p.add_argument("--max-utterances", type=int, default=200)
|
|
126
|
+
p.add_argument("--clone", action="store_true",
|
|
127
|
+
help="v2 models: clone each utterance's speaker from its "
|
|
128
|
+
"reference clip")
|
|
129
|
+
p.add_argument("--out", default=None, help=".md or .json report path")
|
|
130
|
+
p.add_argument("--device", default="auto")
|
|
131
|
+
args = p.parse_args(argv)
|
|
132
|
+
|
|
133
|
+
from .inference.evaluate import evaluate
|
|
134
|
+
|
|
135
|
+
rows: Dict[str, Dict] = {}
|
|
136
|
+
for model_dir in args.models:
|
|
137
|
+
name = os.path.basename(os.path.normpath(model_dir))
|
|
138
|
+
print(f"\n== {name} ==")
|
|
139
|
+
report = evaluate(model_dir, args.metadata, args.audio_root,
|
|
140
|
+
out_path=None, asr_model=args.asr_model,
|
|
141
|
+
max_utterances=args.max_utterances,
|
|
142
|
+
device=args.device, verbose=True, clone=args.clone)
|
|
143
|
+
rows[name] = report["summary"]
|
|
144
|
+
|
|
145
|
+
if args.include_mms:
|
|
146
|
+
print("\n== mms-tts-kin (baseline) ==")
|
|
147
|
+
s = _mms_eval(args.metadata, args.audio_root, args.asr_model,
|
|
148
|
+
args.max_utterances, args.device)
|
|
149
|
+
if s:
|
|
150
|
+
rows["mms-tts-kin"] = s
|
|
151
|
+
|
|
152
|
+
md = _markdown(rows)
|
|
153
|
+
print("\n" + md)
|
|
154
|
+
if args.out:
|
|
155
|
+
os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True)
|
|
156
|
+
with open(args.out, "w", encoding="utf-8") as f:
|
|
157
|
+
if args.out.endswith(".json"):
|
|
158
|
+
json.dump(rows, f, ensure_ascii=False, indent=2)
|
|
159
|
+
else:
|
|
160
|
+
f.write(md + "\n")
|
|
161
|
+
print(f"\nReport -> {args.out}")
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
if __name__ == "__main__":
|
|
165
|
+
main()
|