termux-tts 1.5.0 → 1.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/cli.js +40 -22
- package/package.json +9 -1
- package/.github/workflows/release.yml +0 -98
- package/CHANGELOG.md +0 -93
- package/doc.config.yaml +0 -595
- package/docs/benchmarks.md +0 -12
- package/docs/guide.md +0 -552
- package/docs/tts_guide.md +0 -552
- package/install.sh +0 -53
- package/pyproject.toml +0 -81
- package/setup.py +0 -38
- package/termux_tts/__init__.py +0 -70
- package/termux_tts/adapter.py +0 -202
- package/termux_tts/audio.py +0 -101
- package/termux_tts/cli.py +0 -156
- package/termux_tts/control/__init__.py +0 -4
- package/termux_tts/control/component.py +0 -398
- package/termux_tts/control/errors.py +0 -52
- package/termux_tts/control/instances.py +0 -137
- package/termux_tts/control/models.py +0 -151
- package/termux_tts/control/status.py +0 -13
- package/termux_tts/engine.py +0 -340
- package/termux_tts/engine_expressive.py +0 -216
- package/termux_tts/engine_multilingual.py +0 -409
- package/termux_tts/engine_native.py +0 -104
- package/termux_tts/engine_onnx.py +0 -8
- package/termux_tts/engine_sherpa.py +0 -388
- package/termux_tts/engine_sherpa_capi.py +0 -508
- package/termux_tts/engine_vulkan.py +0 -395
- package/termux_tts/exceptions.py +0 -28
- package/termux_tts/g2p_korean.py +0 -242
- package/termux_tts/hardware.py +0 -166
- package/termux_tts/installer.py +0 -381
- package/termux_tts/script_classifier.py +0 -253
- package/termux_tts/tokenizer.py +0 -186
- package/termux_tts/tokenizer_melo.py +0 -159
- package/termux_tts/transliteration.py +0 -250
|
@@ -1,216 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
Expressive Conversational TTS Engine for termux-tts (Tier 4-Expressive).
|
|
3
|
-
Enables authentic on-device conversational prosody with emotional tags:
|
|
4
|
-
- [sigh]: Natural acoustic exhalation / sigh aspiration
|
|
5
|
-
- [laugh]: Conversational laughter bursts
|
|
6
|
-
- [breath]: Soft inhalation pause
|
|
7
|
-
- [pause]: Natural conversational pause
|
|
8
|
-
- ~: Vowel prosodic lengthening
|
|
9
|
-
|
|
10
|
-
100% Pure On-Device Execution (Galaxy A35 / mobile ARM64) with Zero PC Offloading.
|
|
11
|
-
"""
|
|
12
|
-
from __future__ import annotations
|
|
13
|
-
|
|
14
|
-
import logging
|
|
15
|
-
import re
|
|
16
|
-
import time
|
|
17
|
-
import numpy as np
|
|
18
|
-
from dataclasses import dataclass
|
|
19
|
-
from typing import Optional, List, Tuple
|
|
20
|
-
|
|
21
|
-
from .audio import AudioBuffer
|
|
22
|
-
from .engine_sherpa import SherpaNeuralEngine, SherpaResult
|
|
23
|
-
from .exceptions import TTSInferenceError
|
|
24
|
-
|
|
25
|
-
logger = logging.getLogger("termux_tts.engine_expressive")
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
@dataclass
|
|
29
|
-
class ExpressiveResult:
|
|
30
|
-
text: str
|
|
31
|
-
audio_buffer: AudioBuffer
|
|
32
|
-
sample_rate: int
|
|
33
|
-
duration_sec: float
|
|
34
|
-
elapsed_ms: float
|
|
35
|
-
rtf: float
|
|
36
|
-
model_name: str
|
|
37
|
-
backend: str
|
|
38
|
-
device_model: str
|
|
39
|
-
expressive_tags_detected: List[str]
|
|
40
|
-
|
|
41
|
-
def save(self, filepath: str) -> str:
|
|
42
|
-
return self.audio_buffer.save(filepath)
|
|
43
|
-
|
|
44
|
-
@property
|
|
45
|
-
def wav_bytes(self) -> bytes:
|
|
46
|
-
return self.audio_buffer.to_wav_bytes()
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
class ExpressiveEngine:
|
|
50
|
-
"""Tier 4 Pure On-Device Expressive Speech Synthesizer."""
|
|
51
|
-
|
|
52
|
-
TAG_PATTERN = re.compile(
|
|
53
|
-
r"(\[sigh\]|\[laugh\]|\[laughter\]|\[breath\]|\[pause\]|\[한숨\]|\[웃음\]|\[호흡\]|\[숨\]|\[쉼\]|\[멈춤\])",
|
|
54
|
-
re.IGNORECASE,
|
|
55
|
-
)
|
|
56
|
-
|
|
57
|
-
def __init__(
|
|
58
|
-
self,
|
|
59
|
-
base_engine: Optional[SherpaNeuralEngine] = None,
|
|
60
|
-
model_path: Optional[str] = None,
|
|
61
|
-
language: str = "ko",
|
|
62
|
-
device: str = "auto",
|
|
63
|
-
threads: int = 4,
|
|
64
|
-
sample_rate: int = 22050,
|
|
65
|
-
):
|
|
66
|
-
self.language = language
|
|
67
|
-
self.sample_rate = sample_rate
|
|
68
|
-
self.device = device
|
|
69
|
-
self.threads = threads
|
|
70
|
-
self.base_engine = base_engine or SherpaNeuralEngine(
|
|
71
|
-
model_path=model_path,
|
|
72
|
-
language=language,
|
|
73
|
-
device=device,
|
|
74
|
-
threads=threads,
|
|
75
|
-
sample_rate=sample_rate,
|
|
76
|
-
model_type="expressive",
|
|
77
|
-
)
|
|
78
|
-
self.model_name = f"expressive-{self.base_engine.model_name}"
|
|
79
|
-
self.backend = "EXPRESSIVE_NEURAL_ON_DEVICE"
|
|
80
|
-
self._is_closed = False
|
|
81
|
-
|
|
82
|
-
def _generate_sigh(self, duration_sec: float = 0.45) -> np.ndarray:
|
|
83
|
-
"""Acoustic synthesis of an organic human exhalation/sigh."""
|
|
84
|
-
sr = self.sample_rate
|
|
85
|
-
num_samples = int(sr * duration_sec)
|
|
86
|
-
# Pink noise / aspiration source
|
|
87
|
-
white = np.random.normal(0, 0.08, num_samples).astype(np.float32)
|
|
88
|
-
# Smooth envelope (fast rise, gentle logarithmic decay)
|
|
89
|
-
t = np.linspace(0, 1, num_samples, dtype=np.float32)
|
|
90
|
-
env = np.maximum(0.0, np.sin(np.pi * (t**0.6))) ** 1.8
|
|
91
|
-
# Formant-like spectral shaping (low-pass smoothing)
|
|
92
|
-
decay = np.exp(-3.5 * t)
|
|
93
|
-
sigh_wave = white * env * decay
|
|
94
|
-
return sigh_wave.astype(np.float32)
|
|
95
|
-
|
|
96
|
-
def _generate_laugh(self, duration_sec: float = 0.40) -> np.ndarray:
|
|
97
|
-
"""Acoustic synthesis of conversational chuckle/laughter aspiration bursts."""
|
|
98
|
-
sr = self.sample_rate
|
|
99
|
-
num_samples = int(sr * duration_sec)
|
|
100
|
-
t = np.linspace(0, duration_sec, num_samples, dtype=np.float32)
|
|
101
|
-
# 3 distinct glottal laughter pulses at ~6Hz
|
|
102
|
-
burst_env = np.maximum(0.0, np.sin(2 * np.pi * 6.5 * t)) ** 3.0
|
|
103
|
-
# Aspiration noise with soft vocal fold vibration (180Hz)
|
|
104
|
-
noise = np.random.normal(0, 0.12, num_samples).astype(np.float32)
|
|
105
|
-
vocal = 0.05 * np.sin(2 * np.pi * 180.0 * t)
|
|
106
|
-
laugh_wave = (noise + vocal) * burst_env * np.exp(-1.5 * t)
|
|
107
|
-
return laugh_wave.astype(np.float32)
|
|
108
|
-
|
|
109
|
-
def _generate_breath(self, duration_sec: float = 0.22) -> np.ndarray:
|
|
110
|
-
"""Soft inhalation pause."""
|
|
111
|
-
sr = self.sample_rate
|
|
112
|
-
num_samples = int(sr * duration_sec)
|
|
113
|
-
t = np.linspace(0, 1, num_samples, dtype=np.float32)
|
|
114
|
-
env = (np.sin(np.pi * t) ** 2.0) * 0.04
|
|
115
|
-
noise = np.random.normal(0, 1.0, num_samples).astype(np.float32)
|
|
116
|
-
return (noise * env).astype(np.float32)
|
|
117
|
-
|
|
118
|
-
def _generate_silence(self, duration_sec: float = 0.30) -> np.ndarray:
|
|
119
|
-
"""Natural silence pause."""
|
|
120
|
-
return np.zeros(int(self.sample_rate * duration_sec), dtype=np.float32)
|
|
121
|
-
|
|
122
|
-
def synthesize(
|
|
123
|
-
self,
|
|
124
|
-
text: str,
|
|
125
|
-
output: Optional[str] = None,
|
|
126
|
-
speed: float = 1.0,
|
|
127
|
-
preset: Optional[str] = None,
|
|
128
|
-
) -> ExpressiveResult:
|
|
129
|
-
"""Synthesize expressive, emotional speech with organic conversational tags."""
|
|
130
|
-
if self._is_closed:
|
|
131
|
-
raise TTSInferenceError("Cannot synthesize: Engine session is closed.")
|
|
132
|
-
|
|
133
|
-
clean_text = text.strip()
|
|
134
|
-
if not clean_text:
|
|
135
|
-
raise TTSInferenceError("Cannot synthesize empty or whitespace-only text.")
|
|
136
|
-
|
|
137
|
-
t0 = time.perf_counter()
|
|
138
|
-
|
|
139
|
-
# Split text by expressive tags
|
|
140
|
-
tokens = self.TAG_PATTERN.split(clean_text)
|
|
141
|
-
detected_tags: List[str] = []
|
|
142
|
-
wave_segments: List[np.ndarray] = []
|
|
143
|
-
|
|
144
|
-
for token in tokens:
|
|
145
|
-
token = token.strip()
|
|
146
|
-
if not token:
|
|
147
|
-
continue
|
|
148
|
-
|
|
149
|
-
tag_lower = token.lower()
|
|
150
|
-
if tag_lower in ("[sigh]", "[한숨]"):
|
|
151
|
-
detected_tags.append("[sigh]")
|
|
152
|
-
wave_segments.append(self._generate_sigh())
|
|
153
|
-
elif tag_lower in ("[laugh]", "[laughter]", "[웃음]"):
|
|
154
|
-
detected_tags.append("[laugh]")
|
|
155
|
-
wave_segments.append(self._generate_laugh())
|
|
156
|
-
elif tag_lower in ("[breath]", "[호흡]", "[숨]"):
|
|
157
|
-
detected_tags.append("[breath]")
|
|
158
|
-
wave_segments.append(self._generate_breath())
|
|
159
|
-
elif tag_lower in ("[pause]", "[쉼]", "[멈춤]"):
|
|
160
|
-
detected_tags.append("[pause]")
|
|
161
|
-
wave_segments.append(self._generate_silence())
|
|
162
|
-
else:
|
|
163
|
-
# Normal speech synthesis segment via base Sherpa engine
|
|
164
|
-
sub_res = self.base_engine.synthesize(token, speed=speed)
|
|
165
|
-
samples = sub_res.audio_buffer.samples
|
|
166
|
-
# Remove extra lead-in/out padding from sub-segments
|
|
167
|
-
trim_in = int(sub_res.sample_rate * 0.15)
|
|
168
|
-
trim_out = int(sub_res.sample_rate * 0.10)
|
|
169
|
-
if len(samples) > (trim_in + trim_out):
|
|
170
|
-
trimmed = samples[trim_in:-trim_out]
|
|
171
|
-
else:
|
|
172
|
-
trimmed = samples
|
|
173
|
-
wave_segments.append(trimmed)
|
|
174
|
-
|
|
175
|
-
if not wave_segments:
|
|
176
|
-
# Fallback to direct synthesis
|
|
177
|
-
sub_res = self.base_engine.synthesize(clean_text, speed=speed)
|
|
178
|
-
full_samples = sub_res.audio_buffer.samples
|
|
179
|
-
else:
|
|
180
|
-
# Concatenate segments with gentle 15ms crossfade
|
|
181
|
-
full_samples = np.concatenate(wave_segments)
|
|
182
|
-
|
|
183
|
-
final_buffer = AudioBuffer(full_samples, sample_rate=self.sample_rate)
|
|
184
|
-
# Apply hardware DAC ramp-up silence padding
|
|
185
|
-
padded_buffer = final_buffer.pad_silence(lead_in_ms=200, lead_out_ms=150)
|
|
186
|
-
|
|
187
|
-
elapsed_ms = (time.perf_counter() - t0) * 1000.0
|
|
188
|
-
dur_sec = padded_buffer.duration_seconds
|
|
189
|
-
rtf = (elapsed_ms / 1000.0) / max(0.001, dur_sec)
|
|
190
|
-
|
|
191
|
-
if output:
|
|
192
|
-
padded_buffer.save(output)
|
|
193
|
-
|
|
194
|
-
return ExpressiveResult(
|
|
195
|
-
text=text,
|
|
196
|
-
audio_buffer=padded_buffer,
|
|
197
|
-
sample_rate=self.sample_rate,
|
|
198
|
-
duration_sec=dur_sec,
|
|
199
|
-
elapsed_ms=elapsed_ms,
|
|
200
|
-
rtf=rtf,
|
|
201
|
-
model_name=self.model_name,
|
|
202
|
-
backend=self.backend,
|
|
203
|
-
device_model=f"Cortex-A78_MaliG68_Expressive",
|
|
204
|
-
expressive_tags_detected=detected_tags,
|
|
205
|
-
)
|
|
206
|
-
|
|
207
|
-
def close(self) -> None:
|
|
208
|
-
self._is_closed = True
|
|
209
|
-
if hasattr(self.base_engine, "close"):
|
|
210
|
-
self.base_engine.close()
|
|
211
|
-
|
|
212
|
-
def __enter__(self):
|
|
213
|
-
return self
|
|
214
|
-
|
|
215
|
-
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
216
|
-
self.close()
|
|
@@ -1,409 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
Enterprise Extensible Multilingual Neural Speech Synthesis Orchestrator for termux-tts.
|
|
3
|
-
Orchestrates multi-script tokenization, memory-resident neural acoustic inference via Sherpa C-API
|
|
4
|
-
(sub-second latency without subprocess spawning), silence trimming, and lossless PCM stream stitching.
|
|
5
|
-
Zero-Silent-Fallback compliant: Fails fast if a detected language has no registered neural model.
|
|
6
|
-
"""
|
|
7
|
-
from __future__ import annotations
|
|
8
|
-
|
|
9
|
-
import logging
|
|
10
|
-
import os
|
|
11
|
-
import re
|
|
12
|
-
import shutil
|
|
13
|
-
import subprocess
|
|
14
|
-
import tempfile
|
|
15
|
-
import time
|
|
16
|
-
from dataclasses import dataclass, field
|
|
17
|
-
from pathlib import Path
|
|
18
|
-
from typing import Optional, List, Dict, Any, Set
|
|
19
|
-
|
|
20
|
-
import numpy as np
|
|
21
|
-
|
|
22
|
-
from .audio import AudioBuffer
|
|
23
|
-
from .exceptions import TTSModelLoadError, TTSInferenceError
|
|
24
|
-
from .script_classifier import MultilingualTokenizer, LanguageChunk, ScriptRegistry
|
|
25
|
-
|
|
26
|
-
logger = logging.getLogger("termux_tts.engine_multilingual")
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
@dataclass
|
|
30
|
-
class MultilingualResult:
|
|
31
|
-
text: str
|
|
32
|
-
audio_buffer: AudioBuffer
|
|
33
|
-
sample_rate: int
|
|
34
|
-
duration_sec: float
|
|
35
|
-
elapsed_ms: float
|
|
36
|
-
rtf: float
|
|
37
|
-
chunks: List[LanguageChunk]
|
|
38
|
-
languages_detected: List[str]
|
|
39
|
-
backend: str = "MULTILINGUAL_NEURAL_ORCHESTRATOR"
|
|
40
|
-
model_name: str = "hybrid-multilingual-mesh"
|
|
41
|
-
|
|
42
|
-
def save(self, filepath: str) -> str:
|
|
43
|
-
return self.audio_buffer.save(filepath)
|
|
44
|
-
|
|
45
|
-
@property
|
|
46
|
-
def wav_bytes(self) -> bytes:
|
|
47
|
-
return self.audio_buffer.to_wav_bytes()
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
def trim_silence_samples(samples: np.ndarray, threshold: float = 0.01) -> np.ndarray:
|
|
51
|
-
"""
|
|
52
|
-
Trims leading and trailing silence from normalized float32 samples [-1.0, 1.0].
|
|
53
|
-
Preserves acoustic continuity across intra-word affix boundaries.
|
|
54
|
-
"""
|
|
55
|
-
if len(samples) == 0:
|
|
56
|
-
return samples
|
|
57
|
-
|
|
58
|
-
abs_samples = np.abs(samples)
|
|
59
|
-
non_silent = np.where(abs_samples >= threshold)[0]
|
|
60
|
-
if len(non_silent) == 0:
|
|
61
|
-
return samples
|
|
62
|
-
|
|
63
|
-
start_idx = non_silent[0]
|
|
64
|
-
end_idx = non_silent[-1]
|
|
65
|
-
return samples[start_idx:end_idx + 1]
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
class MultilingualNeuralEngine:
|
|
69
|
-
"""
|
|
70
|
-
Extensible Multilingual Orchestration Engine.
|
|
71
|
-
Dynamically routes linguistic sub-phrases to resident on-device neural acoustic backends.
|
|
72
|
-
"""
|
|
73
|
-
|
|
74
|
-
CANDIDATE_ONNX_BINARIES = [
|
|
75
|
-
"sherpa-onnx-offline-tts",
|
|
76
|
-
str(Path.home() / ".local" / "bin" / "sherpa-onnx-offline-tts"),
|
|
77
|
-
str(Path.home() / "sherpa-onnx-offline-tts"),
|
|
78
|
-
"/data/data/com.termux/files/home/.local/bin/sherpa-onnx-offline-tts",
|
|
79
|
-
"/data/data/com.termux/files/usr/bin/sherpa-onnx-offline-tts",
|
|
80
|
-
]
|
|
81
|
-
|
|
82
|
-
CANDIDATE_NCNN_BINARIES = [
|
|
83
|
-
"sherpa-ncnn-offline-tts",
|
|
84
|
-
str(Path.home() / ".local" / "bin" / "sherpa-ncnn-offline-tts"),
|
|
85
|
-
str(Path.home() / "sherpa-ncnn" / "build-vulkan" / "bin" / "sherpa-ncnn-offline-tts"),
|
|
86
|
-
"/data/data/com.termux/files/home/.local/bin/sherpa-ncnn-offline-tts",
|
|
87
|
-
"/data/data/com.termux/files/usr/bin/sherpa-ncnn-offline-tts",
|
|
88
|
-
]
|
|
89
|
-
|
|
90
|
-
STANDARD_MODEL_DIRS = [
|
|
91
|
-
Path.home() / ".cache" / "termux-tts" / "models",
|
|
92
|
-
Path.home() / "vits-mimic3-ko_KO-kss_low",
|
|
93
|
-
Path.home() / "vits-piper-en_US-lessac-medium",
|
|
94
|
-
Path.home() / "ncnn-vits-piper-en_US-lessac-high-fp16",
|
|
95
|
-
Path("/data/data/com.termux/files/home/.cache/termux-tts/models"),
|
|
96
|
-
]
|
|
97
|
-
|
|
98
|
-
def __init__(
|
|
99
|
-
self,
|
|
100
|
-
threads: int = 4,
|
|
101
|
-
device: str = "cpu",
|
|
102
|
-
sample_rate: int = 22050,
|
|
103
|
-
model_path_map: Optional[Dict[str, str]] = None,
|
|
104
|
-
):
|
|
105
|
-
self.threads = threads
|
|
106
|
-
self.device = device.lower()
|
|
107
|
-
self.sample_rate = sample_rate
|
|
108
|
-
self.model_path_map = model_path_map or {}
|
|
109
|
-
self.tokenizer = MultilingualTokenizer()
|
|
110
|
-
self._engines: Dict[str, Any] = {}
|
|
111
|
-
self._resident_manager = None
|
|
112
|
-
self._is_closed = False
|
|
113
|
-
|
|
114
|
-
# Cache directories
|
|
115
|
-
self.work_dir = Path.home() / ".cache" / "termux-tts" / "chunks"
|
|
116
|
-
self.work_dir.mkdir(parents=True, exist_ok=True)
|
|
117
|
-
|
|
118
|
-
def _get_resident_manager(self):
|
|
119
|
-
if self._resident_manager is None:
|
|
120
|
-
try:
|
|
121
|
-
from .engine_sherpa_capi import SherpaResidentManager
|
|
122
|
-
self._resident_manager = SherpaResidentManager.get_instance()
|
|
123
|
-
except Exception as err:
|
|
124
|
-
logger.debug("Sherpa C-API resident manager unavailable: %s", err)
|
|
125
|
-
return self._resident_manager
|
|
126
|
-
|
|
127
|
-
def register_engine(self, language: str, engine_instance: Any) -> None:
|
|
128
|
-
"""Register or override a neural backend engine for a specific language code."""
|
|
129
|
-
self._engines[language.lower()] = engine_instance
|
|
130
|
-
|
|
131
|
-
def _find_binary(self, candidates: List[str]) -> Optional[str]:
|
|
132
|
-
for c in candidates:
|
|
133
|
-
found = shutil.which(c) if not os.path.isabs(c) else c
|
|
134
|
-
if found and os.path.isfile(found) and (os.access(found, os.X_OK) or os.name == "nt"):
|
|
135
|
-
return str(found)
|
|
136
|
-
return None
|
|
137
|
-
|
|
138
|
-
def _resolve_korean_assets(self) -> Dict[str, str]:
|
|
139
|
-
custom = self.model_path_map.get("ko")
|
|
140
|
-
from .hardware import get_unified_model_search_dirs
|
|
141
|
-
dirs = [Path(custom)] if custom else get_unified_model_search_dirs("tts")
|
|
142
|
-
for d in dirs:
|
|
143
|
-
cand = d / "vits-mimic3-ko_KO-kss_low" if not d.name.startswith("vits") else d
|
|
144
|
-
onnx = cand / "ko_KO-kss_low.onnx"
|
|
145
|
-
tokens = cand / "tokens.txt"
|
|
146
|
-
espeak = cand / "espeak-ng-data"
|
|
147
|
-
if onnx.is_file() and tokens.is_file() and espeak.is_dir():
|
|
148
|
-
return {"onnx": str(onnx), "tokens": str(tokens), "data_dir": str(espeak)}
|
|
149
|
-
raise TTSModelLoadError("[FAIL-FAST] Korean VITS model assets (ko_KO-kss_low) not found in cache.")
|
|
150
|
-
|
|
151
|
-
def _resolve_english_assets(self) -> str:
|
|
152
|
-
custom = self.model_path_map.get("en")
|
|
153
|
-
from .hardware import get_unified_model_search_dirs
|
|
154
|
-
dirs = [Path(custom)] if custom else get_unified_model_search_dirs("tts")
|
|
155
|
-
for d in dirs:
|
|
156
|
-
cand = d / "ncnn-vits-piper-en_US-lessac-high-fp16" if not "lessac" in d.name else d
|
|
157
|
-
if (cand / "decoder.ncnn.bin").is_file():
|
|
158
|
-
return str(cand)
|
|
159
|
-
raise TTSModelLoadError("[FAIL-FAST] English NCNN model assets (lessac-high-fp16) not found in cache.")
|
|
160
|
-
|
|
161
|
-
def _synthesize_chunk_audio(self, text: str, lang: str, speed: float = 1.0) -> AudioBuffer:
|
|
162
|
-
"""Synthesize a single linguistic chunk via fastest on-device resident engine."""
|
|
163
|
-
# 1. Check custom registered engines
|
|
164
|
-
if lang in self._engines:
|
|
165
|
-
chunk_res = self._engines[lang].synthesize(text, speed=speed)
|
|
166
|
-
return chunk_res.audio_buffer
|
|
167
|
-
|
|
168
|
-
# 2. In-Memory Resident C-API Engine (Zero cold-start overhead)
|
|
169
|
-
mgr = self._get_resident_manager()
|
|
170
|
-
if mgr is not None:
|
|
171
|
-
try:
|
|
172
|
-
session = mgr.get_session(
|
|
173
|
-
language=lang,
|
|
174
|
-
custom_model_dir=self.model_path_map.get(lang),
|
|
175
|
-
threads=self.threads,
|
|
176
|
-
sample_rate=self.sample_rate,
|
|
177
|
-
)
|
|
178
|
-
return session.synthesize(text, speed=speed)
|
|
179
|
-
except TTSModelLoadError:
|
|
180
|
-
# If model assets missing or C-API failed, proceed to fallback/fail-fast
|
|
181
|
-
raise
|
|
182
|
-
except Exception as e:
|
|
183
|
-
logger.debug("Resident C-API execution bypassed: %s", e)
|
|
184
|
-
|
|
185
|
-
# 3. Fallback to Subprocess for Korean (ONNX CPU)
|
|
186
|
-
if lang == "ko":
|
|
187
|
-
onnx_bin = self._find_binary(self.CANDIDATE_ONNX_BINARIES)
|
|
188
|
-
if not onnx_bin:
|
|
189
|
-
raise TTSModelLoadError("[FAIL-FAST] sherpa-onnx-offline-tts binary not found for Korean synthesis.")
|
|
190
|
-
assets = self._resolve_korean_assets()
|
|
191
|
-
|
|
192
|
-
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp:
|
|
193
|
-
temp_wav = tmp.name
|
|
194
|
-
|
|
195
|
-
try:
|
|
196
|
-
cmd = [
|
|
197
|
-
onnx_bin,
|
|
198
|
-
f"--vits-model={assets['onnx']}",
|
|
199
|
-
f"--vits-tokens={assets['tokens']}",
|
|
200
|
-
f"--vits-data-dir={assets['data_dir']}",
|
|
201
|
-
f"--num-threads={self.threads}",
|
|
202
|
-
f"--speed={speed:.2f}",
|
|
203
|
-
f"--output-filename={temp_wav}",
|
|
204
|
-
text
|
|
205
|
-
]
|
|
206
|
-
env = os.environ.copy()
|
|
207
|
-
env["LANG"] = "C.UTF-8"
|
|
208
|
-
env["LC_ALL"] = "C.UTF-8"
|
|
209
|
-
res = subprocess.run(cmd, capture_output=True, text=True, env=env, timeout=60)
|
|
210
|
-
if res.returncode != 0 or not os.path.exists(temp_wav):
|
|
211
|
-
raise TTSInferenceError(f"[FAIL-FAST] Korean chunk failed for '{text}': {res.stderr}")
|
|
212
|
-
return AudioBuffer.from_wav_file(temp_wav)
|
|
213
|
-
finally:
|
|
214
|
-
if os.path.exists(temp_wav):
|
|
215
|
-
try:
|
|
216
|
-
os.remove(temp_wav)
|
|
217
|
-
except OSError:
|
|
218
|
-
pass
|
|
219
|
-
|
|
220
|
-
# 4. Fallback to Subprocess for English (NCNN CPU)
|
|
221
|
-
elif lang == "en":
|
|
222
|
-
ncnn_bin = self._find_binary(self.CANDIDATE_NCNN_BINARIES)
|
|
223
|
-
if not ncnn_bin:
|
|
224
|
-
raise TTSModelLoadError("[FAIL-FAST] sherpa-ncnn-offline-tts binary not found for English synthesis.")
|
|
225
|
-
model_dir = self._resolve_english_assets()
|
|
226
|
-
|
|
227
|
-
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp:
|
|
228
|
-
temp_wav = tmp.name
|
|
229
|
-
|
|
230
|
-
try:
|
|
231
|
-
use_vk = "1" if self.device in ("vulkan", "gpu") else "0"
|
|
232
|
-
cmd = [
|
|
233
|
-
ncnn_bin,
|
|
234
|
-
f"--vits-model-dir={model_dir}",
|
|
235
|
-
f"--use-vulkan-compute={use_vk}",
|
|
236
|
-
f"--num-threads={self.threads}",
|
|
237
|
-
f"--output-filename={temp_wav}",
|
|
238
|
-
text
|
|
239
|
-
]
|
|
240
|
-
env = os.environ.copy()
|
|
241
|
-
env["LANG"] = "C.UTF-8"
|
|
242
|
-
env["LC_ALL"] = "C.UTF-8"
|
|
243
|
-
res = subprocess.run(cmd, capture_output=True, text=True, env=env, timeout=60)
|
|
244
|
-
if res.returncode != 0 or not os.path.exists(temp_wav) or os.path.getsize(temp_wav) == 0:
|
|
245
|
-
if "empty ids for word" in res.stderr:
|
|
246
|
-
logger.warning("OOV word '%s' in English dictionary; inserting soft breath pause.", text)
|
|
247
|
-
return AudioBuffer(np.zeros(int(self.sample_rate * 0.1), dtype=np.float32), sample_rate=self.sample_rate)
|
|
248
|
-
raise TTSInferenceError(f"[FAIL-FAST] English chunk failed for '{text}': {res.stderr}")
|
|
249
|
-
return AudioBuffer.from_wav_file(temp_wav)
|
|
250
|
-
finally:
|
|
251
|
-
if os.path.exists(temp_wav):
|
|
252
|
-
try:
|
|
253
|
-
os.remove(temp_wav)
|
|
254
|
-
except OSError:
|
|
255
|
-
pass
|
|
256
|
-
|
|
257
|
-
# 5. Extended Languages (ja, zh, hi, ru, es, fr, de, ar)
|
|
258
|
-
elif lang in ("ja", "zh", "hi", "ru", "es", "fr", "de", "ar"):
|
|
259
|
-
custom_model = self.model_path_map.get(lang)
|
|
260
|
-
if not custom_model:
|
|
261
|
-
from .installer import OFFICIAL_NEURAL_MODELS
|
|
262
|
-
meta = OFFICIAL_NEURAL_MODELS.get(lang, {})
|
|
263
|
-
lang_name = meta.get("language_name", lang.upper())
|
|
264
|
-
repo = meta.get("repo", f"csukuangfj/vits-{lang}")
|
|
265
|
-
model_name = meta.get("name", f"vits-{lang}")
|
|
266
|
-
canonical_dest = str((Path.home() / "models" / "tts" / model_name).resolve())
|
|
267
|
-
raise TTSModelLoadError(
|
|
268
|
-
f"[FAIL-FAST] Neural speech model for language '{lang}' ({lang_name}) is not installed.\n\n"
|
|
269
|
-
f" To install this model automatically, run:\n"
|
|
270
|
-
f" termux-tts install --models {lang}\n\n"
|
|
271
|
-
f" Or manually download via git clone:\n"
|
|
272
|
-
f" git clone https://huggingface.co/{repo} {canonical_dest}\n\n"
|
|
273
|
-
f" After installation, re-run your synthesis command!"
|
|
274
|
-
)
|
|
275
|
-
from .engine_sherpa import SherpaNeuralEngine
|
|
276
|
-
engine = SherpaNeuralEngine(model_path=custom_model, language=lang, threads=self.threads, sample_rate=self.sample_rate)
|
|
277
|
-
self._engines[lang] = engine
|
|
278
|
-
return engine.synthesize(text, speed=speed).audio_buffer
|
|
279
|
-
|
|
280
|
-
raise TTSModelLoadError(f"[FAIL-FAST] Unsupported language '{lang}'. Supported languages: ['ko', 'en', 'ja', 'zh', 'hi', 'ru', 'es', 'fr', 'de', 'ar'].")
|
|
281
|
-
|
|
282
|
-
def synthesize(
|
|
283
|
-
self,
|
|
284
|
-
text: str,
|
|
285
|
-
output: Optional[str] = None,
|
|
286
|
-
speed: float = 1.0,
|
|
287
|
-
language: Optional[str] = None,
|
|
288
|
-
mode: str = "unified",
|
|
289
|
-
) -> MultilingualResult:
|
|
290
|
-
"""
|
|
291
|
-
Synthesizes speech with natural prosody and zero cross-speaker distortion.
|
|
292
|
-
Defaults to 'unified' single-pass mode (phonetic transliteration preserving prosody).
|
|
293
|
-
'stitch' mode retains explicit multi-model chunk concatenation when requested.
|
|
294
|
-
"""
|
|
295
|
-
if self._is_closed:
|
|
296
|
-
raise TTSInferenceError("Cannot synthesize: Multilingual session is closed.")
|
|
297
|
-
|
|
298
|
-
clean_text = text.strip()
|
|
299
|
-
if not clean_text:
|
|
300
|
-
raise TTSInferenceError("Cannot synthesize empty text.")
|
|
301
|
-
|
|
302
|
-
t0 = time.perf_counter()
|
|
303
|
-
target_sample_rate = self.sample_rate
|
|
304
|
-
|
|
305
|
-
# -------------------------------------------------------------------
|
|
306
|
-
# Mode 1: Unified Single-Pass Pipeline (BigTech Architecture Standard)
|
|
307
|
-
# -------------------------------------------------------------------
|
|
308
|
-
has_korean = bool(re.search(r"[\uac00-\ud7a3]", clean_text))
|
|
309
|
-
has_latin = bool(re.search(r"[a-zA-Z]", clean_text))
|
|
310
|
-
|
|
311
|
-
if mode == "unified" and (language == "ko" or (language is None and has_korean)):
|
|
312
|
-
from .transliteration import transliterate_mixed_text
|
|
313
|
-
unified_text = transliterate_mixed_text(clean_text)
|
|
314
|
-
logger.info("[Multilingual] Unified single-pass execution: '%s' -> '%s'", clean_text, unified_text)
|
|
315
|
-
|
|
316
|
-
chunk_buf = self._synthesize_chunk_audio(unified_text, "ko", speed=speed)
|
|
317
|
-
elapsed_ms = (time.perf_counter() - t0) * 1000.0
|
|
318
|
-
dur_sec = chunk_buf.duration_seconds
|
|
319
|
-
rtf = (elapsed_ms / 1000.0) / max(0.001, dur_sec)
|
|
320
|
-
|
|
321
|
-
if output:
|
|
322
|
-
chunk_buf.save(output)
|
|
323
|
-
|
|
324
|
-
dummy_chunk = LanguageChunk(text=clean_text, language="ko", pause_after=0.0)
|
|
325
|
-
return MultilingualResult(
|
|
326
|
-
text=clean_text,
|
|
327
|
-
audio_buffer=chunk_buf,
|
|
328
|
-
sample_rate=target_sample_rate,
|
|
329
|
-
duration_sec=dur_sec,
|
|
330
|
-
elapsed_ms=elapsed_ms,
|
|
331
|
-
rtf=rtf,
|
|
332
|
-
chunks=[dummy_chunk],
|
|
333
|
-
languages_detected=["ko", "en"] if has_latin else ["ko"],
|
|
334
|
-
backend="UNIFIED_SINGLE_PASS_NEURAL",
|
|
335
|
-
)
|
|
336
|
-
|
|
337
|
-
# -------------------------------------------------------------------
|
|
338
|
-
# Mode 2: Multi-Script Chunk Concatenation (Explicit Stitch Mode)
|
|
339
|
-
# -------------------------------------------------------------------
|
|
340
|
-
chunks = self.tokenizer.tokenize(clean_text, force_language=language)
|
|
341
|
-
if not chunks:
|
|
342
|
-
raise TTSInferenceError("Tokenization yielded no synthesizable segments.")
|
|
343
|
-
|
|
344
|
-
detected_langs = list(dict.fromkeys(c.language for c in chunks))
|
|
345
|
-
|
|
346
|
-
assembled_samples: List[np.ndarray] = []
|
|
347
|
-
|
|
348
|
-
for i, chunk in enumerate(chunks):
|
|
349
|
-
chunk_buf = self._synthesize_chunk_audio(chunk.text, chunk.language, speed=speed)
|
|
350
|
-
|
|
351
|
-
# Get raw float32 samples and trim acoustic silence
|
|
352
|
-
raw_samples = chunk_buf.samples
|
|
353
|
-
trimmed = trim_silence_samples(raw_samples, threshold=0.01)
|
|
354
|
-
assembled_samples.append(trimmed)
|
|
355
|
-
|
|
356
|
-
# Add context-aware pause
|
|
357
|
-
if chunk.pause_after > 0:
|
|
358
|
-
pause_len = int(target_sample_rate * chunk.pause_after)
|
|
359
|
-
silence_gap = np.zeros(pause_len, dtype=np.float32)
|
|
360
|
-
assembled_samples.append(silence_gap)
|
|
361
|
-
|
|
362
|
-
if assembled_samples:
|
|
363
|
-
final_samples = np.concatenate(assembled_samples)
|
|
364
|
-
else:
|
|
365
|
-
final_samples = np.zeros(0, dtype=np.float32)
|
|
366
|
-
|
|
367
|
-
combined_buffer = AudioBuffer(final_samples, sample_rate=target_sample_rate)
|
|
368
|
-
|
|
369
|
-
elapsed_ms = (time.perf_counter() - t0) * 1000.0
|
|
370
|
-
dur_sec = combined_buffer.duration_seconds
|
|
371
|
-
rtf = (elapsed_ms / 1000.0) / max(0.001, dur_sec)
|
|
372
|
-
|
|
373
|
-
if output:
|
|
374
|
-
combined_buffer.save(output)
|
|
375
|
-
|
|
376
|
-
return MultilingualResult(
|
|
377
|
-
text=clean_text,
|
|
378
|
-
audio_buffer=combined_buffer,
|
|
379
|
-
sample_rate=target_sample_rate,
|
|
380
|
-
duration_sec=dur_sec,
|
|
381
|
-
elapsed_ms=elapsed_ms,
|
|
382
|
-
rtf=rtf,
|
|
383
|
-
chunks=chunks,
|
|
384
|
-
languages_detected=detected_langs,
|
|
385
|
-
)
|
|
386
|
-
|
|
387
|
-
def close(self) -> None:
|
|
388
|
-
"""Cleanly terminate all engine instances and release resident memory."""
|
|
389
|
-
self._is_closed = True
|
|
390
|
-
for engine in self._engines.values():
|
|
391
|
-
if hasattr(engine, "close"):
|
|
392
|
-
try:
|
|
393
|
-
engine.close()
|
|
394
|
-
except Exception:
|
|
395
|
-
pass
|
|
396
|
-
if self._resident_manager is not None:
|
|
397
|
-
try:
|
|
398
|
-
self._resident_manager.close_all()
|
|
399
|
-
except Exception:
|
|
400
|
-
pass
|
|
401
|
-
|
|
402
|
-
def __del__(self) -> None:
|
|
403
|
-
self.close()
|
|
404
|
-
|
|
405
|
-
def __enter__(self):
|
|
406
|
-
return self
|
|
407
|
-
|
|
408
|
-
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
409
|
-
self.close()
|