ownvoice-cli 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ownvoice/__init__.py +10 -0
- ownvoice/cli.py +216 -0
- ownvoice/data.py +139 -0
- ownvoice/infer.py +227 -0
- ownvoice/score.py +90 -0
- ownvoice/train.py +477 -0
- ownvoice_cli-0.1.1.dist-info/METADATA +190 -0
- ownvoice_cli-0.1.1.dist-info/RECORD +11 -0
- ownvoice_cli-0.1.1.dist-info/WHEEL +4 -0
- ownvoice_cli-0.1.1.dist-info/entry_points.txt +2 -0
- ownvoice_cli-0.1.1.dist-info/licenses/LICENSE +21 -0
ownvoice/__init__.py
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"""OwnVoice: train a LoRA voice adapter for pocket-tts, own the resulting model.
|
|
2
|
+
|
|
3
|
+
OwnVoice wraps kyutai-labs/pocket-tts (MIT-licensed, CPU-capable local
|
|
4
|
+
text-to-speech) with a LoRA fine-tuning workflow. The output is an adapter
|
|
5
|
+
file you keep and run yourself, not an API subscription.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
__version__ = "0.1.0"
|
|
9
|
+
|
|
10
|
+
__all__ = ["__version__"]
|
ownvoice/cli.py
ADDED
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
"""OwnVoice CLI: `check`, `train`, `infer` subcommands.
|
|
2
|
+
|
|
3
|
+
Every subcommand supports `--json` for a structured, machine-parseable
|
|
4
|
+
output mode alongside the default human-readable text, so a script or an
|
|
5
|
+
agent invoking `ownvoice` programmatically can parse the result reliably.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json as json_module
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Optional
|
|
13
|
+
|
|
14
|
+
import typer
|
|
15
|
+
|
|
16
|
+
from ownvoice import __version__
|
|
17
|
+
from ownvoice.infer import infer as run_infer
|
|
18
|
+
from ownvoice.train import (
|
|
19
|
+
DEFAULT_EPOCHS,
|
|
20
|
+
DEFAULT_EVAL_TEXT,
|
|
21
|
+
DEFAULT_LEARNING_RATE,
|
|
22
|
+
DEFAULT_LORA_ALPHA,
|
|
23
|
+
DEFAULT_LORA_DROPOUT,
|
|
24
|
+
DEFAULT_LORA_RANK,
|
|
25
|
+
DEFAULT_OUT_DIR,
|
|
26
|
+
TrainingConfig,
|
|
27
|
+
check_compatibility,
|
|
28
|
+
)
|
|
29
|
+
from ownvoice.train import train as run_train
|
|
30
|
+
|
|
31
|
+
app = typer.Typer(
|
|
32
|
+
name="ownvoice",
|
|
33
|
+
help="Train a LoRA voice adapter for pocket-tts and own the resulting model. No API lock-in.",
|
|
34
|
+
no_args_is_help=True,
|
|
35
|
+
add_completion=False,
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _version_callback(value: bool) -> None:
|
|
40
|
+
if value:
|
|
41
|
+
typer.echo(f"ownvoice {__version__}")
|
|
42
|
+
raise typer.Exit()
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@app.callback()
|
|
46
|
+
def main(
|
|
47
|
+
version: Optional[bool] = typer.Option(
|
|
48
|
+
None,
|
|
49
|
+
"--version",
|
|
50
|
+
callback=_version_callback,
|
|
51
|
+
is_eager=True,
|
|
52
|
+
help="Print the OwnVoice version and exit.",
|
|
53
|
+
),
|
|
54
|
+
) -> None:
|
|
55
|
+
"""OwnVoice: own your voice model, no API lock-in."""
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
@app.command()
|
|
59
|
+
def check(
|
|
60
|
+
json_output: bool = typer.Option(
|
|
61
|
+
False, "--json", help="Print machine-readable JSON instead of human-readable text."
|
|
62
|
+
),
|
|
63
|
+
) -> None:
|
|
64
|
+
"""Free, CPU-only compatibility check: load pocket-tts and dry-run the LoRA injection.
|
|
65
|
+
|
|
66
|
+
No GPU and no training required. Run this before renting a GPU for
|
|
67
|
+
`ownvoice train` -- it is the Day-0 compatibility spike, exposed as a
|
|
68
|
+
command.
|
|
69
|
+
|
|
70
|
+
Example:
|
|
71
|
+
ownvoice check
|
|
72
|
+
"""
|
|
73
|
+
result = check_compatibility()
|
|
74
|
+
|
|
75
|
+
if json_output:
|
|
76
|
+
typer.echo(
|
|
77
|
+
json_module.dumps(
|
|
78
|
+
{
|
|
79
|
+
"success": result.success,
|
|
80
|
+
"message": result.message,
|
|
81
|
+
"module_tree": result.module_tree,
|
|
82
|
+
}
|
|
83
|
+
)
|
|
84
|
+
)
|
|
85
|
+
else:
|
|
86
|
+
if result.success:
|
|
87
|
+
typer.secho(f"[ownvoice check] PASS: {result.message}", fg=typer.colors.GREEN)
|
|
88
|
+
else:
|
|
89
|
+
typer.secho(f"[ownvoice check] FAIL: {result.message}", fg=typer.colors.RED)
|
|
90
|
+
if result.module_tree:
|
|
91
|
+
typer.echo("\nModule tree (for debugging / for the issue #30 blocker post):")
|
|
92
|
+
typer.echo(result.module_tree)
|
|
93
|
+
|
|
94
|
+
raise typer.Exit(code=0 if result.success else 1)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
@app.command()
|
|
98
|
+
def train(
|
|
99
|
+
voice_clips: Path = typer.Option(
|
|
100
|
+
...,
|
|
101
|
+
"--voice-clips",
|
|
102
|
+
exists=True,
|
|
103
|
+
file_okay=False,
|
|
104
|
+
dir_okay=True,
|
|
105
|
+
help="Directory of .wav voice-clip recordings to train from. Required.",
|
|
106
|
+
),
|
|
107
|
+
out: Path = typer.Option(
|
|
108
|
+
DEFAULT_OUT_DIR, "--out", help="Directory to write adapter.safetensors + metadata.json to."
|
|
109
|
+
),
|
|
110
|
+
epochs: int = typer.Option(DEFAULT_EPOCHS, "--epochs", min=1, help="Number of training epochs."),
|
|
111
|
+
lora_rank: int = typer.Option(DEFAULT_LORA_RANK, "--lora-rank", min=1, help="LoRA rank."),
|
|
112
|
+
lora_alpha: int = typer.Option(DEFAULT_LORA_ALPHA, "--lora-alpha", min=1, help="LoRA alpha."),
|
|
113
|
+
lora_dropout: float = typer.Option(
|
|
114
|
+
DEFAULT_LORA_DROPOUT, "--lora-dropout", min=0.0, max=1.0, help="LoRA dropout."
|
|
115
|
+
),
|
|
116
|
+
learning_rate: float = typer.Option(
|
|
117
|
+
DEFAULT_LEARNING_RATE, "--learning-rate", help="Optimizer learning rate."
|
|
118
|
+
),
|
|
119
|
+
eval_text: str = typer.Option(
|
|
120
|
+
DEFAULT_EVAL_TEXT,
|
|
121
|
+
"--eval-text",
|
|
122
|
+
help="Sentence synthesized after training to score against the reference voice.",
|
|
123
|
+
),
|
|
124
|
+
json_output: bool = typer.Option(
|
|
125
|
+
False, "--json", help="Print machine-readable JSON instead of human-readable text."
|
|
126
|
+
),
|
|
127
|
+
) -> None:
|
|
128
|
+
"""Train a LoRA voice adapter from a directory of .wav voice clips.
|
|
129
|
+
|
|
130
|
+
Only --voice-clips is required; every other flag has a sensible default.
|
|
131
|
+
Exits 0 on a successful run whether or not the similarity score clears
|
|
132
|
+
the usable threshold (0.75) -- a below-threshold result is a labeled
|
|
133
|
+
outcome with a next step, not a crash. Only a data-loading failure or a
|
|
134
|
+
caught PEFT-injection failure exits non-zero.
|
|
135
|
+
|
|
136
|
+
Example:
|
|
137
|
+
ownvoice train --voice-clips ./my-voice-clips
|
|
138
|
+
"""
|
|
139
|
+
config = TrainingConfig(
|
|
140
|
+
voice_clips_dir=voice_clips,
|
|
141
|
+
out_dir=out,
|
|
142
|
+
epochs=epochs,
|
|
143
|
+
lora_rank=lora_rank,
|
|
144
|
+
lora_alpha=lora_alpha,
|
|
145
|
+
lora_dropout=lora_dropout,
|
|
146
|
+
learning_rate=learning_rate,
|
|
147
|
+
eval_text=eval_text,
|
|
148
|
+
)
|
|
149
|
+
result = run_train(config)
|
|
150
|
+
|
|
151
|
+
if json_output:
|
|
152
|
+
typer.echo(
|
|
153
|
+
json_module.dumps(
|
|
154
|
+
{
|
|
155
|
+
"success": result.success,
|
|
156
|
+
"out_dir": str(result.out_dir) if result.out_dir else None,
|
|
157
|
+
"similarity_score": result.similarity_score,
|
|
158
|
+
"usable": result.usable,
|
|
159
|
+
"message": result.message,
|
|
160
|
+
"infer_command": result.infer_command,
|
|
161
|
+
}
|
|
162
|
+
)
|
|
163
|
+
)
|
|
164
|
+
else:
|
|
165
|
+
if result.success:
|
|
166
|
+
label = "USABLE ADAPTER" if result.usable else "BELOW THRESHOLD"
|
|
167
|
+
color = typer.colors.GREEN if result.usable else typer.colors.YELLOW
|
|
168
|
+
typer.secho(f"[ownvoice train] {label}", fg=color, bold=True)
|
|
169
|
+
typer.echo(result.message)
|
|
170
|
+
else:
|
|
171
|
+
typer.secho("[ownvoice train] FAILED", fg=typer.colors.RED, bold=True)
|
|
172
|
+
typer.echo(result.message)
|
|
173
|
+
|
|
174
|
+
raise typer.Exit(code=0 if result.success else 1)
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
@app.command()
|
|
178
|
+
def infer(
|
|
179
|
+
adapter: Path = typer.Option(
|
|
180
|
+
..., "--adapter", exists=True, help="Path to a trained adapter.safetensors file. Required."
|
|
181
|
+
),
|
|
182
|
+
text: str = typer.Option(..., "--text", help="Text to synthesize in the trained voice. Required."),
|
|
183
|
+
out: Path = typer.Option(Path("./ownvoice-output.wav"), "--out", help="Output .wav file path."),
|
|
184
|
+
reference_audio: Optional[Path] = typer.Option(
|
|
185
|
+
None,
|
|
186
|
+
"--reference-audio",
|
|
187
|
+
help="Override the reference clip OwnVoice recorded in metadata.json at train time.",
|
|
188
|
+
),
|
|
189
|
+
json_output: bool = typer.Option(
|
|
190
|
+
False, "--json", help="Print machine-readable JSON instead of human-readable text."
|
|
191
|
+
),
|
|
192
|
+
) -> None:
|
|
193
|
+
"""Generate speech in the trained voice from a saved adapter, and save it to a .wav file.
|
|
194
|
+
|
|
195
|
+
Example:
|
|
196
|
+
ownvoice infer --adapter ./ownvoice-adapter/adapter.safetensors --text "Hello, this is my own voice."
|
|
197
|
+
"""
|
|
198
|
+
try:
|
|
199
|
+
out_path = run_infer(adapter, text, out_path=out, reference_audio_override=reference_audio)
|
|
200
|
+
except Exception as exc: # noqa: BLE001
|
|
201
|
+
if json_output:
|
|
202
|
+
typer.echo(json_module.dumps({"success": False, "error": str(exc)}))
|
|
203
|
+
else:
|
|
204
|
+
typer.secho(f"[ownvoice infer] FAILED: {exc}", fg=typer.colors.RED, bold=True)
|
|
205
|
+
raise typer.Exit(code=1) from exc
|
|
206
|
+
|
|
207
|
+
if json_output:
|
|
208
|
+
typer.echo(json_module.dumps({"success": True, "out_path": str(out_path)}))
|
|
209
|
+
else:
|
|
210
|
+
typer.secho(f"[ownvoice infer] Wrote {out_path}", fg=typer.colors.GREEN)
|
|
211
|
+
|
|
212
|
+
raise typer.Exit(code=0)
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
if __name__ == "__main__":
|
|
216
|
+
app()
|
ownvoice/data.py
ADDED
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
"""Voice-clip loading and validation for OwnVoice training data.
|
|
2
|
+
|
|
3
|
+
pocket-tts's own sample rate is read from the loaded model at train time
|
|
4
|
+
(`TTSModel.sample_rate`); this module only validates raw input files and
|
|
5
|
+
normalizes them to whatever target rate the caller asks for.
|
|
6
|
+
|
|
7
|
+
Audio I/O uses `soundfile` directly rather than `torchaudio.load`/
|
|
8
|
+
`torchaudio.info`: as of the torchaudio version this project pins against
|
|
9
|
+
(2.9+), both of those now route through an optional `torchcodec`/ffmpeg
|
|
10
|
+
backend and raise `ImportError` without it installed. `soundfile` (libsndfile)
|
|
11
|
+
reads plain .wav files with no extra runtime dependency. Resampling still
|
|
12
|
+
goes through `torchaudio.transforms.Resample` per the locked design decision,
|
|
13
|
+
since that's a pure-tensor operation with no I/O backend involved.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import dataclasses
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
|
|
21
|
+
import soundfile as sf
|
|
22
|
+
import torch
|
|
23
|
+
import torchaudio
|
|
24
|
+
|
|
25
|
+
MIN_CLIP_SECONDS = 1.0
|
|
26
|
+
MAX_CLIP_SECONDS = 30.0
|
|
27
|
+
SUPPORTED_SUFFIXES = {".wav"}
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class VoiceClipError(Exception):
|
|
31
|
+
"""Raised when a voice-clip directory or file fails validation."""
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclasses.dataclass(frozen=True)
|
|
35
|
+
class VoiceClip:
|
|
36
|
+
"""Metadata for one validated voice-clip file, not yet loaded into memory."""
|
|
37
|
+
|
|
38
|
+
path: Path
|
|
39
|
+
sample_rate: int
|
|
40
|
+
num_frames: int
|
|
41
|
+
num_channels: int
|
|
42
|
+
|
|
43
|
+
@property
|
|
44
|
+
def duration_seconds(self) -> float:
|
|
45
|
+
return self.num_frames / self.sample_rate
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _iter_candidate_files(voice_clips_dir: Path) -> list[Path]:
|
|
49
|
+
if not voice_clips_dir.exists():
|
|
50
|
+
raise VoiceClipError(f"Voice clips directory does not exist: {voice_clips_dir}")
|
|
51
|
+
if not voice_clips_dir.is_dir():
|
|
52
|
+
raise VoiceClipError(f"Voice clips path is not a directory: {voice_clips_dir}")
|
|
53
|
+
return sorted(
|
|
54
|
+
p
|
|
55
|
+
for p in voice_clips_dir.iterdir()
|
|
56
|
+
if p.is_file() and p.suffix.lower() in SUPPORTED_SUFFIXES
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def load_voice_clips(
|
|
61
|
+
voice_clips_dir: Path | str,
|
|
62
|
+
*,
|
|
63
|
+
min_seconds: float = MIN_CLIP_SECONDS,
|
|
64
|
+
max_seconds: float = MAX_CLIP_SECONDS,
|
|
65
|
+
) -> list[VoiceClip]:
|
|
66
|
+
"""Validate every .wav file in `voice_clips_dir` and return clip metadata.
|
|
67
|
+
|
|
68
|
+
A file that individually fails to decode, is empty, or falls outside the
|
|
69
|
+
duration bounds is skipped (not fatal on its own) -- only an empty result
|
|
70
|
+
set raises. Raises VoiceClipError with an actionable message either way
|
|
71
|
+
the directory itself doesn't exist/isn't a directory, has no .wav files
|
|
72
|
+
at all, or every candidate file was skipped.
|
|
73
|
+
"""
|
|
74
|
+
voice_clips_dir = Path(voice_clips_dir)
|
|
75
|
+
candidates = _iter_candidate_files(voice_clips_dir)
|
|
76
|
+
if not candidates:
|
|
77
|
+
raise VoiceClipError(
|
|
78
|
+
f"No .wav files found in {voice_clips_dir}. OwnVoice needs at least one "
|
|
79
|
+
"clean .wav recording of the target voice to train from."
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
clips: list[VoiceClip] = []
|
|
83
|
+
skipped: list[str] = []
|
|
84
|
+
for path in candidates:
|
|
85
|
+
try:
|
|
86
|
+
info = sf.info(str(path))
|
|
87
|
+
except Exception as exc: # noqa: BLE001 - soundfile raises backend-specific errors
|
|
88
|
+
skipped.append(f"{path.name}: could not read audio ({exc})")
|
|
89
|
+
continue
|
|
90
|
+
|
|
91
|
+
if info.frames <= 0:
|
|
92
|
+
skipped.append(f"{path.name}: empty audio (0 frames)")
|
|
93
|
+
continue
|
|
94
|
+
|
|
95
|
+
duration = info.frames / info.samplerate
|
|
96
|
+
if duration < min_seconds:
|
|
97
|
+
skipped.append(f"{path.name}: too short ({duration:.2f}s, minimum is {min_seconds}s)")
|
|
98
|
+
continue
|
|
99
|
+
if duration > max_seconds:
|
|
100
|
+
skipped.append(f"{path.name}: too long ({duration:.2f}s, maximum is {max_seconds}s per clip)")
|
|
101
|
+
continue
|
|
102
|
+
|
|
103
|
+
clips.append(
|
|
104
|
+
VoiceClip(
|
|
105
|
+
path=path,
|
|
106
|
+
sample_rate=info.samplerate,
|
|
107
|
+
num_frames=info.frames,
|
|
108
|
+
num_channels=info.channels,
|
|
109
|
+
)
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
if not clips:
|
|
113
|
+
detail = "; ".join(skipped) if skipped else "no readable clips"
|
|
114
|
+
raise VoiceClipError(
|
|
115
|
+
f"No usable voice clips found in {voice_clips_dir} ({detail}). "
|
|
116
|
+
f"Each clip must be a valid mono or stereo .wav file between "
|
|
117
|
+
f"{min_seconds}s and {max_seconds}s."
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
return clips
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def load_waveform(clip: VoiceClip, target_sample_rate: int) -> torch.Tensor:
|
|
124
|
+
"""Load a clip's waveform, downmix to mono, and resample to `target_sample_rate`.
|
|
125
|
+
|
|
126
|
+
Returns a 1D float32 tensor. This is the shared normalization path used
|
|
127
|
+
both by the training data pipeline and (via score.py) by evaluation, so
|
|
128
|
+
every clip reaches the model or the scorer in a consistent format
|
|
129
|
+
regardless of its original sample rate or channel count.
|
|
130
|
+
"""
|
|
131
|
+
audio, sample_rate = sf.read(str(clip.path), dtype="float32", always_2d=True)
|
|
132
|
+
# soundfile returns (frames, channels); torchaudio's convention is (channels, frames).
|
|
133
|
+
waveform = torch.from_numpy(audio.T).to(torch.float32)
|
|
134
|
+
if waveform.shape[0] > 1:
|
|
135
|
+
waveform = waveform.mean(dim=0, keepdim=True)
|
|
136
|
+
if sample_rate != target_sample_rate:
|
|
137
|
+
resampler = torchaudio.transforms.Resample(orig_freq=sample_rate, new_freq=target_sample_rate)
|
|
138
|
+
waveform = resampler(waveform)
|
|
139
|
+
return waveform.squeeze(0).to(torch.float32)
|
ownvoice/infer.py
ADDED
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
"""Load a trained LoRA adapter and generate speech with it.
|
|
2
|
+
|
|
3
|
+
Real-API note: pocket-tts's `TTSModel.generate_audio(model_state, text)`
|
|
4
|
+
requires a `model_state` dict built by `get_state_for_audio_prompt(audio_
|
|
5
|
+
conditioning, ...)`, and `audio_conditioning` has no default -- it must be a
|
|
6
|
+
real path, URL, or tensor (confirmed against pocket_tts/models/tts_model.py,
|
|
7
|
+
not assumed). OwnVoice's whole differentiation is that the *voice* comes
|
|
8
|
+
from the trained LoRA weights, not a runtime reference clip, but pocket-tts's
|
|
9
|
+
generation call still needs *some* audio prompt to build the initial
|
|
10
|
+
conditioning state. OwnVoice resolves this by reusing the same reference
|
|
11
|
+
clip that `ownvoice train` recorded in `metadata.json` (the first voice clip
|
|
12
|
+
in the training set) by default, with an optional override.
|
|
13
|
+
|
|
14
|
+
Verified for real this session (a real 2-epoch LoRA training run, followed
|
|
15
|
+
by a real `ownvoice infer` call against the resulting adapter, produced a
|
|
16
|
+
4.7s non-silent, finite, NaN-free .wav file): `load_adapter()` and
|
|
17
|
+
`generate_speech()` below are exercised end to end against real loaded
|
|
18
|
+
pocket-tts weights, not just reasoned about. Two things this session found
|
|
19
|
+
that the original (structurally reasonable but unverified) approach in
|
|
20
|
+
`generate_speech()` got wrong are documented on that function directly --
|
|
21
|
+
in short, no `base_model.flow_lm` swap is needed or safe, and OwnVoice
|
|
22
|
+
must pre-load and resample the reference clip itself rather than pass a
|
|
23
|
+
raw path/URL, because the publicly downloadable pocket-tts weights are the
|
|
24
|
+
non-voice-cloning checkpoint by default.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
from __future__ import annotations
|
|
28
|
+
|
|
29
|
+
import json
|
|
30
|
+
import os
|
|
31
|
+
from pathlib import Path
|
|
32
|
+
from typing import Any
|
|
33
|
+
|
|
34
|
+
import torch
|
|
35
|
+
|
|
36
|
+
from ownvoice.train import LORA_TARGET_MODULES, DEFAULT_LORA_ALPHA, DEFAULT_LORA_DROPOUT, DEFAULT_LORA_RANK
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class AdapterLoadError(Exception):
|
|
40
|
+
"""Raised when an adapter file or its sibling metadata.json can't be loaded."""
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _read_metadata(adapter_path: Path) -> dict[str, Any]:
|
|
44
|
+
metadata_path = adapter_path.parent / "metadata.json"
|
|
45
|
+
if not metadata_path.exists():
|
|
46
|
+
raise AdapterLoadError(
|
|
47
|
+
f"No metadata.json found next to {adapter_path}. OwnVoice adapters are always "
|
|
48
|
+
"saved with a sibling metadata.json (written by `ownvoice train`) -- if this "
|
|
49
|
+
"adapter was moved on its own, put metadata.json back alongside it."
|
|
50
|
+
)
|
|
51
|
+
return json.loads(metadata_path.read_text())
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def load_adapter(base_model: Any, adapter_path: Path | str):
|
|
55
|
+
"""Load a saved LoRA adapter (`adapter.safetensors`) onto pocket-tts's `flow_lm`.
|
|
56
|
+
|
|
57
|
+
Reconstructs the same LoraConfig used at train time (rank/alpha/
|
|
58
|
+
dropout, read back from the adapter's sibling metadata.json) and loads
|
|
59
|
+
the saved state dict via PEFT's `set_peft_model_state_dict`. The
|
|
60
|
+
reload round trip is verified against a real trained adapter this
|
|
61
|
+
session: `ownvoice train` saved a real adapter.safetensors via
|
|
62
|
+
`get_peft_model_state_dict`, and this function loaded it back and used
|
|
63
|
+
it for real generation via `ownvoice infer` without error.
|
|
64
|
+
"""
|
|
65
|
+
from peft import LoraConfig, get_peft_model, set_peft_model_state_dict
|
|
66
|
+
from safetensors.torch import load_file
|
|
67
|
+
|
|
68
|
+
adapter_path = Path(adapter_path)
|
|
69
|
+
metadata = _read_metadata(adapter_path)
|
|
70
|
+
train_config = metadata.get("config", {})
|
|
71
|
+
|
|
72
|
+
lora_config = LoraConfig(
|
|
73
|
+
r=train_config.get("lora_rank", DEFAULT_LORA_RANK),
|
|
74
|
+
lora_alpha=train_config.get("lora_alpha", DEFAULT_LORA_ALPHA),
|
|
75
|
+
lora_dropout=train_config.get("lora_dropout", DEFAULT_LORA_DROPOUT),
|
|
76
|
+
target_modules=LORA_TARGET_MODULES,
|
|
77
|
+
bias="none",
|
|
78
|
+
)
|
|
79
|
+
peft_model = get_peft_model(base_model.flow_lm, lora_config)
|
|
80
|
+
state_dict = load_file(str(adapter_path))
|
|
81
|
+
set_peft_model_state_dict(peft_model, state_dict)
|
|
82
|
+
peft_model.eval()
|
|
83
|
+
return peft_model
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def resolve_reference_audio(adapter_path: Path | str, override: Path | str | None = None) -> Path:
|
|
87
|
+
"""Pick the audio-conditioning clip generate_speech() needs.
|
|
88
|
+
|
|
89
|
+
Uses `override` if given, otherwise falls back to the reference clip
|
|
90
|
+
OwnVoice recorded in the adapter's metadata.json at training time.
|
|
91
|
+
"""
|
|
92
|
+
if override is not None:
|
|
93
|
+
return Path(override)
|
|
94
|
+
|
|
95
|
+
adapter_path = Path(adapter_path)
|
|
96
|
+
metadata = _read_metadata(adapter_path)
|
|
97
|
+
reference_clip = metadata.get("reference_clip")
|
|
98
|
+
if not reference_clip:
|
|
99
|
+
raise AdapterLoadError(
|
|
100
|
+
f"metadata.json next to {adapter_path} has no recorded reference_clip, and no "
|
|
101
|
+
"override was given. Pass one explicitly."
|
|
102
|
+
)
|
|
103
|
+
reference_path = Path(reference_clip)
|
|
104
|
+
if reference_path.is_absolute():
|
|
105
|
+
raise AdapterLoadError(
|
|
106
|
+
f"Recorded reference clip {reference_path} in metadata.json is an absolute path, "
|
|
107
|
+
"which OwnVoice does not trust from a shared adapter's metadata.json. Pass an "
|
|
108
|
+
"override path to a voice clip explicitly."
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
adapter_dir = adapter_path.parent
|
|
112
|
+
resolved_path = adapter_dir / reference_path
|
|
113
|
+
real_adapter_dir = os.path.realpath(adapter_dir)
|
|
114
|
+
real_resolved_path = os.path.realpath(resolved_path)
|
|
115
|
+
if os.path.commonpath([real_adapter_dir, real_resolved_path]) != real_adapter_dir:
|
|
116
|
+
raise AdapterLoadError(
|
|
117
|
+
f"Recorded reference clip {reference_path} in metadata.json resolves outside of "
|
|
118
|
+
f"{adapter_dir}, which OwnVoice does not trust from a shared adapter's "
|
|
119
|
+
"metadata.json. Pass an override path to a voice clip explicitly."
|
|
120
|
+
)
|
|
121
|
+
if not resolved_path.exists():
|
|
122
|
+
raise AdapterLoadError(
|
|
123
|
+
f"Recorded reference clip {resolved_path} no longer exists. Pass an override "
|
|
124
|
+
"path to a voice clip explicitly."
|
|
125
|
+
)
|
|
126
|
+
return resolved_path
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def generate_speech(base_model: Any, peft_model: Any, text: str, reference_audio_path: Path | str) -> torch.Tensor:
|
|
130
|
+
"""Generate speech audio for `text` using the base model wrapped with the LoRA adapter.
|
|
131
|
+
|
|
132
|
+
Verified against real loaded pocket-tts weights this session, and the
|
|
133
|
+
naive approach the previous TODO here anticipated turned out to be
|
|
134
|
+
wrong for two independent, real reasons:
|
|
135
|
+
|
|
136
|
+
1. No flow_lm swap is needed, and the previous
|
|
137
|
+
`base_model.flow_lm = peft_model` swap actively breaks generation.
|
|
138
|
+
PEFT's `get_peft_model()` replaces `base_model.flow_lm`'s targeted
|
|
139
|
+
`nn.Linear` submodules with LoRA layers IN PLACE -- it mutates the
|
|
140
|
+
same live module graph rather than returning a copy, so
|
|
141
|
+
`base_model.flow_lm` already carries the trained adapter's weights
|
|
142
|
+
by the time this function runs (confirmed: `id(base_model.flow_lm)`
|
|
143
|
+
is unchanged before/after `get_peft_model()`, and its Linear
|
|
144
|
+
submodules are already `peft.tuners.lora.layer.Linear` instances
|
|
145
|
+
afterward). Reassigning `base_model.flow_lm` to the outer
|
|
146
|
+
`PeftModel` wrapper instead breaks pocket-tts's own streaming state
|
|
147
|
+
machinery: `StatefulModule.get_state()` looks up KV-cache state by a
|
|
148
|
+
fixed `_module_absolute_name` string baked onto each submodule once
|
|
149
|
+
at `TTSModel.load_model()` time (e.g. "transformer.layers.0.
|
|
150
|
+
self_attn"), but `get_state_for_audio_prompt()` rebuilds the
|
|
151
|
+
`model_state` dict fresh via `init_states(base_model.flow_lm, ...)`,
|
|
152
|
+
which keys it off `named_modules()` on whatever module is currently
|
|
153
|
+
assigned to `base_model.flow_lm`. PEFT's `PeftModel`/`LoraModel`
|
|
154
|
+
wrapper inserts a "base_model.model." prefix into those paths, so
|
|
155
|
+
swapping to the wrapper produces a `model_state` dict whose keys no
|
|
156
|
+
longer match the fixed `_module_absolute_name` strings, and
|
|
157
|
+
generation fails with `KeyError: 'transformer.layers.0.self_attn'`
|
|
158
|
+
-- reproduced against real weights this session.
|
|
159
|
+
2. The publicly downloadable pocket-tts weights
|
|
160
|
+
(`kyutai/pocket-tts-without-voice-cloning`, the checkpoint
|
|
161
|
+
`TTSModel.load_model()` falls back to whenever the gated
|
|
162
|
+
`kyutai/pocket-tts` voice-cloning weights aren't accessible, which
|
|
163
|
+
is the common case since that repo requires manual approval on
|
|
164
|
+
Hugging Face) set `has_voice_cloning = False`. With that flag set,
|
|
165
|
+
`get_state_for_audio_prompt()` raises `ValueError` for ANY
|
|
166
|
+
`str`/`Path` audio-conditioning argument, including OwnVoice's own
|
|
167
|
+
reference clip -- reproduced against real weights this session.
|
|
168
|
+
Passing a pre-loaded, pre-resampled `torch.Tensor` instead is
|
|
169
|
+
pocket-tts's own documented input type for this argument (`audio_
|
|
170
|
+
conditioning: Path | str | torch.Tensor`) and is not gated -- it
|
|
171
|
+
reaches the exact same Mimi-encode-then-project conditioning path
|
|
172
|
+
either way. So OwnVoice loads and resamples the reference clip
|
|
173
|
+
itself here rather than depending on gated HF access most users of
|
|
174
|
+
OwnVoice won't have. Voice-cloning fidelity from the without-voice-
|
|
175
|
+
cloning checkpoint is a known limitation of the base model itself
|
|
176
|
+
(per pocket-tts's own `VOICE_CLONING_UNSUPPORTED` message), not an
|
|
177
|
+
OwnVoice bug -- users who want kyutai's best-quality cloning weights
|
|
178
|
+
can request gated access at https://huggingface.co/kyutai/pocket-tts
|
|
179
|
+
and run `hf auth login`; OwnVoice works either way.
|
|
180
|
+
"""
|
|
181
|
+
from pocket_tts.data.audio import audio_read
|
|
182
|
+
from pocket_tts.data.audio_utils import convert_audio
|
|
183
|
+
|
|
184
|
+
del peft_model # injection already mutated base_model.flow_lm in place; see docstring point 1
|
|
185
|
+
|
|
186
|
+
reference_audio_path = Path(reference_audio_path)
|
|
187
|
+
raw_audio, source_sample_rate = audio_read(reference_audio_path)
|
|
188
|
+
audio_conditioning = convert_audio(
|
|
189
|
+
raw_audio, source_sample_rate, base_model.config.mimi.sample_rate, 1
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
model_state = base_model.get_state_for_audio_prompt(audio_conditioning)
|
|
193
|
+
return base_model.generate_audio(model_state, text)
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def save_wav(waveform: torch.Tensor, sample_rate: int, out_path: Path | str) -> Path:
|
|
197
|
+
"""Save a 1D (or (1, N)) PCM waveform tensor to a .wav file.
|
|
198
|
+
|
|
199
|
+
Uses `soundfile` directly rather than `torchaudio.save` so writing a wav
|
|
200
|
+
file doesn't pull in an ffmpeg/torchcodec dependency chain just to save
|
|
201
|
+
plain PCM audio.
|
|
202
|
+
"""
|
|
203
|
+
import soundfile as sf
|
|
204
|
+
|
|
205
|
+
out_path = Path(out_path)
|
|
206
|
+
out_path.parent.mkdir(parents=True, exist_ok=True)
|
|
207
|
+
if waveform.dim() == 2:
|
|
208
|
+
waveform = waveform.squeeze(0)
|
|
209
|
+
sf.write(str(out_path), waveform.to(torch.float32).cpu().numpy(), sample_rate)
|
|
210
|
+
return out_path
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def infer(
|
|
214
|
+
adapter_path: Path | str,
|
|
215
|
+
text: str,
|
|
216
|
+
out_path: Path | str = Path("./ownvoice-output.wav"),
|
|
217
|
+
reference_audio_override: Path | str | None = None,
|
|
218
|
+
) -> Path:
|
|
219
|
+
"""End-to-end infer flow used by `ownvoice infer`: load, generate, save."""
|
|
220
|
+
from ownvoice.train import load_base_model
|
|
221
|
+
|
|
222
|
+
adapter_path = Path(adapter_path)
|
|
223
|
+
base_model = load_base_model(device="cpu")
|
|
224
|
+
peft_model = load_adapter(base_model, adapter_path)
|
|
225
|
+
reference_audio_path = resolve_reference_audio(adapter_path, reference_audio_override)
|
|
226
|
+
waveform = generate_speech(base_model, peft_model, text, reference_audio_path)
|
|
227
|
+
return save_wav(waveform, base_model.sample_rate, out_path)
|
ownvoice/score.py
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"""Speaker-similarity scoring: resample to 16kHz mono, Resemblyzer cosine similarity.
|
|
2
|
+
|
|
3
|
+
Per the locked design (Approach A), 0.75 cosine similarity between the
|
|
4
|
+
reference clip's and the generated clip's Resemblyzer speaker embeddings is
|
|
5
|
+
the provisional bar for "usable adapter." Clearing it or not is reported as
|
|
6
|
+
a labeled result, never a crash -- see ownvoice/train.py's train() flow.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
import soundfile as sf
|
|
14
|
+
import torch
|
|
15
|
+
import torchaudio
|
|
16
|
+
|
|
17
|
+
RESEMBLYZER_SAMPLE_RATE = 16000
|
|
18
|
+
USABLE_THRESHOLD = 0.75
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def resample_to_16k_mono(waveform: torch.Tensor, orig_sample_rate: int) -> torch.Tensor:
|
|
22
|
+
"""Resample a waveform tensor to 16kHz mono for Resemblyzer's embedding model.
|
|
23
|
+
|
|
24
|
+
Accepts a 1D (samples,) or 2D (channels, samples) tensor. Returns a 1D
|
|
25
|
+
float32 tensor. A no-op resample (already 16kHz) still runs through
|
|
26
|
+
downmixing and dtype normalization for a consistent return shape.
|
|
27
|
+
"""
|
|
28
|
+
if waveform.dim() == 1:
|
|
29
|
+
waveform = waveform.unsqueeze(0)
|
|
30
|
+
if waveform.shape[0] > 1:
|
|
31
|
+
waveform = waveform.mean(dim=0, keepdim=True)
|
|
32
|
+
if orig_sample_rate != RESEMBLYZER_SAMPLE_RATE:
|
|
33
|
+
resampler = torchaudio.transforms.Resample(
|
|
34
|
+
orig_freq=orig_sample_rate, new_freq=RESEMBLYZER_SAMPLE_RATE
|
|
35
|
+
)
|
|
36
|
+
waveform = resampler(waveform)
|
|
37
|
+
return waveform.squeeze(0).to(torch.float32)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def load_and_resample(path: Path | str) -> torch.Tensor:
|
|
41
|
+
"""Load a wav file from disk and return it as 16kHz mono float32.
|
|
42
|
+
|
|
43
|
+
Reads via `soundfile` (not `torchaudio.load`, see the note in
|
|
44
|
+
ownvoice/data.py for why), then resamples with
|
|
45
|
+
`torchaudio.transforms.Resample`.
|
|
46
|
+
"""
|
|
47
|
+
audio, sample_rate = sf.read(str(path), dtype="float32", always_2d=True)
|
|
48
|
+
waveform = torch.from_numpy(audio.T).to(torch.float32)
|
|
49
|
+
return resample_to_16k_mono(waveform, sample_rate)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def cosine_similarity(a, b) -> float:
|
|
53
|
+
"""Cosine similarity between two 1D embedding vectors (numpy or list-like).
|
|
54
|
+
|
|
55
|
+
Pure math, no Resemblyzer/model dependency -- kept separate from
|
|
56
|
+
compute_similarity() so the threshold logic is unit-testable without
|
|
57
|
+
downloading Resemblyzer's pretrained encoder weights.
|
|
58
|
+
"""
|
|
59
|
+
import numpy as np
|
|
60
|
+
|
|
61
|
+
a = np.asarray(a, dtype=np.float64)
|
|
62
|
+
b = np.asarray(b, dtype=np.float64)
|
|
63
|
+
denom = float(np.linalg.norm(a) * np.linalg.norm(b))
|
|
64
|
+
if denom == 0.0:
|
|
65
|
+
return 0.0
|
|
66
|
+
return float(np.dot(a, b) / denom)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def compute_similarity(
|
|
70
|
+
reference_waveform_16k: torch.Tensor,
|
|
71
|
+
generated_waveform_16k: torch.Tensor,
|
|
72
|
+
) -> float:
|
|
73
|
+
"""Cosine similarity between the two waveforms' Resemblyzer speaker embeddings.
|
|
74
|
+
|
|
75
|
+
Both inputs must already be 16kHz mono float32 tensors (use
|
|
76
|
+
resample_to_16k_mono or load_and_resample first). Resemblyzer is
|
|
77
|
+
imported lazily here, not at module load, so importing ownvoice.score
|
|
78
|
+
doesn't require Resemblyzer's pretrained weights to be downloaded yet.
|
|
79
|
+
"""
|
|
80
|
+
from resemblyzer import VoiceEncoder
|
|
81
|
+
|
|
82
|
+
encoder = VoiceEncoder()
|
|
83
|
+
ref_embed = encoder.embed_utterance(reference_waveform_16k.numpy())
|
|
84
|
+
gen_embed = encoder.embed_utterance(generated_waveform_16k.numpy())
|
|
85
|
+
return cosine_similarity(ref_embed, gen_embed)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def is_usable(score: float, threshold: float = USABLE_THRESHOLD) -> bool:
|
|
89
|
+
"""True if `score` clears the usable-adapter bar (>= threshold, default 0.75)."""
|
|
90
|
+
return score >= threshold
|