ownvoice-cli 0.1.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
ownvoice/__init__.py ADDED
@@ -0,0 +1,10 @@
1
+ """OwnVoice: train a LoRA voice adapter for pocket-tts, own the resulting model.
2
+
3
+ OwnVoice wraps kyutai-labs/pocket-tts (MIT-licensed, CPU-capable local
4
+ text-to-speech) with a LoRA fine-tuning workflow. The output is an adapter
5
+ file you keep and run yourself, not an API subscription.
6
+ """
7
+
8
+ __version__ = "0.1.0"
9
+
10
+ __all__ = ["__version__"]
ownvoice/cli.py ADDED
@@ -0,0 +1,216 @@
1
+ """OwnVoice CLI: `check`, `train`, `infer` subcommands.
2
+
3
+ Every subcommand supports `--json` for a structured, machine-parseable
4
+ output mode alongside the default human-readable text, so a script or an
5
+ agent invoking `ownvoice` programmatically can parse the result reliably.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import json as json_module
11
+ from pathlib import Path
12
+ from typing import Optional
13
+
14
+ import typer
15
+
16
+ from ownvoice import __version__
17
+ from ownvoice.infer import infer as run_infer
18
+ from ownvoice.train import (
19
+ DEFAULT_EPOCHS,
20
+ DEFAULT_EVAL_TEXT,
21
+ DEFAULT_LEARNING_RATE,
22
+ DEFAULT_LORA_ALPHA,
23
+ DEFAULT_LORA_DROPOUT,
24
+ DEFAULT_LORA_RANK,
25
+ DEFAULT_OUT_DIR,
26
+ TrainingConfig,
27
+ check_compatibility,
28
+ )
29
+ from ownvoice.train import train as run_train
30
+
31
+ app = typer.Typer(
32
+ name="ownvoice",
33
+ help="Train a LoRA voice adapter for pocket-tts and own the resulting model. No API lock-in.",
34
+ no_args_is_help=True,
35
+ add_completion=False,
36
+ )
37
+
38
+
39
+ def _version_callback(value: bool) -> None:
40
+ if value:
41
+ typer.echo(f"ownvoice {__version__}")
42
+ raise typer.Exit()
43
+
44
+
45
+ @app.callback()
46
+ def main(
47
+ version: Optional[bool] = typer.Option(
48
+ None,
49
+ "--version",
50
+ callback=_version_callback,
51
+ is_eager=True,
52
+ help="Print the OwnVoice version and exit.",
53
+ ),
54
+ ) -> None:
55
+ """OwnVoice: own your voice model, no API lock-in."""
56
+
57
+
58
+ @app.command()
59
+ def check(
60
+ json_output: bool = typer.Option(
61
+ False, "--json", help="Print machine-readable JSON instead of human-readable text."
62
+ ),
63
+ ) -> None:
64
+ """Free, CPU-only compatibility check: load pocket-tts and dry-run the LoRA injection.
65
+
66
+ No GPU and no training required. Run this before renting a GPU for
67
+ `ownvoice train` -- it is the Day-0 compatibility spike, exposed as a
68
+ command.
69
+
70
+ Example:
71
+ ownvoice check
72
+ """
73
+ result = check_compatibility()
74
+
75
+ if json_output:
76
+ typer.echo(
77
+ json_module.dumps(
78
+ {
79
+ "success": result.success,
80
+ "message": result.message,
81
+ "module_tree": result.module_tree,
82
+ }
83
+ )
84
+ )
85
+ else:
86
+ if result.success:
87
+ typer.secho(f"[ownvoice check] PASS: {result.message}", fg=typer.colors.GREEN)
88
+ else:
89
+ typer.secho(f"[ownvoice check] FAIL: {result.message}", fg=typer.colors.RED)
90
+ if result.module_tree:
91
+ typer.echo("\nModule tree (for debugging / for the issue #30 blocker post):")
92
+ typer.echo(result.module_tree)
93
+
94
+ raise typer.Exit(code=0 if result.success else 1)
95
+
96
+
97
+ @app.command()
98
+ def train(
99
+ voice_clips: Path = typer.Option(
100
+ ...,
101
+ "--voice-clips",
102
+ exists=True,
103
+ file_okay=False,
104
+ dir_okay=True,
105
+ help="Directory of .wav voice-clip recordings to train from. Required.",
106
+ ),
107
+ out: Path = typer.Option(
108
+ DEFAULT_OUT_DIR, "--out", help="Directory to write adapter.safetensors + metadata.json to."
109
+ ),
110
+ epochs: int = typer.Option(DEFAULT_EPOCHS, "--epochs", min=1, help="Number of training epochs."),
111
+ lora_rank: int = typer.Option(DEFAULT_LORA_RANK, "--lora-rank", min=1, help="LoRA rank."),
112
+ lora_alpha: int = typer.Option(DEFAULT_LORA_ALPHA, "--lora-alpha", min=1, help="LoRA alpha."),
113
+ lora_dropout: float = typer.Option(
114
+ DEFAULT_LORA_DROPOUT, "--lora-dropout", min=0.0, max=1.0, help="LoRA dropout."
115
+ ),
116
+ learning_rate: float = typer.Option(
117
+ DEFAULT_LEARNING_RATE, "--learning-rate", help="Optimizer learning rate."
118
+ ),
119
+ eval_text: str = typer.Option(
120
+ DEFAULT_EVAL_TEXT,
121
+ "--eval-text",
122
+ help="Sentence synthesized after training to score against the reference voice.",
123
+ ),
124
+ json_output: bool = typer.Option(
125
+ False, "--json", help="Print machine-readable JSON instead of human-readable text."
126
+ ),
127
+ ) -> None:
128
+ """Train a LoRA voice adapter from a directory of .wav voice clips.
129
+
130
+ Only --voice-clips is required; every other flag has a sensible default.
131
+ Exits 0 on a successful run whether or not the similarity score clears
132
+ the usable threshold (0.75) -- a below-threshold result is a labeled
133
+ outcome with a next step, not a crash. Only a data-loading failure or a
134
+ caught PEFT-injection failure exits non-zero.
135
+
136
+ Example:
137
+ ownvoice train --voice-clips ./my-voice-clips
138
+ """
139
+ config = TrainingConfig(
140
+ voice_clips_dir=voice_clips,
141
+ out_dir=out,
142
+ epochs=epochs,
143
+ lora_rank=lora_rank,
144
+ lora_alpha=lora_alpha,
145
+ lora_dropout=lora_dropout,
146
+ learning_rate=learning_rate,
147
+ eval_text=eval_text,
148
+ )
149
+ result = run_train(config)
150
+
151
+ if json_output:
152
+ typer.echo(
153
+ json_module.dumps(
154
+ {
155
+ "success": result.success,
156
+ "out_dir": str(result.out_dir) if result.out_dir else None,
157
+ "similarity_score": result.similarity_score,
158
+ "usable": result.usable,
159
+ "message": result.message,
160
+ "infer_command": result.infer_command,
161
+ }
162
+ )
163
+ )
164
+ else:
165
+ if result.success:
166
+ label = "USABLE ADAPTER" if result.usable else "BELOW THRESHOLD"
167
+ color = typer.colors.GREEN if result.usable else typer.colors.YELLOW
168
+ typer.secho(f"[ownvoice train] {label}", fg=color, bold=True)
169
+ typer.echo(result.message)
170
+ else:
171
+ typer.secho("[ownvoice train] FAILED", fg=typer.colors.RED, bold=True)
172
+ typer.echo(result.message)
173
+
174
+ raise typer.Exit(code=0 if result.success else 1)
175
+
176
+
177
+ @app.command()
178
+ def infer(
179
+ adapter: Path = typer.Option(
180
+ ..., "--adapter", exists=True, help="Path to a trained adapter.safetensors file. Required."
181
+ ),
182
+ text: str = typer.Option(..., "--text", help="Text to synthesize in the trained voice. Required."),
183
+ out: Path = typer.Option(Path("./ownvoice-output.wav"), "--out", help="Output .wav file path."),
184
+ reference_audio: Optional[Path] = typer.Option(
185
+ None,
186
+ "--reference-audio",
187
+ help="Override the reference clip OwnVoice recorded in metadata.json at train time.",
188
+ ),
189
+ json_output: bool = typer.Option(
190
+ False, "--json", help="Print machine-readable JSON instead of human-readable text."
191
+ ),
192
+ ) -> None:
193
+ """Generate speech in the trained voice from a saved adapter, and save it to a .wav file.
194
+
195
+ Example:
196
+ ownvoice infer --adapter ./ownvoice-adapter/adapter.safetensors --text "Hello, this is my own voice."
197
+ """
198
+ try:
199
+ out_path = run_infer(adapter, text, out_path=out, reference_audio_override=reference_audio)
200
+ except Exception as exc: # noqa: BLE001
201
+ if json_output:
202
+ typer.echo(json_module.dumps({"success": False, "error": str(exc)}))
203
+ else:
204
+ typer.secho(f"[ownvoice infer] FAILED: {exc}", fg=typer.colors.RED, bold=True)
205
+ raise typer.Exit(code=1) from exc
206
+
207
+ if json_output:
208
+ typer.echo(json_module.dumps({"success": True, "out_path": str(out_path)}))
209
+ else:
210
+ typer.secho(f"[ownvoice infer] Wrote {out_path}", fg=typer.colors.GREEN)
211
+
212
+ raise typer.Exit(code=0)
213
+
214
+
215
+ if __name__ == "__main__":
216
+ app()
ownvoice/data.py ADDED
@@ -0,0 +1,139 @@
1
+ """Voice-clip loading and validation for OwnVoice training data.
2
+
3
+ pocket-tts's own sample rate is read from the loaded model at train time
4
+ (`TTSModel.sample_rate`); this module only validates raw input files and
5
+ normalizes them to whatever target rate the caller asks for.
6
+
7
+ Audio I/O uses `soundfile` directly rather than `torchaudio.load`/
8
+ `torchaudio.info`: as of the torchaudio version this project pins against
9
+ (2.9+), both of those now route through an optional `torchcodec`/ffmpeg
10
+ backend and raise `ImportError` without it installed. `soundfile` (libsndfile)
11
+ reads plain .wav files with no extra runtime dependency. Resampling still
12
+ goes through `torchaudio.transforms.Resample` per the locked design decision,
13
+ since that's a pure-tensor operation with no I/O backend involved.
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import dataclasses
19
+ from pathlib import Path
20
+
21
+ import soundfile as sf
22
+ import torch
23
+ import torchaudio
24
+
25
+ MIN_CLIP_SECONDS = 1.0
26
+ MAX_CLIP_SECONDS = 30.0
27
+ SUPPORTED_SUFFIXES = {".wav"}
28
+
29
+
30
+ class VoiceClipError(Exception):
31
+ """Raised when a voice-clip directory or file fails validation."""
32
+
33
+
34
+ @dataclasses.dataclass(frozen=True)
35
+ class VoiceClip:
36
+ """Metadata for one validated voice-clip file, not yet loaded into memory."""
37
+
38
+ path: Path
39
+ sample_rate: int
40
+ num_frames: int
41
+ num_channels: int
42
+
43
+ @property
44
+ def duration_seconds(self) -> float:
45
+ return self.num_frames / self.sample_rate
46
+
47
+
48
+ def _iter_candidate_files(voice_clips_dir: Path) -> list[Path]:
49
+ if not voice_clips_dir.exists():
50
+ raise VoiceClipError(f"Voice clips directory does not exist: {voice_clips_dir}")
51
+ if not voice_clips_dir.is_dir():
52
+ raise VoiceClipError(f"Voice clips path is not a directory: {voice_clips_dir}")
53
+ return sorted(
54
+ p
55
+ for p in voice_clips_dir.iterdir()
56
+ if p.is_file() and p.suffix.lower() in SUPPORTED_SUFFIXES
57
+ )
58
+
59
+
60
+ def load_voice_clips(
61
+ voice_clips_dir: Path | str,
62
+ *,
63
+ min_seconds: float = MIN_CLIP_SECONDS,
64
+ max_seconds: float = MAX_CLIP_SECONDS,
65
+ ) -> list[VoiceClip]:
66
+ """Validate every .wav file in `voice_clips_dir` and return clip metadata.
67
+
68
+ A file that individually fails to decode, is empty, or falls outside the
69
+ duration bounds is skipped (not fatal on its own) -- only an empty result
70
+ set raises. Raises VoiceClipError with an actionable message either way
71
+ the directory itself doesn't exist/isn't a directory, has no .wav files
72
+ at all, or every candidate file was skipped.
73
+ """
74
+ voice_clips_dir = Path(voice_clips_dir)
75
+ candidates = _iter_candidate_files(voice_clips_dir)
76
+ if not candidates:
77
+ raise VoiceClipError(
78
+ f"No .wav files found in {voice_clips_dir}. OwnVoice needs at least one "
79
+ "clean .wav recording of the target voice to train from."
80
+ )
81
+
82
+ clips: list[VoiceClip] = []
83
+ skipped: list[str] = []
84
+ for path in candidates:
85
+ try:
86
+ info = sf.info(str(path))
87
+ except Exception as exc: # noqa: BLE001 - soundfile raises backend-specific errors
88
+ skipped.append(f"{path.name}: could not read audio ({exc})")
89
+ continue
90
+
91
+ if info.frames <= 0:
92
+ skipped.append(f"{path.name}: empty audio (0 frames)")
93
+ continue
94
+
95
+ duration = info.frames / info.samplerate
96
+ if duration < min_seconds:
97
+ skipped.append(f"{path.name}: too short ({duration:.2f}s, minimum is {min_seconds}s)")
98
+ continue
99
+ if duration > max_seconds:
100
+ skipped.append(f"{path.name}: too long ({duration:.2f}s, maximum is {max_seconds}s per clip)")
101
+ continue
102
+
103
+ clips.append(
104
+ VoiceClip(
105
+ path=path,
106
+ sample_rate=info.samplerate,
107
+ num_frames=info.frames,
108
+ num_channels=info.channels,
109
+ )
110
+ )
111
+
112
+ if not clips:
113
+ detail = "; ".join(skipped) if skipped else "no readable clips"
114
+ raise VoiceClipError(
115
+ f"No usable voice clips found in {voice_clips_dir} ({detail}). "
116
+ f"Each clip must be a valid mono or stereo .wav file between "
117
+ f"{min_seconds}s and {max_seconds}s."
118
+ )
119
+
120
+ return clips
121
+
122
+
123
+ def load_waveform(clip: VoiceClip, target_sample_rate: int) -> torch.Tensor:
124
+ """Load a clip's waveform, downmix to mono, and resample to `target_sample_rate`.
125
+
126
+ Returns a 1D float32 tensor. This is the shared normalization path used
127
+ both by the training data pipeline and (via score.py) by evaluation, so
128
+ every clip reaches the model or the scorer in a consistent format
129
+ regardless of its original sample rate or channel count.
130
+ """
131
+ audio, sample_rate = sf.read(str(clip.path), dtype="float32", always_2d=True)
132
+ # soundfile returns (frames, channels); torchaudio's convention is (channels, frames).
133
+ waveform = torch.from_numpy(audio.T).to(torch.float32)
134
+ if waveform.shape[0] > 1:
135
+ waveform = waveform.mean(dim=0, keepdim=True)
136
+ if sample_rate != target_sample_rate:
137
+ resampler = torchaudio.transforms.Resample(orig_freq=sample_rate, new_freq=target_sample_rate)
138
+ waveform = resampler(waveform)
139
+ return waveform.squeeze(0).to(torch.float32)
ownvoice/infer.py ADDED
@@ -0,0 +1,227 @@
1
+ """Load a trained LoRA adapter and generate speech with it.
2
+
3
+ Real-API note: pocket-tts's `TTSModel.generate_audio(model_state, text)`
4
+ requires a `model_state` dict built by `get_state_for_audio_prompt(audio_
5
+ conditioning, ...)`, and `audio_conditioning` has no default -- it must be a
6
+ real path, URL, or tensor (confirmed against pocket_tts/models/tts_model.py,
7
+ not assumed). OwnVoice's whole differentiation is that the *voice* comes
8
+ from the trained LoRA weights, not a runtime reference clip, but pocket-tts's
9
+ generation call still needs *some* audio prompt to build the initial
10
+ conditioning state. OwnVoice resolves this by reusing the same reference
11
+ clip that `ownvoice train` recorded in `metadata.json` (the first voice clip
12
+ in the training set) by default, with an optional override.
13
+
14
+ Verified for real this session (a real 2-epoch LoRA training run, followed
15
+ by a real `ownvoice infer` call against the resulting adapter, produced a
16
+ 4.7s non-silent, finite, NaN-free .wav file): `load_adapter()` and
17
+ `generate_speech()` below are exercised end to end against real loaded
18
+ pocket-tts weights, not just reasoned about. Two things this session found
19
+ that the original (structurally reasonable but unverified) approach in
20
+ `generate_speech()` got wrong are documented on that function directly --
21
+ in short, no `base_model.flow_lm` swap is needed or safe, and OwnVoice
22
+ must pre-load and resample the reference clip itself rather than pass a
23
+ raw path/URL, because the publicly downloadable pocket-tts weights are the
24
+ non-voice-cloning checkpoint by default.
25
+ """
26
+
27
+ from __future__ import annotations
28
+
29
+ import json
30
+ import os
31
+ from pathlib import Path
32
+ from typing import Any
33
+
34
+ import torch
35
+
36
+ from ownvoice.train import LORA_TARGET_MODULES, DEFAULT_LORA_ALPHA, DEFAULT_LORA_DROPOUT, DEFAULT_LORA_RANK
37
+
38
+
39
+ class AdapterLoadError(Exception):
40
+ """Raised when an adapter file or its sibling metadata.json can't be loaded."""
41
+
42
+
43
+ def _read_metadata(adapter_path: Path) -> dict[str, Any]:
44
+ metadata_path = adapter_path.parent / "metadata.json"
45
+ if not metadata_path.exists():
46
+ raise AdapterLoadError(
47
+ f"No metadata.json found next to {adapter_path}. OwnVoice adapters are always "
48
+ "saved with a sibling metadata.json (written by `ownvoice train`) -- if this "
49
+ "adapter was moved on its own, put metadata.json back alongside it."
50
+ )
51
+ return json.loads(metadata_path.read_text())
52
+
53
+
54
+ def load_adapter(base_model: Any, adapter_path: Path | str):
55
+ """Load a saved LoRA adapter (`adapter.safetensors`) onto pocket-tts's `flow_lm`.
56
+
57
+ Reconstructs the same LoraConfig used at train time (rank/alpha/
58
+ dropout, read back from the adapter's sibling metadata.json) and loads
59
+ the saved state dict via PEFT's `set_peft_model_state_dict`. The
60
+ reload round trip is verified against a real trained adapter this
61
+ session: `ownvoice train` saved a real adapter.safetensors via
62
+ `get_peft_model_state_dict`, and this function loaded it back and used
63
+ it for real generation via `ownvoice infer` without error.
64
+ """
65
+ from peft import LoraConfig, get_peft_model, set_peft_model_state_dict
66
+ from safetensors.torch import load_file
67
+
68
+ adapter_path = Path(adapter_path)
69
+ metadata = _read_metadata(adapter_path)
70
+ train_config = metadata.get("config", {})
71
+
72
+ lora_config = LoraConfig(
73
+ r=train_config.get("lora_rank", DEFAULT_LORA_RANK),
74
+ lora_alpha=train_config.get("lora_alpha", DEFAULT_LORA_ALPHA),
75
+ lora_dropout=train_config.get("lora_dropout", DEFAULT_LORA_DROPOUT),
76
+ target_modules=LORA_TARGET_MODULES,
77
+ bias="none",
78
+ )
79
+ peft_model = get_peft_model(base_model.flow_lm, lora_config)
80
+ state_dict = load_file(str(adapter_path))
81
+ set_peft_model_state_dict(peft_model, state_dict)
82
+ peft_model.eval()
83
+ return peft_model
84
+
85
+
86
+ def resolve_reference_audio(adapter_path: Path | str, override: Path | str | None = None) -> Path:
87
+ """Pick the audio-conditioning clip generate_speech() needs.
88
+
89
+ Uses `override` if given, otherwise falls back to the reference clip
90
+ OwnVoice recorded in the adapter's metadata.json at training time.
91
+ """
92
+ if override is not None:
93
+ return Path(override)
94
+
95
+ adapter_path = Path(adapter_path)
96
+ metadata = _read_metadata(adapter_path)
97
+ reference_clip = metadata.get("reference_clip")
98
+ if not reference_clip:
99
+ raise AdapterLoadError(
100
+ f"metadata.json next to {adapter_path} has no recorded reference_clip, and no "
101
+ "override was given. Pass one explicitly."
102
+ )
103
+ reference_path = Path(reference_clip)
104
+ if reference_path.is_absolute():
105
+ raise AdapterLoadError(
106
+ f"Recorded reference clip {reference_path} in metadata.json is an absolute path, "
107
+ "which OwnVoice does not trust from a shared adapter's metadata.json. Pass an "
108
+ "override path to a voice clip explicitly."
109
+ )
110
+
111
+ adapter_dir = adapter_path.parent
112
+ resolved_path = adapter_dir / reference_path
113
+ real_adapter_dir = os.path.realpath(adapter_dir)
114
+ real_resolved_path = os.path.realpath(resolved_path)
115
+ if os.path.commonpath([real_adapter_dir, real_resolved_path]) != real_adapter_dir:
116
+ raise AdapterLoadError(
117
+ f"Recorded reference clip {reference_path} in metadata.json resolves outside of "
118
+ f"{adapter_dir}, which OwnVoice does not trust from a shared adapter's "
119
+ "metadata.json. Pass an override path to a voice clip explicitly."
120
+ )
121
+ if not resolved_path.exists():
122
+ raise AdapterLoadError(
123
+ f"Recorded reference clip {resolved_path} no longer exists. Pass an override "
124
+ "path to a voice clip explicitly."
125
+ )
126
+ return resolved_path
127
+
128
+
129
+ def generate_speech(base_model: Any, peft_model: Any, text: str, reference_audio_path: Path | str) -> torch.Tensor:
130
+ """Generate speech audio for `text` using the base model wrapped with the LoRA adapter.
131
+
132
+ Verified against real loaded pocket-tts weights this session, and the
133
+ naive approach the previous TODO here anticipated turned out to be
134
+ wrong for two independent, real reasons:
135
+
136
+ 1. No flow_lm swap is needed, and the previous
137
+ `base_model.flow_lm = peft_model` swap actively breaks generation.
138
+ PEFT's `get_peft_model()` replaces `base_model.flow_lm`'s targeted
139
+ `nn.Linear` submodules with LoRA layers IN PLACE -- it mutates the
140
+ same live module graph rather than returning a copy, so
141
+ `base_model.flow_lm` already carries the trained adapter's weights
142
+ by the time this function runs (confirmed: `id(base_model.flow_lm)`
143
+ is unchanged before/after `get_peft_model()`, and its Linear
144
+ submodules are already `peft.tuners.lora.layer.Linear` instances
145
+ afterward). Reassigning `base_model.flow_lm` to the outer
146
+ `PeftModel` wrapper instead breaks pocket-tts's own streaming state
147
+ machinery: `StatefulModule.get_state()` looks up KV-cache state by a
148
+ fixed `_module_absolute_name` string baked onto each submodule once
149
+ at `TTSModel.load_model()` time (e.g. "transformer.layers.0.
150
+ self_attn"), but `get_state_for_audio_prompt()` rebuilds the
151
+ `model_state` dict fresh via `init_states(base_model.flow_lm, ...)`,
152
+ which keys it off `named_modules()` on whatever module is currently
153
+ assigned to `base_model.flow_lm`. PEFT's `PeftModel`/`LoraModel`
154
+ wrapper inserts a "base_model.model." prefix into those paths, so
155
+ swapping to the wrapper produces a `model_state` dict whose keys no
156
+ longer match the fixed `_module_absolute_name` strings, and
157
+ generation fails with `KeyError: 'transformer.layers.0.self_attn'`
158
+ -- reproduced against real weights this session.
159
+ 2. The publicly downloadable pocket-tts weights
160
+ (`kyutai/pocket-tts-without-voice-cloning`, the checkpoint
161
+ `TTSModel.load_model()` falls back to whenever the gated
162
+ `kyutai/pocket-tts` voice-cloning weights aren't accessible, which
163
+ is the common case since that repo requires manual approval on
164
+ Hugging Face) set `has_voice_cloning = False`. With that flag set,
165
+ `get_state_for_audio_prompt()` raises `ValueError` for ANY
166
+ `str`/`Path` audio-conditioning argument, including OwnVoice's own
167
+ reference clip -- reproduced against real weights this session.
168
+ Passing a pre-loaded, pre-resampled `torch.Tensor` instead is
169
+ pocket-tts's own documented input type for this argument (`audio_
170
+ conditioning: Path | str | torch.Tensor`) and is not gated -- it
171
+ reaches the exact same Mimi-encode-then-project conditioning path
172
+ either way. So OwnVoice loads and resamples the reference clip
173
+ itself here rather than depending on gated HF access most users of
174
+ OwnVoice won't have. Voice-cloning fidelity from the without-voice-
175
+ cloning checkpoint is a known limitation of the base model itself
176
+ (per pocket-tts's own `VOICE_CLONING_UNSUPPORTED` message), not an
177
+ OwnVoice bug -- users who want kyutai's best-quality cloning weights
178
+ can request gated access at https://huggingface.co/kyutai/pocket-tts
179
+ and run `hf auth login`; OwnVoice works either way.
180
+ """
181
+ from pocket_tts.data.audio import audio_read
182
+ from pocket_tts.data.audio_utils import convert_audio
183
+
184
+ del peft_model # injection already mutated base_model.flow_lm in place; see docstring point 1
185
+
186
+ reference_audio_path = Path(reference_audio_path)
187
+ raw_audio, source_sample_rate = audio_read(reference_audio_path)
188
+ audio_conditioning = convert_audio(
189
+ raw_audio, source_sample_rate, base_model.config.mimi.sample_rate, 1
190
+ )
191
+
192
+ model_state = base_model.get_state_for_audio_prompt(audio_conditioning)
193
+ return base_model.generate_audio(model_state, text)
194
+
195
+
196
+ def save_wav(waveform: torch.Tensor, sample_rate: int, out_path: Path | str) -> Path:
197
+ """Save a 1D (or (1, N)) PCM waveform tensor to a .wav file.
198
+
199
+ Uses `soundfile` directly rather than `torchaudio.save` so writing a wav
200
+ file doesn't pull in an ffmpeg/torchcodec dependency chain just to save
201
+ plain PCM audio.
202
+ """
203
+ import soundfile as sf
204
+
205
+ out_path = Path(out_path)
206
+ out_path.parent.mkdir(parents=True, exist_ok=True)
207
+ if waveform.dim() == 2:
208
+ waveform = waveform.squeeze(0)
209
+ sf.write(str(out_path), waveform.to(torch.float32).cpu().numpy(), sample_rate)
210
+ return out_path
211
+
212
+
213
+ def infer(
214
+ adapter_path: Path | str,
215
+ text: str,
216
+ out_path: Path | str = Path("./ownvoice-output.wav"),
217
+ reference_audio_override: Path | str | None = None,
218
+ ) -> Path:
219
+ """End-to-end infer flow used by `ownvoice infer`: load, generate, save."""
220
+ from ownvoice.train import load_base_model
221
+
222
+ adapter_path = Path(adapter_path)
223
+ base_model = load_base_model(device="cpu")
224
+ peft_model = load_adapter(base_model, adapter_path)
225
+ reference_audio_path = resolve_reference_audio(adapter_path, reference_audio_override)
226
+ waveform = generate_speech(base_model, peft_model, text, reference_audio_path)
227
+ return save_wav(waveform, base_model.sample_rate, out_path)
ownvoice/score.py ADDED
@@ -0,0 +1,90 @@
1
+ """Speaker-similarity scoring: resample to 16kHz mono, Resemblyzer cosine similarity.
2
+
3
+ Per the locked design (Approach A), 0.75 cosine similarity between the
4
+ reference clip's and the generated clip's Resemblyzer speaker embeddings is
5
+ the provisional bar for "usable adapter." Clearing it or not is reported as
6
+ a labeled result, never a crash -- see ownvoice/train.py's train() flow.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from pathlib import Path
12
+
13
+ import soundfile as sf
14
+ import torch
15
+ import torchaudio
16
+
17
+ RESEMBLYZER_SAMPLE_RATE = 16000
18
+ USABLE_THRESHOLD = 0.75
19
+
20
+
21
+ def resample_to_16k_mono(waveform: torch.Tensor, orig_sample_rate: int) -> torch.Tensor:
22
+ """Resample a waveform tensor to 16kHz mono for Resemblyzer's embedding model.
23
+
24
+ Accepts a 1D (samples,) or 2D (channels, samples) tensor. Returns a 1D
25
+ float32 tensor. A no-op resample (already 16kHz) still runs through
26
+ downmixing and dtype normalization for a consistent return shape.
27
+ """
28
+ if waveform.dim() == 1:
29
+ waveform = waveform.unsqueeze(0)
30
+ if waveform.shape[0] > 1:
31
+ waveform = waveform.mean(dim=0, keepdim=True)
32
+ if orig_sample_rate != RESEMBLYZER_SAMPLE_RATE:
33
+ resampler = torchaudio.transforms.Resample(
34
+ orig_freq=orig_sample_rate, new_freq=RESEMBLYZER_SAMPLE_RATE
35
+ )
36
+ waveform = resampler(waveform)
37
+ return waveform.squeeze(0).to(torch.float32)
38
+
39
+
40
+ def load_and_resample(path: Path | str) -> torch.Tensor:
41
+ """Load a wav file from disk and return it as 16kHz mono float32.
42
+
43
+ Reads via `soundfile` (not `torchaudio.load`, see the note in
44
+ ownvoice/data.py for why), then resamples with
45
+ `torchaudio.transforms.Resample`.
46
+ """
47
+ audio, sample_rate = sf.read(str(path), dtype="float32", always_2d=True)
48
+ waveform = torch.from_numpy(audio.T).to(torch.float32)
49
+ return resample_to_16k_mono(waveform, sample_rate)
50
+
51
+
52
+ def cosine_similarity(a, b) -> float:
53
+ """Cosine similarity between two 1D embedding vectors (numpy or list-like).
54
+
55
+ Pure math, no Resemblyzer/model dependency -- kept separate from
56
+ compute_similarity() so the threshold logic is unit-testable without
57
+ downloading Resemblyzer's pretrained encoder weights.
58
+ """
59
+ import numpy as np
60
+
61
+ a = np.asarray(a, dtype=np.float64)
62
+ b = np.asarray(b, dtype=np.float64)
63
+ denom = float(np.linalg.norm(a) * np.linalg.norm(b))
64
+ if denom == 0.0:
65
+ return 0.0
66
+ return float(np.dot(a, b) / denom)
67
+
68
+
69
+ def compute_similarity(
70
+ reference_waveform_16k: torch.Tensor,
71
+ generated_waveform_16k: torch.Tensor,
72
+ ) -> float:
73
+ """Cosine similarity between the two waveforms' Resemblyzer speaker embeddings.
74
+
75
+ Both inputs must already be 16kHz mono float32 tensors (use
76
+ resample_to_16k_mono or load_and_resample first). Resemblyzer is
77
+ imported lazily here, not at module load, so importing ownvoice.score
78
+ doesn't require Resemblyzer's pretrained weights to be downloaded yet.
79
+ """
80
+ from resemblyzer import VoiceEncoder
81
+
82
+ encoder = VoiceEncoder()
83
+ ref_embed = encoder.embed_utterance(reference_waveform_16k.numpy())
84
+ gen_embed = encoder.embed_utterance(generated_waveform_16k.numpy())
85
+ return cosine_similarity(ref_embed, gen_embed)
86
+
87
+
88
+ def is_usable(score: float, threshold: float = USABLE_THRESHOLD) -> bool:
89
+ """True if `score` clears the usable-adapter bar (>= threshold, default 0.75)."""
90
+ return score >= threshold