tiny-voca-ru 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tiny_voca_ru/__init__.py +78 -0
- tiny_voca_ru/checkpoint.py +135 -0
- tiny_voca_ru/config.py +101 -0
- tiny_voca_ru/export.py +374 -0
- tiny_voca_ru/frontend/__init__.py +96 -0
- tiny_voca_ru/frontend/clean.py +90 -0
- tiny_voca_ru/frontend/data/LICENSE-ruaccent +15 -0
- tiny_voca_ru/frontend/data/accent.onnx +0 -0
- tiny_voca_ru/frontend/data/stress.txt.gz +0 -0
- tiny_voca_ru/frontend/g2p.py +381 -0
- tiny_voca_ru/frontend/lexicon.py +218 -0
- tiny_voca_ru/frontend/normalize.py +856 -0
- tiny_voca_ru/frontend/numbers.py +387 -0
- tiny_voca_ru/frontend/stress.py +428 -0
- tiny_voca_ru/frontend/symbols.py +103 -0
- tiny_voca_ru/frontend/tables.py +784 -0
- tiny_voca_ru/frontend/tokenize.py +222 -0
- tiny_voca_ru/infer.py +868 -0
- tiny_voca_ru/model.py +525 -0
- tiny_voca_ru/train/__init__.py +58 -0
- tiny_voca_ru/train/align.py +374 -0
- tiny_voca_ru/train/dataset.py +217 -0
- tiny_voca_ru/train/losses.py +214 -0
- tiny_voca_ru/train/prepare.py +373 -0
- tiny_voca_ru/train/train.py +655 -0
- tiny_voca_ru-0.1.0.dist-info/METADATA +252 -0
- tiny_voca_ru-0.1.0.dist-info/RECORD +33 -0
- tiny_voca_ru-0.1.0.dist-info/WHEEL +5 -0
- tiny_voca_ru-0.1.0.dist-info/entry_points.txt +6 -0
- tiny_voca_ru-0.1.0.dist-info/licenses/LICENSE +202 -0
- tiny_voca_ru-0.1.0.dist-info/licenses/NOTICE +24 -0
- tiny_voca_ru-0.1.0.dist-info/licenses/src/tiny_voca_ru/frontend/data/LICENSE-ruaccent +15 -0
- tiny_voca_ru-0.1.0.dist-info/top_level.txt +1 -0
tiny_voca_ru/__init__.py
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""tiny-voca-ru: a lightweight 4.5M-parameter Russian text-to-speech model.
|
|
2
|
+
|
|
3
|
+
Importing the package root or ``tiny_voca_ru.frontend`` never imports PyTorch.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from typing import TYPE_CHECKING, Any
|
|
9
|
+
|
|
10
|
+
from .config import Config
|
|
11
|
+
from .frontend import (
|
|
12
|
+
SYMBOLS,
|
|
13
|
+
PhoneSequence,
|
|
14
|
+
accentuate,
|
|
15
|
+
normalize,
|
|
16
|
+
phonemize,
|
|
17
|
+
transcribe,
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
if TYPE_CHECKING:
|
|
21
|
+
from .checkpoint import CheckpointMeta, convert_legacy_pt, load_checkpoint, save_checkpoint
|
|
22
|
+
from .export import export_onnx
|
|
23
|
+
from .infer import (
|
|
24
|
+
DEFAULT_HF_REPO_ID,
|
|
25
|
+
OnnxSynthesizer,
|
|
26
|
+
SynthesisResult,
|
|
27
|
+
Synthesizer,
|
|
28
|
+
TorchSynthesizer,
|
|
29
|
+
)
|
|
30
|
+
from .model import TinyVoca
|
|
31
|
+
|
|
32
|
+
__version__ = "0.1.0"
|
|
33
|
+
|
|
34
|
+
__all__ = [
|
|
35
|
+
"__version__",
|
|
36
|
+
"DEFAULT_HF_REPO_ID",
|
|
37
|
+
"SYMBOLS",
|
|
38
|
+
"CheckpointMeta",
|
|
39
|
+
"Config",
|
|
40
|
+
"OnnxSynthesizer",
|
|
41
|
+
"PhoneSequence",
|
|
42
|
+
"SynthesisResult",
|
|
43
|
+
"Synthesizer",
|
|
44
|
+
"TinyVoca",
|
|
45
|
+
"TorchSynthesizer",
|
|
46
|
+
"accentuate",
|
|
47
|
+
"convert_legacy_pt",
|
|
48
|
+
"export_onnx",
|
|
49
|
+
"load_checkpoint",
|
|
50
|
+
"normalize",
|
|
51
|
+
"phonemize",
|
|
52
|
+
"save_checkpoint",
|
|
53
|
+
"transcribe",
|
|
54
|
+
]
|
|
55
|
+
|
|
56
|
+
_LAZY_EXPORTS: dict[str, tuple[str, str]] = {
|
|
57
|
+
"CheckpointMeta": (".checkpoint", "CheckpointMeta"),
|
|
58
|
+
"convert_legacy_pt": (".checkpoint", "convert_legacy_pt"),
|
|
59
|
+
"load_checkpoint": (".checkpoint", "load_checkpoint"),
|
|
60
|
+
"save_checkpoint": (".checkpoint", "save_checkpoint"),
|
|
61
|
+
"export_onnx": (".export", "export_onnx"),
|
|
62
|
+
"DEFAULT_HF_REPO_ID": (".infer", "DEFAULT_HF_REPO_ID"),
|
|
63
|
+
"OnnxSynthesizer": (".infer", "OnnxSynthesizer"),
|
|
64
|
+
"SynthesisResult": (".infer", "SynthesisResult"),
|
|
65
|
+
"Synthesizer": (".infer", "Synthesizer"),
|
|
66
|
+
"TorchSynthesizer": (".infer", "TorchSynthesizer"),
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def __getattr__(name: str) -> Any:
|
|
71
|
+
target = _LAZY_EXPORTS.get(name)
|
|
72
|
+
if target is None:
|
|
73
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
74
|
+
module_name, attr_name = target
|
|
75
|
+
from importlib import import_module
|
|
76
|
+
|
|
77
|
+
module = import_module(module_name, __name__)
|
|
78
|
+
return getattr(module, attr_name)
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
"""Safetensors checkpoint I/O and legacy ``.pt`` conversion."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
import torch
|
|
12
|
+
from safetensors.torch import load_file, save_file
|
|
13
|
+
|
|
14
|
+
from .config import Config
|
|
15
|
+
from .frontend.symbols import SYMBOLS
|
|
16
|
+
from .model import TinyVoca
|
|
17
|
+
|
|
18
|
+
CHECKPOINT_FORMAT_VERSION = 1
|
|
19
|
+
WEIGHTS_FILENAME = "model.safetensors"
|
|
20
|
+
CONFIG_FILENAME = "config.json"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass(frozen=True)
|
|
24
|
+
class CheckpointMeta:
|
|
25
|
+
"""Metadata stored alongside ``model.safetensors`` in ``config.json``."""
|
|
26
|
+
|
|
27
|
+
format_version: int
|
|
28
|
+
step: int | None
|
|
29
|
+
symbols: tuple[str, ...]
|
|
30
|
+
config: Config
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _resolve_checkpoint_paths(path: str | Path) -> tuple[Path, Path]:
|
|
34
|
+
target = Path(path)
|
|
35
|
+
if not target.exists():
|
|
36
|
+
raise FileNotFoundError(f"checkpoint path does not exist: {target}")
|
|
37
|
+
if target.is_dir():
|
|
38
|
+
return target / WEIGHTS_FILENAME, target / CONFIG_FILENAME
|
|
39
|
+
if target.suffix == ".safetensors":
|
|
40
|
+
return target, target.parent / CONFIG_FILENAME
|
|
41
|
+
raise ValueError(f"expected a checkpoint directory or {WEIGHTS_FILENAME} file, got {target}")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def save_checkpoint(path: str | Path, model: TinyVoca, *, step: int | None = None) -> Path:
|
|
45
|
+
"""Save ``model`` as ``model.safetensors`` + ``config.json`` inside directory ``path``."""
|
|
46
|
+
directory = Path(path)
|
|
47
|
+
directory.mkdir(parents=True, exist_ok=True)
|
|
48
|
+
tensors = {k: v.detach().cpu().contiguous() for k, v in model.state_dict().items()}
|
|
49
|
+
tmp_weights = directory / f"{WEIGHTS_FILENAME}.tmp"
|
|
50
|
+
save_file(tensors, str(tmp_weights))
|
|
51
|
+
tmp_weights.replace(directory / WEIGHTS_FILENAME)
|
|
52
|
+
|
|
53
|
+
meta: dict[str, Any] = {
|
|
54
|
+
"format_version": CHECKPOINT_FORMAT_VERSION,
|
|
55
|
+
"step": step,
|
|
56
|
+
"symbols": list(SYMBOLS),
|
|
57
|
+
"config": model.config.to_dict(),
|
|
58
|
+
}
|
|
59
|
+
tmp_config = directory / f"{CONFIG_FILENAME}.tmp"
|
|
60
|
+
tmp_config.write_text(json.dumps(meta, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
61
|
+
tmp_config.replace(directory / CONFIG_FILENAME)
|
|
62
|
+
return directory
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def load_checkpoint(
|
|
66
|
+
path: str | Path,
|
|
67
|
+
device: str | torch.device = "cpu",
|
|
68
|
+
) -> tuple[TinyVoca, CheckpointMeta]:
|
|
69
|
+
"""Load a safetensors checkpoint directory (or ``model.safetensors`` path) for inference."""
|
|
70
|
+
weights_path, config_path = _resolve_checkpoint_paths(path)
|
|
71
|
+
if not weights_path.is_file() or not config_path.is_file():
|
|
72
|
+
raise FileNotFoundError(
|
|
73
|
+
f"checkpoint must contain {WEIGHTS_FILENAME} and {CONFIG_FILENAME}: {path}"
|
|
74
|
+
)
|
|
75
|
+
raw = json.loads(config_path.read_text(encoding="utf-8"))
|
|
76
|
+
version = raw.get("format_version")
|
|
77
|
+
if version != CHECKPOINT_FORMAT_VERSION:
|
|
78
|
+
raise ValueError(
|
|
79
|
+
f"unsupported format_version {version!r}; expected {CHECKPOINT_FORMAT_VERSION}"
|
|
80
|
+
)
|
|
81
|
+
symbols = raw.get("symbols")
|
|
82
|
+
if symbols != list(SYMBOLS):
|
|
83
|
+
raise ValueError(
|
|
84
|
+
"checkpoint symbol inventory does not match tiny_voca_ru.frontend.symbols.SYMBOLS"
|
|
85
|
+
)
|
|
86
|
+
config = Config.from_dict(raw["config"])
|
|
87
|
+
if config.vocab_size != len(SYMBOLS):
|
|
88
|
+
raise ValueError(f"checkpoint vocab_size {config.vocab_size} != {len(SYMBOLS)}")
|
|
89
|
+
|
|
90
|
+
model = TinyVoca(config)
|
|
91
|
+
state = load_file(str(weights_path), device="cpu")
|
|
92
|
+
model.load_state_dict(state, strict=True)
|
|
93
|
+
model = model.to(device).eval()
|
|
94
|
+
meta = CheckpointMeta(
|
|
95
|
+
format_version=int(version),
|
|
96
|
+
step=int(raw["step"]) if raw.get("step") is not None else None,
|
|
97
|
+
symbols=tuple(symbols),
|
|
98
|
+
config=config,
|
|
99
|
+
)
|
|
100
|
+
return model, meta
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def convert_legacy_pt(source_pt: str | Path, output_dir: str | Path) -> Path:
|
|
104
|
+
"""Convert a legacy PyTorch ``.pt`` checkpoint into ``model.safetensors`` + ``config.json``."""
|
|
105
|
+
payload = torch.load(Path(source_pt), map_location="cpu", weights_only=True)
|
|
106
|
+
if not isinstance(payload, dict) or "model" not in payload or "config" not in payload:
|
|
107
|
+
raise ValueError("legacy checkpoint must be a mapping containing 'config' and 'model'")
|
|
108
|
+
frontend_info = payload.get("frontend")
|
|
109
|
+
if (
|
|
110
|
+
isinstance(frontend_info, dict)
|
|
111
|
+
and "symbols" in frontend_info
|
|
112
|
+
and list(frontend_info["symbols"]) != list(SYMBOLS)
|
|
113
|
+
):
|
|
114
|
+
raise ValueError("legacy checkpoint symbols do not match SYMBOLS")
|
|
115
|
+
config = Config.from_dict(payload["config"])
|
|
116
|
+
model = TinyVoca(config)
|
|
117
|
+
model.load_state_dict(payload["model"], strict=True)
|
|
118
|
+
step = int(payload["step"]) if payload.get("step") is not None else None
|
|
119
|
+
return save_checkpoint(output_dir, model, step=step)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def main() -> None:
|
|
123
|
+
parser = argparse.ArgumentParser(
|
|
124
|
+
description="Convert a legacy .pt checkpoint to safetensors + config.json"
|
|
125
|
+
)
|
|
126
|
+
parser.add_argument("--source", type=Path, required=True, help="Path to legacy .pt checkpoint")
|
|
127
|
+
parser.add_argument("--output", type=Path, required=True, help="Output checkpoint directory")
|
|
128
|
+
args = parser.parse_args()
|
|
129
|
+
out = convert_legacy_pt(args.source, args.output)
|
|
130
|
+
weights = out / WEIGHTS_FILENAME
|
|
131
|
+
print(f"Converted checkpoint to {out} ({weights.stat().st_size / 1e6:.2f} MB)")
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
if __name__ == "__main__":
|
|
135
|
+
main()
|
tiny_voca_ru/config.py
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"""Model and acoustic feature configuration."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from dataclasses import asdict, dataclass
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
from .frontend.symbols import SYMBOLS
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass(frozen=True)
|
|
14
|
+
class Config:
|
|
15
|
+
"""Architecture and acoustic settings for ``TinyVoca``."""
|
|
16
|
+
|
|
17
|
+
# Acoustic features
|
|
18
|
+
sample_rate: int = 24000
|
|
19
|
+
hop: int = 256
|
|
20
|
+
n_fft: int = 1024
|
|
21
|
+
n_mels: int = 80
|
|
22
|
+
f0_min: float = 50.0
|
|
23
|
+
f0_max: float = 600.0
|
|
24
|
+
speakers: tuple[str, ...] = ("F", "M")
|
|
25
|
+
# Russian phoneme inventory size (tiny_voca_ru.frontend.symbols.SYMBOLS)
|
|
26
|
+
vocab_size: int = 61
|
|
27
|
+
# Text encoder and frame-level transformers
|
|
28
|
+
channels: int = 192
|
|
29
|
+
heads: int = 4
|
|
30
|
+
feedforward: int = 640
|
|
31
|
+
kernel: int = 7
|
|
32
|
+
encoder_layers: int = 4
|
|
33
|
+
prosody_layers: int = 2
|
|
34
|
+
acoustic_layers: int = 5
|
|
35
|
+
dropout: float = 0.1
|
|
36
|
+
predictor_channels: int = 128
|
|
37
|
+
# Source-filter waveform head
|
|
38
|
+
synth_fft: int = 512
|
|
39
|
+
synth_hop: int = 128
|
|
40
|
+
noise_fft: int = 256
|
|
41
|
+
noise_hop: int = 64
|
|
42
|
+
cep_causal: int = 80
|
|
43
|
+
cep_anti: int = 16
|
|
44
|
+
noise_cep: int = 32
|
|
45
|
+
harmonic_limit: float = 0.47
|
|
46
|
+
max_tokens: int = 512
|
|
47
|
+
max_frames: int = 3000
|
|
48
|
+
mel_mean: float = -5.0
|
|
49
|
+
mel_scale: float = 5.0
|
|
50
|
+
energy_mean: float = -4.0
|
|
51
|
+
energy_scale: float = 2.0
|
|
52
|
+
pitch_reference_hz: float = 200.0
|
|
53
|
+
|
|
54
|
+
def __post_init__(self) -> None:
|
|
55
|
+
if self.vocab_size != len(SYMBOLS):
|
|
56
|
+
raise ValueError(f"vocab_size must equal the phoneme inventory size {len(SYMBOLS)}")
|
|
57
|
+
if self.channels % self.heads or (self.channels // self.heads) % 2:
|
|
58
|
+
raise ValueError("channels must be divisible by heads with an even head width")
|
|
59
|
+
if not self.speakers or len(set(self.speakers)) != len(self.speakers):
|
|
60
|
+
raise ValueError("speaker names must be unique and nonempty")
|
|
61
|
+
for fft, hop in ((self.synth_fft, self.synth_hop), (self.noise_fft, self.noise_hop)):
|
|
62
|
+
if fft != 4 * hop or self.hop % hop:
|
|
63
|
+
raise ValueError(
|
|
64
|
+
"synthesis windows need 4x overlap and a hop dividing the acoustic hop"
|
|
65
|
+
)
|
|
66
|
+
if (
|
|
67
|
+
self.cep_causal + self.cep_anti >= self.synth_fft // 2
|
|
68
|
+
or self.noise_cep >= self.noise_fft // 2
|
|
69
|
+
):
|
|
70
|
+
raise ValueError("cepstrum orders must be well below half the synthesis FFT size")
|
|
71
|
+
if (
|
|
72
|
+
min(self.encoder_layers, self.prosody_layers, self.acoustic_layers, self.kernel) < 1
|
|
73
|
+
or not self.kernel % 2
|
|
74
|
+
):
|
|
75
|
+
raise ValueError("layer counts must be positive and the kernel odd")
|
|
76
|
+
if min(self.mel_scale, self.energy_scale) <= 0 or not 0 < self.harmonic_limit < 0.5:
|
|
77
|
+
raise ValueError("invalid normalization scale or harmonic limit")
|
|
78
|
+
|
|
79
|
+
def to_dict(self) -> dict[str, Any]:
|
|
80
|
+
data = asdict(self)
|
|
81
|
+
data["speakers"] = list(self.speakers)
|
|
82
|
+
return data
|
|
83
|
+
|
|
84
|
+
@classmethod
|
|
85
|
+
def from_dict(cls, value: dict[str, Any]) -> Config:
|
|
86
|
+
return cls(**{**value, "speakers": tuple(value["speakers"])})
|
|
87
|
+
|
|
88
|
+
def save(self, path: str | Path) -> Path:
|
|
89
|
+
target = Path(path)
|
|
90
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
91
|
+
target.write_text(
|
|
92
|
+
json.dumps(self.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
|
|
93
|
+
)
|
|
94
|
+
return target
|
|
95
|
+
|
|
96
|
+
@classmethod
|
|
97
|
+
def load(cls, path: str | Path) -> Config:
|
|
98
|
+
data = json.loads(Path(path).read_text(encoding="utf-8"))
|
|
99
|
+
if "config" in data and isinstance(data["config"], dict):
|
|
100
|
+
data = data["config"]
|
|
101
|
+
return cls.from_dict(data)
|
tiny_voca_ru/export.py
ADDED
|
@@ -0,0 +1,374 @@
|
|
|
1
|
+
"""ONNX export for single-utterance free-running synthesis."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
import math
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any, cast
|
|
10
|
+
|
|
11
|
+
import torch
|
|
12
|
+
from torch import Tensor, nn
|
|
13
|
+
from torch.nn import functional as F
|
|
14
|
+
|
|
15
|
+
from .checkpoint import CheckpointMeta, load_checkpoint
|
|
16
|
+
from .frontend.symbols import SYMBOLS
|
|
17
|
+
from .model import TinyVoca
|
|
18
|
+
|
|
19
|
+
INPUTS: tuple[str, ...] = ("tokens", "stress", "voice", "speed", "pitch_shift", "pitch_range")
|
|
20
|
+
OUTPUTS: tuple[str, ...] = ("audio", "durations")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _cepstrum_matrices(positions: list[int], fft: int) -> tuple[Tensor, Tensor]:
|
|
24
|
+
buffer = torch.zeros(len(positions), fft, dtype=torch.float64)
|
|
25
|
+
buffer[torch.arange(len(positions)), torch.tensor(positions)] = 1
|
|
26
|
+
spectrum = torch.fft.rfft(buffer, dim=-1)
|
|
27
|
+
return spectrum.real.float(), spectrum.imag.float()
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _analysis_kernels(window: Tensor) -> tuple[Tensor, Tensor]:
|
|
31
|
+
fft = len(window)
|
|
32
|
+
spectrum = torch.fft.rfft(torch.eye(fft, dtype=torch.float64) * window.double(), dim=-1)
|
|
33
|
+
return spectrum.real.T[:, None].float().contiguous(), spectrum.imag.T[
|
|
34
|
+
:, None
|
|
35
|
+
].float().contiguous()
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _synthesis_kernel(window: Tensor) -> Tensor:
|
|
39
|
+
fft = len(window)
|
|
40
|
+
bins = fft // 2 + 1
|
|
41
|
+
unit = torch.eye(bins, dtype=torch.complex128)
|
|
42
|
+
real = torch.fft.irfft(unit, n=fft, dim=-1)
|
|
43
|
+
imaginary = torch.fft.irfft(unit * 1j, n=fft, dim=-1)
|
|
44
|
+
return (torch.cat((real, imaginary)) * window.double())[:, None].float().contiguous()
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class Exportable(nn.Module):
|
|
48
|
+
"""Single-utterance ``TinyVoca`` wrapper rewritten in ONNX-compatible ops."""
|
|
49
|
+
|
|
50
|
+
harmonic_real: Tensor
|
|
51
|
+
harmonic_imaginary: Tensor
|
|
52
|
+
noise_real: Tensor
|
|
53
|
+
noise_imaginary: Tensor
|
|
54
|
+
synth_analysis_real: Tensor
|
|
55
|
+
synth_analysis_imaginary: Tensor
|
|
56
|
+
synth_synthesis: Tensor
|
|
57
|
+
synth_envelope: Tensor
|
|
58
|
+
noise_analysis_real: Tensor
|
|
59
|
+
noise_analysis_imaginary: Tensor
|
|
60
|
+
noise_synthesis: Tensor
|
|
61
|
+
noise_envelope: Tensor
|
|
62
|
+
ramp: Tensor
|
|
63
|
+
|
|
64
|
+
def __init__(self, model: TinyVoca, external_noise: bool = False) -> None:
|
|
65
|
+
super().__init__()
|
|
66
|
+
self.model = model
|
|
67
|
+
self.external_noise = external_noise
|
|
68
|
+
c = model.config
|
|
69
|
+
v = model.vocoder
|
|
70
|
+
causal = list(range(c.cep_causal))
|
|
71
|
+
anti = [c.synth_fft - 1 - i for i in range(c.cep_anti)]
|
|
72
|
+
for name, (real, imaginary) in (
|
|
73
|
+
("harmonic", _cepstrum_matrices(causal + anti, c.synth_fft)),
|
|
74
|
+
("noise", _cepstrum_matrices(list(range(c.noise_cep)), c.noise_fft)),
|
|
75
|
+
):
|
|
76
|
+
self.register_buffer(f"{name}_real", real, persistent=False)
|
|
77
|
+
self.register_buffer(f"{name}_imaginary", imaginary, persistent=False)
|
|
78
|
+
for name, window in (("synth", v.synth_window), ("noise", v.noise_window)):
|
|
79
|
+
real, imaginary = _analysis_kernels(window)
|
|
80
|
+
self.register_buffer(f"{name}_analysis_real", real, persistent=False)
|
|
81
|
+
self.register_buffer(f"{name}_analysis_imaginary", imaginary, persistent=False)
|
|
82
|
+
self.register_buffer(f"{name}_synthesis", _synthesis_kernel(window), persistent=False)
|
|
83
|
+
self.register_buffer(
|
|
84
|
+
f"{name}_envelope",
|
|
85
|
+
(window.double() ** 2).float()[None, None].contiguous(),
|
|
86
|
+
persistent=False,
|
|
87
|
+
)
|
|
88
|
+
self.register_buffer(
|
|
89
|
+
"ramp", torch.arange(c.hop, dtype=torch.float32) / c.hop, persistent=False
|
|
90
|
+
)
|
|
91
|
+
self.phase_steps = 1 << 24
|
|
92
|
+
|
|
93
|
+
def _filter(
|
|
94
|
+
self, source: Tensor, real: Tensor, imaginary: Tensor, name: str, fft: int, hop: int
|
|
95
|
+
) -> Tensor:
|
|
96
|
+
samples = source.shape[1]
|
|
97
|
+
padded = F.pad(source[:, None], (fft // 2, fft))
|
|
98
|
+
s_real = F.conv1d(padded, getattr(self, f"{name}_analysis_real"), stride=hop)
|
|
99
|
+
s_imaginary = F.conv1d(padded, getattr(self, f"{name}_analysis_imaginary"), stride=hop)
|
|
100
|
+
real, imaginary = real.transpose(1, 2), imaginary.transpose(1, 2)
|
|
101
|
+
product = torch.cat(
|
|
102
|
+
(s_real * real - s_imaginary * imaginary, s_real * imaginary + s_imaginary * real),
|
|
103
|
+
dim=1,
|
|
104
|
+
)
|
|
105
|
+
wave = F.conv_transpose1d(product, getattr(self, f"{name}_synthesis"), stride=hop)
|
|
106
|
+
envelope = F.conv_transpose1d(
|
|
107
|
+
torch.ones_like(s_real[:, :1]), getattr(self, f"{name}_envelope"), stride=hop
|
|
108
|
+
)
|
|
109
|
+
return (wave / envelope)[:, 0, fft // 2 : fft // 2 + samples]
|
|
110
|
+
|
|
111
|
+
@staticmethod
|
|
112
|
+
def _response(cepstrum: Tensor, real: Tensor, imaginary: Tensor) -> tuple[Tensor, Tensor]:
|
|
113
|
+
magnitude = torch.exp((cepstrum @ real).clamp(-14, 6))
|
|
114
|
+
angle = cepstrum @ imaginary
|
|
115
|
+
return magnitude * torch.cos(angle), magnitude * torch.sin(angle)
|
|
116
|
+
|
|
117
|
+
def _pulses(self, f0: Tensor) -> Tensor:
|
|
118
|
+
"""Float32 pulse train using 24-bit integer phase accumulation across frames."""
|
|
119
|
+
c = self.model.config
|
|
120
|
+
knots = torch.log(f0.clamp_min(1.0))
|
|
121
|
+
knots = torch.cat((knots, knots[:, -1:]), dim=1)
|
|
122
|
+
low, high = knots[:, :-1, None], knots[:, 1:, None]
|
|
123
|
+
frequency = torch.exp(low * (1 - self.ramp) + high * self.ramp)
|
|
124
|
+
within = torch.cumsum(frequency / c.sample_rate, dim=2)
|
|
125
|
+
whole_frame = within[:, :, -1]
|
|
126
|
+
steps = torch.round((whole_frame - torch.floor(whole_frame)) * self.phase_steps).long()
|
|
127
|
+
start = (
|
|
128
|
+
torch.remainder(torch.cumsum(steps, dim=1) - steps, self.phase_steps).to(within.dtype)
|
|
129
|
+
/ self.phase_steps
|
|
130
|
+
)
|
|
131
|
+
cycles = within + start[:, :, None]
|
|
132
|
+
theta = (2 * math.pi * (cycles - torch.floor(cycles))).reshape(1, -1)
|
|
133
|
+
frequency = frequency.reshape(1, -1)
|
|
134
|
+
harmonics = (c.harmonic_limit * c.sample_rate / frequency).clamp_min(1.0)
|
|
135
|
+
whole = torch.floor(harmonics)
|
|
136
|
+
denominator = 2 * torch.sin(theta / 2)
|
|
137
|
+
tiny = denominator.abs() < 1e-6
|
|
138
|
+
safe_denom = torch.where(tiny, torch.ones_like(denominator), denominator)
|
|
139
|
+
core = torch.where(tiny, whole, torch.sin((whole + 0.5) * theta) / safe_denom - 0.5)
|
|
140
|
+
wave = core + (harmonics - whole) * torch.cos((whole + 1) * theta)
|
|
141
|
+
return wave * torch.sqrt(2.0 / harmonics)
|
|
142
|
+
|
|
143
|
+
def _vocoder(self, hidden: Tensor, f0: Tensor, total: Tensor, noise: Tensor | None) -> Tensor:
|
|
144
|
+
c = self.model.config
|
|
145
|
+
v = self.model.vocoder
|
|
146
|
+
last = total - 1
|
|
147
|
+
|
|
148
|
+
# Harmonic branch
|
|
149
|
+
h = v.harmonic(hidden)
|
|
150
|
+
split = c.cep_causal + c.cep_anti
|
|
151
|
+
steps = v.harmonic_steps
|
|
152
|
+
sub = torch.minimum(
|
|
153
|
+
torch.arange(cast(int, steps * total + 3), device=hidden.device),
|
|
154
|
+
steps * total - 1,
|
|
155
|
+
)
|
|
156
|
+
here = torch.div(sub, steps, rounding_mode="floor")
|
|
157
|
+
following = torch.minimum(torch.div(sub + 1, steps, rounding_mode="floor"), last)
|
|
158
|
+
envelope = 0.5 * (
|
|
159
|
+
h[..., :split].index_select(1, here) + h[..., :split].index_select(1, following)
|
|
160
|
+
)
|
|
161
|
+
envelope = envelope * torch.cat((v.causal_lifter, v.anti_lifter))
|
|
162
|
+
gain = h[..., split:].flatten(1).index_select(1, sub)[..., None]
|
|
163
|
+
envelope = torch.cat((envelope[..., :1] + gain, envelope[..., 1:]), dim=-1)
|
|
164
|
+
harmonic_real, harmonic_imaginary = self._response(
|
|
165
|
+
envelope, self.harmonic_real, self.harmonic_imaginary
|
|
166
|
+
)
|
|
167
|
+
|
|
168
|
+
# Noise branch
|
|
169
|
+
n = v.noise(hidden)
|
|
170
|
+
steps = v.noise_steps
|
|
171
|
+
sub = torch.minimum(
|
|
172
|
+
torch.arange(cast(int, steps * total + 3), device=hidden.device),
|
|
173
|
+
steps * total - 1,
|
|
174
|
+
)
|
|
175
|
+
shape = (
|
|
176
|
+
n[..., : c.noise_cep].index_select(1, torch.div(sub, steps, rounding_mode="floor"))
|
|
177
|
+
* v.noise_lifter
|
|
178
|
+
)
|
|
179
|
+
gain = n[..., c.noise_cep :].flatten(1).index_select(1, sub)[..., None]
|
|
180
|
+
shape = torch.cat((shape[..., :1] + gain, shape[..., 1:]), dim=-1)
|
|
181
|
+
noise_real, noise_imaginary = self._response(shape, self.noise_real, self.noise_imaginary)
|
|
182
|
+
|
|
183
|
+
pulses = self._pulses(f0)
|
|
184
|
+
if noise is None:
|
|
185
|
+
noise = torch.randn_like(pulses)
|
|
186
|
+
else:
|
|
187
|
+
noise = noise.index_select(
|
|
188
|
+
1, torch.arange(cast(int, c.hop * total), device=hidden.device)
|
|
189
|
+
)
|
|
190
|
+
return self._filter(
|
|
191
|
+
pulses, harmonic_real, harmonic_imaginary, "synth", c.synth_fft, c.synth_hop
|
|
192
|
+
) + self._filter(noise, noise_real, noise_imaginary, "noise", c.noise_fft, c.noise_hop)
|
|
193
|
+
|
|
194
|
+
def forward(
|
|
195
|
+
self,
|
|
196
|
+
tokens: Tensor,
|
|
197
|
+
stress: Tensor,
|
|
198
|
+
voice: Tensor,
|
|
199
|
+
speed: Tensor,
|
|
200
|
+
pitch_shift: Tensor,
|
|
201
|
+
pitch_range: Tensor,
|
|
202
|
+
noise: Tensor | None = None,
|
|
203
|
+
) -> tuple[Tensor, Tensor]:
|
|
204
|
+
m = self.model
|
|
205
|
+
c = m.config
|
|
206
|
+
token_mask = torch.ones_like(tokens, dtype=torch.bool)
|
|
207
|
+
hidden = m.encoder(
|
|
208
|
+
m.tokens(tokens) + m.stress(stress) + m.voices(voice)[:, None], token_mask
|
|
209
|
+
)
|
|
210
|
+
|
|
211
|
+
output = m.duration(hidden, token_mask)
|
|
212
|
+
frames = (
|
|
213
|
+
torch.round((torch.exp(output[..., 0].clamp(0, math.log1p(400))) - 1) / speed)
|
|
214
|
+
.long()
|
|
215
|
+
.clamp_min(1)
|
|
216
|
+
)
|
|
217
|
+
present = m.phone_tokens[tokens] | (output[..., 1] > 0)
|
|
218
|
+
durations = torch.where(present, frames, torch.zeros_like(frames))
|
|
219
|
+
|
|
220
|
+
ends = durations.cumsum(1)
|
|
221
|
+
total = ends[0, -1]
|
|
222
|
+
frame = torch.arange(cast(int, total), device=tokens.device)[None]
|
|
223
|
+
index = (ends[:, None, :] <= frame[:, :, None]).sum(-1)
|
|
224
|
+
owned = durations.gather(1, index).clamp_min(1)
|
|
225
|
+
offset = frame - (ends.gather(1, index) - durations.gather(1, index))
|
|
226
|
+
relative = (offset.to(hidden.dtype) + 0.5) / owned
|
|
227
|
+
features = torch.stack(
|
|
228
|
+
(relative, 1 - relative, torch.log1p(owned.to(hidden.dtype)) / 4), dim=-1
|
|
229
|
+
)
|
|
230
|
+
expanded = hidden.gather(1, index[..., None].expand(-1, -1, hidden.shape[-1]))
|
|
231
|
+
expanded = expanded + m.frame_position(features) + m.frame_voices(voice)[:, None]
|
|
232
|
+
frame_mask = torch.ones_like(frame, dtype=torch.bool)
|
|
233
|
+
|
|
234
|
+
prosody_hidden = m.prosody_stack(expanded, frame_mask)
|
|
235
|
+
prosody = m.prosody_head(prosody_hidden)
|
|
236
|
+
vowel = m.vowel_tokens[tokens].gather(1, index)
|
|
237
|
+
voicing = torch.where(vowel, prosody[..., 1].clamp_min(2.0), prosody[..., 1])
|
|
238
|
+
voiced = (voicing > 0).to(prosody.dtype)
|
|
239
|
+
mean = (prosody[..., 0] * voiced).sum(1, keepdim=True) / voiced.sum(
|
|
240
|
+
1, keepdim=True
|
|
241
|
+
).clamp_min(1)
|
|
242
|
+
pitch = mean + (prosody[..., 0] - mean) * pitch_range
|
|
243
|
+
condition = torch.stack(
|
|
244
|
+
(pitch.clamp(-2.0, 1.5), voiced, prosody[..., 2].clamp(-4.5, 3.0)), dim=-1
|
|
245
|
+
)
|
|
246
|
+
acoustic = m.acoustic_stack(prosody_hidden + m.prosody_projection(condition), frame_mask)
|
|
247
|
+
f0 = (c.pitch_reference_hz * torch.exp(pitch + pitch_shift)).clamp(c.f0_min, c.f0_max)
|
|
248
|
+
return self._vocoder(acoustic, f0, total, noise), durations
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def example_inputs(length: int = 24) -> tuple[Tensor, ...]:
|
|
252
|
+
tokens = torch.randint(5, len(SYMBOLS), (1, length))
|
|
253
|
+
|
|
254
|
+
def _scalar(value: float) -> Tensor:
|
|
255
|
+
return torch.tensor([value], dtype=torch.float32)
|
|
256
|
+
|
|
257
|
+
return (
|
|
258
|
+
tokens,
|
|
259
|
+
torch.zeros_like(tokens),
|
|
260
|
+
torch.tensor([0]),
|
|
261
|
+
_scalar(1.0),
|
|
262
|
+
_scalar(0.0),
|
|
263
|
+
_scalar(1.0),
|
|
264
|
+
)
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def export_model(
|
|
268
|
+
model: TinyVoca,
|
|
269
|
+
path: str | Path,
|
|
270
|
+
*,
|
|
271
|
+
external_noise: bool = False,
|
|
272
|
+
opset: int = 17,
|
|
273
|
+
) -> Path:
|
|
274
|
+
"""Export an in-memory ``TinyVoca`` instance directly to an ``.onnx`` file."""
|
|
275
|
+
output = Path(path)
|
|
276
|
+
output.parent.mkdir(parents=True, exist_ok=True)
|
|
277
|
+
wrapper = Exportable(model, external_noise).eval()
|
|
278
|
+
inputs: tuple[Tensor, ...] = example_inputs()
|
|
279
|
+
names = list(INPUTS)
|
|
280
|
+
axes: dict[str, dict[int, str]] = {
|
|
281
|
+
"tokens": {1: "tokens"},
|
|
282
|
+
"stress": {1: "tokens"},
|
|
283
|
+
"audio": {1: "samples"},
|
|
284
|
+
"durations": {1: "tokens"},
|
|
285
|
+
}
|
|
286
|
+
if external_noise:
|
|
287
|
+
inputs = (*inputs, torch.randn(1, model.config.hop * model.config.max_frames))
|
|
288
|
+
names.append("noise")
|
|
289
|
+
axes["noise"] = {1: "noise_samples"}
|
|
290
|
+
with torch.no_grad():
|
|
291
|
+
torch.onnx.export(
|
|
292
|
+
wrapper,
|
|
293
|
+
inputs,
|
|
294
|
+
str(output),
|
|
295
|
+
input_names=names,
|
|
296
|
+
output_names=list(OUTPUTS),
|
|
297
|
+
dynamic_axes=axes,
|
|
298
|
+
opset_version=opset,
|
|
299
|
+
do_constant_folding=True,
|
|
300
|
+
dynamo=False,
|
|
301
|
+
)
|
|
302
|
+
return output
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
def build_metadata(
|
|
306
|
+
model: TinyVoca, meta: CheckpointMeta | None = None, *, external_noise: bool = False
|
|
307
|
+
) -> dict[str, Any]:
|
|
308
|
+
"""Build clean ``model.json`` metadata dictionary for an ONNX export."""
|
|
309
|
+
c = model.config
|
|
310
|
+
inputs = [*INPUTS, "noise"] if external_noise else list(INPUTS)
|
|
311
|
+
return {
|
|
312
|
+
"sample_rate": c.sample_rate,
|
|
313
|
+
"hop": c.hop,
|
|
314
|
+
"voices": list(c.speakers),
|
|
315
|
+
"symbols": list(SYMBOLS),
|
|
316
|
+
"max_tokens": c.max_tokens,
|
|
317
|
+
"max_frames": c.max_frames,
|
|
318
|
+
"step": meta.step if meta is not None else None,
|
|
319
|
+
"inputs": inputs,
|
|
320
|
+
"outputs": list(OUTPUTS),
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def export_onnx(
|
|
325
|
+
checkpoint_dir: str | Path,
|
|
326
|
+
output_dir: str | Path,
|
|
327
|
+
*,
|
|
328
|
+
external_noise: bool = False,
|
|
329
|
+
opset: int = 17,
|
|
330
|
+
) -> tuple[Path, Path]:
|
|
331
|
+
"""Export a safetensors checkpoint directory into ``output_dir/{model.onnx,model.json}``."""
|
|
332
|
+
model, meta = load_checkpoint(checkpoint_dir, device="cpu")
|
|
333
|
+
out_dir = Path(output_dir)
|
|
334
|
+
out_dir.mkdir(parents=True, exist_ok=True)
|
|
335
|
+
onnx_path = export_model(
|
|
336
|
+
model, out_dir / "model.onnx", external_noise=external_noise, opset=opset
|
|
337
|
+
)
|
|
338
|
+
meta_path = out_dir / "model.json"
|
|
339
|
+
meta_path.write_text(
|
|
340
|
+
json.dumps(
|
|
341
|
+
build_metadata(model, meta, external_noise=external_noise), ensure_ascii=False, indent=2
|
|
342
|
+
)
|
|
343
|
+
+ "\n",
|
|
344
|
+
encoding="utf-8",
|
|
345
|
+
)
|
|
346
|
+
return onnx_path, meta_path
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def main() -> None:
|
|
350
|
+
parser = argparse.ArgumentParser(
|
|
351
|
+
description="Export a tiny-voca-ru safetensors checkpoint to ONNX"
|
|
352
|
+
)
|
|
353
|
+
parser.add_argument(
|
|
354
|
+
"--checkpoint",
|
|
355
|
+
type=Path,
|
|
356
|
+
required=True,
|
|
357
|
+
help="Checkpoint directory (model.safetensors + config.json)",
|
|
358
|
+
)
|
|
359
|
+
parser.add_argument(
|
|
360
|
+
"--output", type=Path, required=True, help="Output directory for model.onnx and model.json"
|
|
361
|
+
)
|
|
362
|
+
parser.add_argument(
|
|
363
|
+
"--external-noise", action="store_true", help="Accept noise waveform as an ONNX input"
|
|
364
|
+
)
|
|
365
|
+
parser.add_argument("--opset", type=int, default=17)
|
|
366
|
+
args = parser.parse_args()
|
|
367
|
+
onnx_path, meta_path = export_onnx(
|
|
368
|
+
args.checkpoint, args.output, external_noise=args.external_noise, opset=args.opset
|
|
369
|
+
)
|
|
370
|
+
print(f"Exported {onnx_path} ({onnx_path.stat().st_size / 1e6:.2f} MB) and {meta_path.name}")
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
if __name__ == "__main__":
|
|
374
|
+
main()
|