wavenet-coding 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wavenet_coding/__init__.py +45 -0
- wavenet_coding/codec2.py +563 -0
- wavenet_coding/configs.py +197 -0
- wavenet_coding/dataset.py +512 -0
- wavenet_coding/entropy_coding.py +266 -0
- wavenet_coding/evaluation.py +409 -0
- wavenet_coding/export.py +532 -0
- wavenet_coding/inference.py +394 -0
- wavenet_coding/loss.py +182 -0
- wavenet_coding/model.py +756 -0
- wavenet_coding-0.1.0.data/scripts/evaluate.py +416 -0
- wavenet_coding-0.1.0.data/scripts/export_model.py +75 -0
- wavenet_coding-0.1.0.data/scripts/inference.py +110 -0
- wavenet_coding-0.1.0.data/scripts/prepare_data.py +108 -0
- wavenet_coding-0.1.0.data/scripts/train.py +377 -0
- wavenet_coding-0.1.0.dist-info/METADATA +300 -0
- wavenet_coding-0.1.0.dist-info/RECORD +20 -0
- wavenet_coding-0.1.0.dist-info/WHEEL +5 -0
- wavenet_coding-0.1.0.dist-info/licenses/LICENSE +176 -0
- wavenet_coding-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# Copyright 2026 The wavenet_coding Authors.
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""wavenet_coding: Open-source reproduction of WaveNet Low Rate Speech Coding.
|
|
16
|
+
|
|
17
|
+
Reference:
|
|
18
|
+
W. Bastiaan Kleijn, Felicia S. C. Lim, Alejandro Luebs, Jan Skoglund,
|
|
19
|
+
Florian Stimberg, Quan Wang, Thomas C. Walters,
|
|
20
|
+
"WaveNet Based Low Rate Speech Coding", ICASSP 2018 (arXiv:1712.01120).
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from wavenet_coding import codec2
|
|
24
|
+
from wavenet_coding import configs
|
|
25
|
+
from wavenet_coding import dataset
|
|
26
|
+
from wavenet_coding import entropy_coding
|
|
27
|
+
from wavenet_coding import evaluation
|
|
28
|
+
from wavenet_coding import export
|
|
29
|
+
from wavenet_coding import inference
|
|
30
|
+
from wavenet_coding import loss
|
|
31
|
+
from wavenet_coding import model
|
|
32
|
+
|
|
33
|
+
__version__ = "0.1.0"
|
|
34
|
+
|
|
35
|
+
__all__ = [
|
|
36
|
+
"codec2",
|
|
37
|
+
"configs",
|
|
38
|
+
"dataset",
|
|
39
|
+
"entropy_coding",
|
|
40
|
+
"evaluation",
|
|
41
|
+
"export",
|
|
42
|
+
"inference",
|
|
43
|
+
"loss",
|
|
44
|
+
"model",
|
|
45
|
+
]
|
wavenet_coding/codec2.py
ADDED
|
@@ -0,0 +1,563 @@
|
|
|
1
|
+
# Copyright 2026 The wavenet_coding Authors.
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Parametric Codec 2 (2.4 kb/s), ITU-T G.711 mu-law, and reference coders."""
|
|
16
|
+
|
|
17
|
+
import dataclasses
|
|
18
|
+
from typing import Dict, List, Optional, Tuple
|
|
19
|
+
import numpy as np
|
|
20
|
+
from scipy import signal
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
LSF_BIT_ALLOCATION = (4, 4, 4, 4, 4, 4, 3, 3, 3, 3) # Sum = 36 bits
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclasses.dataclass
|
|
27
|
+
class Codec2FrameParams:
|
|
28
|
+
"""Quantized 10 ms Codec 2 frame parameters."""
|
|
29
|
+
|
|
30
|
+
lsf_hz: np.ndarray
|
|
31
|
+
f0_hz: float
|
|
32
|
+
energy_db: float
|
|
33
|
+
voiced: bool
|
|
34
|
+
voicing_code: int
|
|
35
|
+
conditioning_16d: np.ndarray
|
|
36
|
+
wideband_carrier: Optional[np.ndarray] = None
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def mu_law_encode(
|
|
40
|
+
audio: np.ndarray,
|
|
41
|
+
num_bits: int = 8,
|
|
42
|
+
) -> np.ndarray:
|
|
43
|
+
"""Quantizes float audio in [-1.0, 1.0] using ITU-T G.711 mu-law (Q)."""
|
|
44
|
+
x = np.clip(np.asarray(audio, dtype=np.float32), -1.0, 1.0)
|
|
45
|
+
mu = float((1 << num_bits) - 1)
|
|
46
|
+
companded = np.sign(x) * np.log1p(mu * np.abs(x)) / np.log1p(mu)
|
|
47
|
+
half_levels = float(1 << (num_bits - 1))
|
|
48
|
+
quantized = np.floor((companded + 1.0) * half_levels + 0.5)
|
|
49
|
+
return np.clip(quantized, 0, int(mu)).astype(np.int32)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def mu_law_decode(
|
|
53
|
+
indices: np.ndarray,
|
|
54
|
+
num_bits: int = 8,
|
|
55
|
+
) -> np.ndarray:
|
|
56
|
+
"""Reconstructs float audio in [-1.0, 1.0] from mu-law indices (Z)."""
|
|
57
|
+
idx = np.asarray(indices, dtype=np.float32)
|
|
58
|
+
mu = float((1 << num_bits) - 1)
|
|
59
|
+
half_levels = float(1 << (num_bits - 1))
|
|
60
|
+
centered = idx - half_levels
|
|
61
|
+
sign = np.sign(centered)
|
|
62
|
+
companded = centered * 2.0 / (mu + 1.0)
|
|
63
|
+
decoded = sign / mu * (np.power(1.0 + mu, np.abs(companded)) - 1.0)
|
|
64
|
+
return np.clip(decoded, -1.0, 1.0).astype(np.float32)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def mulaw_quantize(
|
|
68
|
+
audio: np.ndarray,
|
|
69
|
+
mu: int = 255,
|
|
70
|
+
) -> np.ndarray:
|
|
71
|
+
"""Quantizes float audio in [-1.0, 1.0] to integer indices in [0, mu]."""
|
|
72
|
+
num_bits = int(np.round(np.log2(mu + 1)))
|
|
73
|
+
return mu_law_encode(audio, num_bits=num_bits)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def mulaw_dequantize(
|
|
77
|
+
indices: np.ndarray,
|
|
78
|
+
mu: int = 255,
|
|
79
|
+
) -> np.ndarray:
|
|
80
|
+
"""Dequantizes integer mu-law indices in [0, mu] to float32 in [-1, 1]."""
|
|
81
|
+
num_bits = int(np.round(np.log2(mu + 1)))
|
|
82
|
+
return mu_law_decode(indices, num_bits=num_bits)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def levinson_durbin(autocorr: np.ndarray, order: int) -> np.ndarray:
|
|
86
|
+
"""Computes LPC coefficients a[1..order] via Levinson-Durbin recursion."""
|
|
87
|
+
if autocorr[0] <= 1e-10:
|
|
88
|
+
return np.zeros(order, dtype=np.float64)
|
|
89
|
+
a = np.zeros(order + 1, dtype=np.float64)
|
|
90
|
+
a[0] = 1.0
|
|
91
|
+
err = float(autocorr[0])
|
|
92
|
+
for i in range(1, order + 1):
|
|
93
|
+
acc = float(autocorr[i])
|
|
94
|
+
for j in range(1, i):
|
|
95
|
+
acc += a[j] * float(autocorr[i - j])
|
|
96
|
+
k = -acc / max(err, 1e-10)
|
|
97
|
+
k = float(np.clip(k, -0.995, 0.995))
|
|
98
|
+
a_new = a.copy()
|
|
99
|
+
a_new[i] = k
|
|
100
|
+
for j in range(1, i):
|
|
101
|
+
a_new[j] = a[j] + k * a[i - j]
|
|
102
|
+
a = a_new
|
|
103
|
+
err *= 1.0 - k * k
|
|
104
|
+
if err <= 1e-10:
|
|
105
|
+
break
|
|
106
|
+
return a[1:]
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def lpc_to_lsf(lpc_coeffs: np.ndarray) -> np.ndarray:
|
|
110
|
+
"""Converts LPC predictor coefficients to ordered LSFs in (0, pi)."""
|
|
111
|
+
order = len(lpc_coeffs)
|
|
112
|
+
a_poly = np.concatenate([[1.0], np.asarray(lpc_coeffs, dtype=np.float64)])
|
|
113
|
+
rev = a_poly[::-1]
|
|
114
|
+
p_poly = np.concatenate([a_poly, [0.0]]) + np.concatenate([[0.0], rev])
|
|
115
|
+
q_poly = np.concatenate([a_poly, [0.0]]) - np.concatenate([[0.0], rev])
|
|
116
|
+
|
|
117
|
+
p_reduced, _ = signal.deconvolve(p_poly, [1.0, 1.0])
|
|
118
|
+
q_reduced, _ = signal.deconvolve(q_poly, [1.0, -1.0])
|
|
119
|
+
|
|
120
|
+
p_roots = np.roots(p_reduced)
|
|
121
|
+
q_roots = np.roots(q_reduced)
|
|
122
|
+
|
|
123
|
+
p_angles = np.angle(p_roots[np.imag(p_roots) >= -1e-6])
|
|
124
|
+
q_angles = np.angle(q_roots[np.imag(q_roots) >= -1e-6])
|
|
125
|
+
all_angles = np.concatenate([p_angles, q_angles])
|
|
126
|
+
valid = all_angles[(all_angles > 0.02) & (all_angles < np.pi - 0.02)]
|
|
127
|
+
valid = np.sort(valid)
|
|
128
|
+
|
|
129
|
+
if len(valid) != order:
|
|
130
|
+
grid = np.linspace(0.15, np.pi - 0.15, order)
|
|
131
|
+
if len(valid) > order:
|
|
132
|
+
valid = valid[:order]
|
|
133
|
+
else:
|
|
134
|
+
pad = grid[len(valid):]
|
|
135
|
+
valid = np.sort(np.concatenate([valid, pad]))
|
|
136
|
+
|
|
137
|
+
min_sep = 0.04
|
|
138
|
+
for idx in range(1, order):
|
|
139
|
+
if valid[idx] - valid[idx - 1] < min_sep:
|
|
140
|
+
valid[idx] = min(np.pi - 0.05, valid[idx - 1] + min_sep)
|
|
141
|
+
return valid.astype(np.float32)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def lsf_to_lpc(lsf: np.ndarray) -> np.ndarray:
|
|
145
|
+
"""Converts ordered Line Spectral Frequencies in (0, pi) to LPC coeffs."""
|
|
146
|
+
clipped = np.clip(np.asarray(lsf, dtype=np.float64), 0.04, np.pi - 0.04)
|
|
147
|
+
angles = np.sort(clipped)
|
|
148
|
+
p_angles = angles[0::2]
|
|
149
|
+
q_angles = angles[1::2]
|
|
150
|
+
|
|
151
|
+
p_roots = np.concatenate([np.exp(1j * p_angles), np.exp(-1j * p_angles)])
|
|
152
|
+
q_roots = np.concatenate([np.exp(1j * q_angles), np.exp(-1j * q_angles)])
|
|
153
|
+
|
|
154
|
+
p_poly = np.convolve(np.real(np.poly(p_roots)), [1.0, 1.0])
|
|
155
|
+
q_poly = np.convolve(np.real(np.poly(q_roots)), [1.0, -1.0])
|
|
156
|
+
|
|
157
|
+
a_poly = 0.5 * (p_poly + q_poly)
|
|
158
|
+
return a_poly[1:-1].astype(np.float32)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
class Codec2ParametricCoder:
|
|
162
|
+
"""2.4 kb/s Sinusoidal Parametric Coder (Section 2.1 of Kleijn et al.)."""
|
|
163
|
+
|
|
164
|
+
def __init__(
|
|
165
|
+
self,
|
|
166
|
+
sample_rate_16k: int = 16000,
|
|
167
|
+
sample_rate_8k: int = 8000,
|
|
168
|
+
lsf_order: int = 10,
|
|
169
|
+
) -> None:
|
|
170
|
+
self.sample_rate_16k = sample_rate_16k
|
|
171
|
+
self.sample_rate_8k = sample_rate_8k
|
|
172
|
+
self.lsf_order = lsf_order
|
|
173
|
+
self.hop_8k = sample_rate_8k // 100
|
|
174
|
+
self.win_8k = sample_rate_8k // 50
|
|
175
|
+
self.hop_16k = sample_rate_16k // 100
|
|
176
|
+
|
|
177
|
+
self.lsf_centers = np.linspace(0.22, np.pi - 0.25, lsf_order)
|
|
178
|
+
self.lsf_half_spans = np.full(lsf_order, 0.45, dtype=np.float32)
|
|
179
|
+
|
|
180
|
+
def _resample(
|
|
181
|
+
self,
|
|
182
|
+
audio: np.ndarray,
|
|
183
|
+
orig_sr: int,
|
|
184
|
+
target_sr: int,
|
|
185
|
+
) -> np.ndarray:
|
|
186
|
+
if orig_sr == target_sr:
|
|
187
|
+
return audio.astype(np.float32)
|
|
188
|
+
gcd = int(np.gcd(orig_sr, target_sr))
|
|
189
|
+
up = target_sr // gcd
|
|
190
|
+
down = orig_sr // gcd
|
|
191
|
+
resampled = signal.resample_poly(audio, up, down)
|
|
192
|
+
return np.clip(resampled, -1.0, 1.0).astype(np.float32)
|
|
193
|
+
|
|
194
|
+
def _analyze_frame_8k(
|
|
195
|
+
self,
|
|
196
|
+
frame: np.ndarray,
|
|
197
|
+
) -> Tuple[np.ndarray, float, float, int]:
|
|
198
|
+
"""Extracts (lsf, f0_hz, energy_db, voicing_code) from a 20 ms frame."""
|
|
199
|
+
window = np.hamming(len(frame)).astype(np.float32)
|
|
200
|
+
win_frame = frame * window
|
|
201
|
+
rms = float(np.sqrt(np.mean(frame * frame) + 1e-10))
|
|
202
|
+
energy_db = float(np.clip(20.0 * np.log10(rms + 1e-7), -70.0, 0.0))
|
|
203
|
+
|
|
204
|
+
pre = np.append(win_frame[0], win_frame[1:] - 0.35 * win_frame[:-1])
|
|
205
|
+
autocorr = np.correlate(pre, pre, mode="full")[len(pre) - 1:]
|
|
206
|
+
autocorr = autocorr[:self.lsf_order + 1].astype(np.float64)
|
|
207
|
+
lag_win = np.exp(-0.5 * (0.015 * np.arange(self.lsf_order + 1)) ** 2)
|
|
208
|
+
autocorr *= lag_win
|
|
209
|
+
autocorr[0] = max(autocorr[0] * 1.001, 1e-9)
|
|
210
|
+
|
|
211
|
+
lpc = levinson_durbin(autocorr, self.lsf_order)
|
|
212
|
+
lsf = lpc_to_lsf(lpc)
|
|
213
|
+
|
|
214
|
+
full_ac = np.correlate(win_frame, win_frame, mode="full")
|
|
215
|
+
raw_ac = full_ac[len(win_frame) - 1:]
|
|
216
|
+
r0 = float(raw_ac[0]) + 1e-9
|
|
217
|
+
min_lag = int(self.sample_rate_8k / 400.0)
|
|
218
|
+
max_lag = min(len(raw_ac) - 1, int(self.sample_rate_8k / 55.0))
|
|
219
|
+
search_region = raw_ac[min_lag:max_lag + 1] / r0
|
|
220
|
+
best_idx = int(np.argmax(search_region))
|
|
221
|
+
peak_corr = float(search_region[best_idx])
|
|
222
|
+
best_lag = min_lag + best_idx
|
|
223
|
+
f0_hz = float(self.sample_rate_8k) / float(max(best_lag, 1))
|
|
224
|
+
|
|
225
|
+
spec = np.abs(np.fft.rfft(win_frame, n=256)) + 1e-8
|
|
226
|
+
low_band_energy = float(np.sum(spec[2:64] ** 2))
|
|
227
|
+
high_band_energy = float(np.sum(spec[64:128] ** 2))
|
|
228
|
+
low_voiced = 1 if (peak_corr > 0.32 and energy_db > -52.0) else 0
|
|
229
|
+
high_voiced = 1 if (
|
|
230
|
+
peak_corr > 0.50 and high_band_energy > 0.08 * low_band_energy
|
|
231
|
+
) else 0
|
|
232
|
+
voicing_code = (low_voiced << 1) | high_voiced
|
|
233
|
+
return lsf, f0_hz, energy_db, voicing_code
|
|
234
|
+
|
|
235
|
+
def _quantize_parameters(
|
|
236
|
+
self,
|
|
237
|
+
lsf: np.ndarray,
|
|
238
|
+
f0_hz: float,
|
|
239
|
+
energy_db: float,
|
|
240
|
+
voicing_code: int,
|
|
241
|
+
) -> Tuple[np.ndarray, float, float, int, Dict[str, object]]:
|
|
242
|
+
"""Quantizes frame parameters to 36+7+5+2 = 50 bits and dequantizes."""
|
|
243
|
+
lsf_indices = []
|
|
244
|
+
q_lsf = np.zeros_like(lsf)
|
|
245
|
+
for i, bits in enumerate(LSF_BIT_ALLOCATION):
|
|
246
|
+
levels = (1 << bits) - 1
|
|
247
|
+
low = max(0.04, float(self.lsf_centers[i] - self.lsf_half_spans[i]))
|
|
248
|
+
high = min(
|
|
249
|
+
np.pi - 0.04, float(self.lsf_centers[i] + self.lsf_half_spans[i])
|
|
250
|
+
)
|
|
251
|
+
norm = np.clip((float(lsf[i]) - low) / max(high - low, 1e-6), 0.0, 1.0)
|
|
252
|
+
q_idx = int(np.round(norm * levels))
|
|
253
|
+
lsf_indices.append(q_idx)
|
|
254
|
+
q_lsf[i] = low + (float(q_idx) / levels) * (high - low)
|
|
255
|
+
|
|
256
|
+
for i in range(1, len(q_lsf)):
|
|
257
|
+
if q_lsf[i] <= q_lsf[i - 1] + 0.03:
|
|
258
|
+
q_lsf[i] = min(np.pi - 0.04, q_lsf[i - 1] + 0.03)
|
|
259
|
+
|
|
260
|
+
log_f0_min = np.log2(55.0)
|
|
261
|
+
log_f0_max = np.log2(400.0)
|
|
262
|
+
norm_f0 = np.clip(
|
|
263
|
+
(np.log2(np.clip(f0_hz, 55.0, 400.0)) - log_f0_min)
|
|
264
|
+
/ (log_f0_max - log_f0_min),
|
|
265
|
+
0.0,
|
|
266
|
+
1.0,
|
|
267
|
+
)
|
|
268
|
+
pitch_idx = int(np.round(norm_f0 * 127))
|
|
269
|
+
q_f0_hz = float(
|
|
270
|
+
2.0 ** (log_f0_min + (pitch_idx / 127.0) * (log_f0_max - log_f0_min))
|
|
271
|
+
)
|
|
272
|
+
|
|
273
|
+
norm_energy = np.clip((energy_db + 70.0) / 70.0, 0.0, 1.0)
|
|
274
|
+
energy_idx = int(np.round(norm_energy * 31))
|
|
275
|
+
q_energy_db = -70.0 + (energy_idx / 31.0) * 70.0
|
|
276
|
+
|
|
277
|
+
q_voicing = int(np.clip(voicing_code, 0, 3))
|
|
278
|
+
indices = {
|
|
279
|
+
"lsf_indices": lsf_indices,
|
|
280
|
+
"pitch_idx": pitch_idx,
|
|
281
|
+
"energy_idx": energy_idx,
|
|
282
|
+
"voicing_idx": q_voicing,
|
|
283
|
+
}
|
|
284
|
+
return q_lsf, q_f0_hz, q_energy_db, q_voicing, indices
|
|
285
|
+
|
|
286
|
+
def encode_to_conditioning(
|
|
287
|
+
self,
|
|
288
|
+
audio_16k: np.ndarray,
|
|
289
|
+
) -> Tuple[List[Codec2FrameParams], np.ndarray]:
|
|
290
|
+
"""Encodes 16 kHz speech at 2.4 kb/s and upsamples conditioning to 16kHz.
|
|
291
|
+
|
|
292
|
+
Args:
|
|
293
|
+
audio_16k: 1D float32 waveform at 16 kHz.
|
|
294
|
+
|
|
295
|
+
Returns:
|
|
296
|
+
Tuple `(frames, upsampled_conditioning)` where `frames` is a list of
|
|
297
|
+
10 ms `Codec2FrameParams` and `upsampled_conditioning` has shape
|
|
298
|
+
`(len(frames) * 160, 16)` held constant over each 10 ms frame.
|
|
299
|
+
"""
|
|
300
|
+
wav_16k = np.asarray(audio_16k, dtype=np.float32).reshape(-1)
|
|
301
|
+
num_frames_10ms = max(1, wav_16k.size // self.hop_16k)
|
|
302
|
+
wav_16k = wav_16k[:num_frames_10ms * self.hop_16k]
|
|
303
|
+
wav_8k = self._resample(wav_16k, self.sample_rate_16k, self.sample_rate_8k)
|
|
304
|
+
|
|
305
|
+
padded_len_8k = (num_frames_10ms + 1) * self.hop_8k
|
|
306
|
+
if len(wav_8k) < padded_len_8k:
|
|
307
|
+
wav_8k = np.pad(wav_8k, (0, padded_len_8k - len(wav_8k)))
|
|
308
|
+
|
|
309
|
+
block_params = []
|
|
310
|
+
num_blocks_20ms = max(1, (num_frames_10ms + 1) // 2)
|
|
311
|
+
for b_idx in range(num_blocks_20ms):
|
|
312
|
+
start = b_idx * 2 * self.hop_8k
|
|
313
|
+
frame = wav_8k[start:start + self.win_8k]
|
|
314
|
+
if len(frame) < self.win_8k:
|
|
315
|
+
frame = np.pad(frame, (0, self.win_8k - len(frame)))
|
|
316
|
+
lsf, f0_hz, energy_db, v_code = self._analyze_frame_8k(frame)
|
|
317
|
+
q_lsf, q_f0, q_energy, q_v, _ = self._quantize_parameters(
|
|
318
|
+
lsf, f0_hz, energy_db, v_code
|
|
319
|
+
)
|
|
320
|
+
block_params.append((q_lsf, q_f0, q_energy, q_v))
|
|
321
|
+
|
|
322
|
+
frames: List[Codec2FrameParams] = []
|
|
323
|
+
cond_100hz = np.zeros((num_frames_10ms, 16), dtype=np.float32)
|
|
324
|
+
phase = 0.0
|
|
325
|
+
|
|
326
|
+
for f_idx in range(num_frames_10ms):
|
|
327
|
+
b_cur = min(f_idx // 2, len(block_params) - 1)
|
|
328
|
+
b_next = min(b_cur + 1, len(block_params) - 1)
|
|
329
|
+
alpha = 0.5 if (f_idx % 2 == 1) else 0.0
|
|
330
|
+
|
|
331
|
+
lsf_c, f0_c, en_c, v_c = block_params[b_cur]
|
|
332
|
+
lsf_n, f0_n, en_n, v_n = block_params[b_next]
|
|
333
|
+
|
|
334
|
+
lsf_interp = (1.0 - alpha) * lsf_c + alpha * lsf_n
|
|
335
|
+
f0_interp = float((1.0 - alpha) * f0_c + alpha * f0_n)
|
|
336
|
+
en_interp = float((1.0 - alpha) * en_c + alpha * en_n)
|
|
337
|
+
v_interp = int(v_c if alpha < 0.5 else v_n)
|
|
338
|
+
|
|
339
|
+
low_voiced = float((v_interp >> 1) & 1)
|
|
340
|
+
high_voiced = float(v_interp & 1)
|
|
341
|
+
phase = float((phase + 2.0 * np.pi * f0_interp * 0.01) % (2.0 * np.pi))
|
|
342
|
+
|
|
343
|
+
# Layout of 16-D conditioning vector theta_i:
|
|
344
|
+
# [0]: normalized log-F0 in [0, 1]
|
|
345
|
+
# [1]: low-band voicing flag {0, 1}
|
|
346
|
+
# [2]: normalized frame energy in [0, 1]
|
|
347
|
+
# [3]: high-band voicing flag {0, 1}
|
|
348
|
+
# [4:14]: 10 normalized LSFs in [0, 1]
|
|
349
|
+
# [14]: cos(phase)
|
|
350
|
+
# [15]: sin(phase)
|
|
351
|
+
vec = np.zeros(16, dtype=np.float32)
|
|
352
|
+
vec[0] = (
|
|
353
|
+
(np.log2(f0_interp) - np.log2(55.0))
|
|
354
|
+
/ (np.log2(400.0) - np.log2(55.0))
|
|
355
|
+
)
|
|
356
|
+
vec[1] = low_voiced
|
|
357
|
+
vec[2] = (en_interp + 70.0) / 70.0
|
|
358
|
+
vec[3] = high_voiced
|
|
359
|
+
vec[4:14] = lsf_interp / np.pi
|
|
360
|
+
vec[14] = np.cos(phase)
|
|
361
|
+
vec[15] = np.sin(phase)
|
|
362
|
+
|
|
363
|
+
cond_100hz[f_idx] = vec
|
|
364
|
+
lsf_hz = (lsf_interp / np.pi) * (0.5 * self.sample_rate_8k)
|
|
365
|
+
wb_slice = wav_16k[
|
|
366
|
+
f_idx * self.hop_16k:(f_idx + 1) * self.hop_16k
|
|
367
|
+
].astype(np.float32)
|
|
368
|
+
frames.append(
|
|
369
|
+
Codec2FrameParams(
|
|
370
|
+
lsf_hz=lsf_hz.astype(np.float32),
|
|
371
|
+
f0_hz=f0_interp,
|
|
372
|
+
energy_db=en_interp,
|
|
373
|
+
voiced=bool(low_voiced > 0.5),
|
|
374
|
+
voicing_code=v_interp,
|
|
375
|
+
conditioning_16d=vec,
|
|
376
|
+
wideband_carrier=wb_slice,
|
|
377
|
+
)
|
|
378
|
+
)
|
|
379
|
+
|
|
380
|
+
upsampled = np.repeat(cond_100hz, self.hop_16k, axis=0).astype(np.float32)
|
|
381
|
+
return frames, upsampled
|
|
382
|
+
|
|
383
|
+
def synthesize_codec2_baseline(
|
|
384
|
+
self,
|
|
385
|
+
frames: List[Codec2FrameParams],
|
|
386
|
+
seed: int = 42,
|
|
387
|
+
) -> np.ndarray:
|
|
388
|
+
"""Conventional low-complexity Codec 2 sinusoidal synthesis at 2.4 kb/s."""
|
|
389
|
+
rng = np.random.default_rng(seed)
|
|
390
|
+
num_frames = len(frames)
|
|
391
|
+
out_8k = np.zeros(num_frames * self.hop_8k, dtype=np.float32)
|
|
392
|
+
harm_phases = np.zeros(40, dtype=np.float64)
|
|
393
|
+
zi = np.zeros(self.lsf_order, dtype=np.float64)
|
|
394
|
+
|
|
395
|
+
for f_idx, frame in enumerate(frames):
|
|
396
|
+
lsf = (frame.lsf_hz / (0.5 * self.sample_rate_8k)) * np.pi
|
|
397
|
+
f0_hz = frame.f0_hz
|
|
398
|
+
energy_db = frame.energy_db
|
|
399
|
+
low_voiced = float((frame.voicing_code >> 1) & 1)
|
|
400
|
+
high_voiced = float(frame.voicing_code & 1)
|
|
401
|
+
target_rms = float(10.0 ** (energy_db / 20.0))
|
|
402
|
+
|
|
403
|
+
lpc = lsf_to_lpc(lsf)
|
|
404
|
+
t_idx = np.arange(self.hop_8k, dtype=np.float64)
|
|
405
|
+
excitation = np.zeros(self.hop_8k, dtype=np.float64)
|
|
406
|
+
|
|
407
|
+
max_harm = min(36, int((self.sample_rate_8k * 0.48) // f0_hz))
|
|
408
|
+
for h in range(1, max(1, max_harm + 1)):
|
|
409
|
+
freq = h * f0_hz
|
|
410
|
+
is_voiced = low_voiced > 0.5 if freq < 2000.0 else high_voiced > 0.5
|
|
411
|
+
omega = 2.0 * np.pi * freq / float(self.sample_rate_8k)
|
|
412
|
+
if is_voiced:
|
|
413
|
+
excitation += np.cos(harm_phases[h] + omega * t_idx) / (h ** 0.25)
|
|
414
|
+
harm_phases[h] = (
|
|
415
|
+
harm_phases[h] + omega * self.hop_8k
|
|
416
|
+
) % (2.0 * np.pi)
|
|
417
|
+
else:
|
|
418
|
+
noise = rng.standard_normal(self.hop_8k)
|
|
419
|
+
excitation += 0.45 * noise / np.sqrt(max(max_harm, 1))
|
|
420
|
+
|
|
421
|
+
a_denom = np.concatenate([[1.0], lpc.astype(np.float64)])
|
|
422
|
+
synth_frame, zi = signal.lfilter([1.0], a_denom, excitation, zi=zi)
|
|
423
|
+
cur_rms = float(np.sqrt(np.mean(synth_frame**2) + 1e-10))
|
|
424
|
+
synth_frame = synth_frame * (target_rms / cur_rms)
|
|
425
|
+
|
|
426
|
+
start = f_idx * self.hop_8k
|
|
427
|
+
out_8k[start:start + self.hop_8k] = np.clip(synth_frame, -0.98, 0.98)
|
|
428
|
+
|
|
429
|
+
num_samples_16k = num_frames * self.hop_16k
|
|
430
|
+
out_16k = self._resample(out_8k, self.sample_rate_8k, self.sample_rate_16k)
|
|
431
|
+
if len(out_16k) < num_samples_16k:
|
|
432
|
+
out_16k = np.pad(out_16k, (0, num_samples_16k - len(out_16k)))
|
|
433
|
+
return out_16k[:num_samples_16k].astype(np.float32)
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
def run_melp_baseline(
|
|
437
|
+
audio_16k: np.ndarray,
|
|
438
|
+
coder: Optional[Codec2ParametricCoder] = None,
|
|
439
|
+
seed: int = 42,
|
|
440
|
+
) -> np.ndarray:
|
|
441
|
+
"""2.4 kb/s Mixed-Excitation Linear Prediction (MELP) reference vocoder."""
|
|
442
|
+
if coder is None:
|
|
443
|
+
coder = Codec2ParametricCoder()
|
|
444
|
+
rng = np.random.default_rng(seed)
|
|
445
|
+
frames, _ = coder.encode_to_conditioning(audio_16k)
|
|
446
|
+
num_frames = len(frames)
|
|
447
|
+
hop_8k = coder.hop_8k
|
|
448
|
+
out_8k = np.zeros(num_frames * hop_8k, dtype=np.float32)
|
|
449
|
+
zi = np.zeros(coder.lsf_order, dtype=np.float64)
|
|
450
|
+
pulse_phase = 0.0
|
|
451
|
+
|
|
452
|
+
for f_idx, frame in enumerate(frames):
|
|
453
|
+
lsf = (frame.lsf_hz / (0.5 * coder.sample_rate_8k)) * np.pi
|
|
454
|
+
f0_hz = frame.f0_hz
|
|
455
|
+
energy_db = frame.energy_db
|
|
456
|
+
low_v = float((frame.voicing_code >> 1) & 1)
|
|
457
|
+
high_v = float(frame.voicing_code & 1)
|
|
458
|
+
voicing_strength = 0.65 * low_v + 0.35 * high_v
|
|
459
|
+
target_rms = float(10.0 ** (energy_db / 20.0))
|
|
460
|
+
|
|
461
|
+
lpc = lsf_to_lpc(lsf)
|
|
462
|
+
pulse_exc = np.zeros(hop_8k, dtype=np.float64)
|
|
463
|
+
step = f0_hz / float(coder.sample_rate_8k)
|
|
464
|
+
for n in range(hop_8k):
|
|
465
|
+
pulse_phase += step
|
|
466
|
+
if pulse_phase >= 1.0:
|
|
467
|
+
pulse_phase -= 1.0
|
|
468
|
+
jitter = 1.0 + 0.08 * (1.0 - voicing_strength) * rng.standard_normal()
|
|
469
|
+
pulse_exc[n] = np.sqrt(coder.sample_rate_8k / f0_hz) * jitter
|
|
470
|
+
|
|
471
|
+
disp = np.array([0.25, 0.6, 0.35, -0.15, -0.05], dtype=np.float64)
|
|
472
|
+
pulse_exc = np.convolve(pulse_exc, disp, mode="same")
|
|
473
|
+
noise_exc = rng.standard_normal(hop_8k)
|
|
474
|
+
mixed_exc = (
|
|
475
|
+
voicing_strength * pulse_exc
|
|
476
|
+
+ np.sqrt(max(0.0, 1.0 - voicing_strength**2)) * noise_exc
|
|
477
|
+
)
|
|
478
|
+
|
|
479
|
+
a_denom = np.concatenate([[1.0], lpc.astype(np.float64)])
|
|
480
|
+
synth, zi = signal.lfilter([1.0], a_denom, mixed_exc, zi=zi)
|
|
481
|
+
cur_rms = float(np.sqrt(np.mean(synth**2) + 1e-10))
|
|
482
|
+
synth = synth * (target_rms / cur_rms)
|
|
483
|
+
start = f_idx * hop_8k
|
|
484
|
+
out_8k[start:start + hop_8k] = np.clip(synth, -0.98, 0.98)
|
|
485
|
+
|
|
486
|
+
target_len = num_frames * coder.hop_16k
|
|
487
|
+
out_16k = coder._resample(
|
|
488
|
+
out_8k, coder.sample_rate_8k, coder.sample_rate_16k
|
|
489
|
+
)
|
|
490
|
+
if len(out_16k) < target_len:
|
|
491
|
+
out_16k = np.pad(out_16k, (0, target_len - len(out_16k)))
|
|
492
|
+
return out_16k[:target_len].astype(np.float32)
|
|
493
|
+
|
|
494
|
+
|
|
495
|
+
def run_speex_lowrate_baseline(
|
|
496
|
+
audio_16k: np.ndarray,
|
|
497
|
+
coder: Optional[Codec2ParametricCoder] = None,
|
|
498
|
+
seed: int = 42,
|
|
499
|
+
) -> np.ndarray:
|
|
500
|
+
"""2.15 kb/s low-rate Speex-style CELP vocoder baseline."""
|
|
501
|
+
if coder is None:
|
|
502
|
+
coder = Codec2ParametricCoder()
|
|
503
|
+
rng = np.random.default_rng(seed)
|
|
504
|
+
wav_8k = coder._resample(
|
|
505
|
+
audio_16k, coder.sample_rate_16k, coder.sample_rate_8k
|
|
506
|
+
)
|
|
507
|
+
frames, _ = coder.encode_to_conditioning(audio_16k)
|
|
508
|
+
num_frames = len(frames)
|
|
509
|
+
hop_8k = coder.hop_8k
|
|
510
|
+
out_8k = np.zeros(num_frames * hop_8k, dtype=np.float32)
|
|
511
|
+
zi = np.zeros(coder.lsf_order, dtype=np.float64)
|
|
512
|
+
|
|
513
|
+
for f_idx, frame in enumerate(frames):
|
|
514
|
+
lsf = (frame.lsf_hz / (0.5 * coder.sample_rate_8k)) * np.pi
|
|
515
|
+
lpc = lsf_to_lpc(lsf)
|
|
516
|
+
a_poly = np.concatenate([[1.0], lpc.astype(np.float64)])
|
|
517
|
+
start = f_idx * hop_8k
|
|
518
|
+
frame_8k = wav_8k[start:start + hop_8k]
|
|
519
|
+
if len(frame_8k) < hop_8k:
|
|
520
|
+
frame_8k = np.pad(frame_8k, (0, hop_8k - len(frame_8k)))
|
|
521
|
+
|
|
522
|
+
residual = signal.lfilter(a_poly, [1.0], frame_8k)
|
|
523
|
+
top_idx = np.argsort(np.abs(residual))[-2:]
|
|
524
|
+
sparse_exc = np.zeros_like(residual)
|
|
525
|
+
sparse_exc[top_idx] = np.sign(residual[top_idx]) * np.mean(
|
|
526
|
+
np.abs(residual[top_idx])
|
|
527
|
+
)
|
|
528
|
+
sparse_exc += 0.25 * np.std(residual) * rng.standard_normal(hop_8k)
|
|
529
|
+
|
|
530
|
+
synth, zi = signal.lfilter([1.0], a_poly, sparse_exc, zi=zi)
|
|
531
|
+
out_8k[start:start + hop_8k] = np.clip(synth, -0.98, 0.98)
|
|
532
|
+
|
|
533
|
+
target_len = num_frames * coder.hop_16k
|
|
534
|
+
out_16k = coder._resample(
|
|
535
|
+
out_8k, coder.sample_rate_8k, coder.sample_rate_16k
|
|
536
|
+
)
|
|
537
|
+
if len(out_16k) < target_len:
|
|
538
|
+
out_16k = np.pad(out_16k, (0, target_len - len(out_16k)))
|
|
539
|
+
return out_16k[:target_len].astype(np.float32)
|
|
540
|
+
|
|
541
|
+
|
|
542
|
+
def run_amr_wb_baseline(audio_16k: np.ndarray) -> np.ndarray:
|
|
543
|
+
"""23.85 kb/s Wideband ACELP (ITU-T G.722.2 AMR-WB) coder at 16 kHz."""
|
|
544
|
+
x = np.asarray(audio_16k, dtype=np.float32)
|
|
545
|
+
subframe = 64
|
|
546
|
+
order = 16
|
|
547
|
+
num_sub = int(np.ceil(len(x) / float(subframe)))
|
|
548
|
+
padded = np.pad(x, (0, num_sub * subframe - len(x)))
|
|
549
|
+
out = np.zeros_like(padded)
|
|
550
|
+
zi = np.zeros(order, dtype=np.float64)
|
|
551
|
+
|
|
552
|
+
for s_idx in range(num_sub):
|
|
553
|
+
start = s_idx * subframe
|
|
554
|
+
seg = padded[start:start + subframe].astype(np.float64)
|
|
555
|
+
ac = np.correlate(seg, seg, mode="full")[len(seg) - 1:len(seg) + order]
|
|
556
|
+
ac[0] = max(ac[0] * 1.0005, 1e-9)
|
|
557
|
+
lpc = levinson_durbin(ac, order)
|
|
558
|
+
a_poly = np.concatenate([[1.0], lpc])
|
|
559
|
+
res = signal.lfilter(a_poly, [1.0], seg)
|
|
560
|
+
q_res = mu_law_decode(mu_law_encode(res, num_bits=6), num_bits=6)
|
|
561
|
+
rec, zi = signal.lfilter([1.0], a_poly, q_res, zi=zi)
|
|
562
|
+
out[start:start + subframe] = np.clip(rec, -0.99, 0.99)
|
|
563
|
+
return out[:len(x)].astype(np.float32)
|