justdata 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- justdata/__init__.py +3 -0
- justdata/acoustic/__init__.py +381 -0
- justdata/acoustic/adapters.py +329 -0
- justdata/acoustic/augment/__init__.py +119 -0
- justdata/acoustic/augment/batch.py +813 -0
- justdata/acoustic/augment/configs.py +204 -0
- justdata/acoustic/augment/patchout.py +331 -0
- justdata/acoustic/augment/spectrogram.py +806 -0
- justdata/acoustic/augment/waveform.py +879 -0
- justdata/acoustic/batching.py +27 -0
- justdata/acoustic/channel.py +53 -0
- justdata/acoustic/compat/__init__.py +95 -0
- justdata/acoustic/compat/ast.py +626 -0
- justdata/acoustic/compat/ced.py +152 -0
- justdata/acoustic/compat/efficientat.py +243 -0
- justdata/acoustic/compat/panns.py +115 -0
- justdata/acoustic/compat/passt.py +191 -0
- justdata/acoustic/configs.py +519 -0
- justdata/acoustic/corruptions/__init__.py +33 -0
- justdata/acoustic/corruptions/codec.py +92 -0
- justdata/acoustic/corruptions/datasets.py +155 -0
- justdata/acoustic/corruptions/device.py +39 -0
- justdata/acoustic/corruptions/dynamics.py +55 -0
- justdata/acoustic/corruptions/filters.py +105 -0
- justdata/acoustic/corruptions/noise.py +173 -0
- justdata/acoustic/corruptions/registry.py +230 -0
- justdata/acoustic/corruptions/reverb.py +49 -0
- justdata/acoustic/corruptions/timefreq.py +127 -0
- justdata/acoustic/datasets.py +52 -0
- justdata/acoustic/dcase2025.py +814 -0
- justdata/acoustic/decoding.py +73 -0
- justdata/acoustic/eval_views.py +54 -0
- justdata/acoustic/evals.py +71 -0
- justdata/acoustic/filters.py +18 -0
- justdata/acoustic/frontends/__init__.py +37 -0
- justdata/acoustic/frontends/ast.py +116 -0
- justdata/acoustic/frontends/kaldi.py +34 -0
- justdata/acoustic/frontends/log.py +94 -0
- justdata/acoustic/frontends/mel.py +173 -0
- justdata/acoustic/frontends/mfcc.py +26 -0
- justdata/acoustic/frontends/pcen.py +26 -0
- justdata/acoustic/frontends/stft.py +174 -0
- justdata/acoustic/jax.py +23 -0
- justdata/acoustic/labels.py +169 -0
- justdata/acoustic/layouts.py +86 -0
- justdata/acoustic/metadata.py +152 -0
- justdata/acoustic/normalization.py +196 -0
- justdata/acoustic/pipelines.py +112 -0
- justdata/acoustic/postprocessing.py +328 -0
- justdata/acoustic/preprocessing.py +175 -0
- justdata/acoustic/presets.py +257 -0
- justdata/acoustic/registry.py +381 -0
- justdata/acoustic/resampling.py +218 -0
- justdata/acoustic/schema.py +51 -0
- justdata/acoustic/segment.py +369 -0
- justdata/acoustic/sources.py +503 -0
- justdata/acoustic/stages.py +59 -0
- justdata/acoustic/stats.py +50 -0
- justdata/acoustic/tasks.py +375 -0
- justdata/audio/__init__.py +20 -0
- justdata/core/__init__.py +74 -0
- justdata/core/adapters.py +46 -0
- justdata/core/augmentations.py +34 -0
- justdata/core/filters.py +94 -0
- justdata/core/finalization.py +173 -0
- justdata/core/label_mixing.py +129 -0
- justdata/core/loader.py +384 -0
- justdata/core/metadata.py +250 -0
- justdata/core/presets.py +155 -0
- justdata/core/registry.py +229 -0
- justdata/core/sources.py +87 -0
- justdata/core/stats.py +227 -0
- justdata/corruptions.py +52 -0
- justdata/py.typed +0 -0
- justdata/vision/__init__.py +17 -0
- justdata/vision/augmentations/__init__.py +31 -0
- justdata/vision/augmentations/auto.py +480 -0
- justdata/vision/augmentations/color.py +163 -0
- justdata/vision/augmentations/composed.py +156 -0
- justdata/vision/augmentations/geometric.py +302 -0
- justdata/vision/augmentations/mixing.py +232 -0
- justdata/vision/augmentations/registry.py +142 -0
- justdata/vision/corruptions/__init__.py +16 -0
- justdata/vision/corruptions/blur.py +65 -0
- justdata/vision/corruptions/digital.py +26 -0
- justdata/vision/corruptions/noise.py +20 -0
- justdata/vision/corruptions/registry.py +113 -0
- justdata/vision/corruptions/weather.py +100 -0
- justdata/vision/datasets.py +34 -0
- justdata/vision/encodings/__init__.py +1 -0
- justdata/vision/encodings/heatmaps.py +196 -0
- justdata/vision/minic.py +69 -0
- justdata/vision/pipelines.py +62 -0
- justdata/vision/presets.py +509 -0
- justdata/vision/sources.py +1080 -0
- justdata/vision/stages.py +271 -0
- justdata/vision/tasks/__init__.py +1 -0
- justdata/vision/tasks/classification.py +321 -0
- justdata/vision/tasks/depth.py +25 -0
- justdata/vision/tasks/segmentation.py +194 -0
- justdata/vision/transforms.py +113 -0
- justdata/vision/utils.py +741 -0
- justdata-1.0.0.dist-info/METADATA +860 -0
- justdata-1.0.0.dist-info/RECORD +106 -0
- justdata-1.0.0.dist-info/WHEEL +4 -0
- justdata-1.0.0.dist-info/licenses/LICENSE +23 -0
|
@@ -0,0 +1,626 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any, Mapping
|
|
4
|
+
|
|
5
|
+
import tensorflow as tf
|
|
6
|
+
|
|
7
|
+
from justdata.acoustic.configs import (
|
|
8
|
+
AudioPreprocessConfig,
|
|
9
|
+
AudioPreset,
|
|
10
|
+
FrontendConfig,
|
|
11
|
+
LabelTransformConfig,
|
|
12
|
+
LogCompressionConfig,
|
|
13
|
+
MelConfig,
|
|
14
|
+
STFTConfig,
|
|
15
|
+
SegmentStrategyConfig,
|
|
16
|
+
)
|
|
17
|
+
from justdata.acoustic.labels import transform_label
|
|
18
|
+
from justdata.acoustic.layouts import convert_audio_layout
|
|
19
|
+
from justdata.acoustic.preprocessing import make_preprocessing
|
|
20
|
+
from justdata.acoustic.schema import FEATURES, LABEL, METADATA, SAMPLE_RATE, WAVEFORM
|
|
21
|
+
from justdata.acoustic.segment import segment_waveform
|
|
22
|
+
from justdata.acoustic.frontends.ast import ast_kaldi_fbank
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
AST_REFERENCE_COMMIT = "31088be8a3f6ef96416145c4b8d43c81f99eba7a"
|
|
26
|
+
AST_MIX_WAVEFORM = "ast_mix_waveform"
|
|
27
|
+
AST_MIX_LABEL = "ast_mix_label"
|
|
28
|
+
AST_MIX_LAMBDA = "ast_mix_lambda"
|
|
29
|
+
|
|
30
|
+
AST_RECIPE_CONFIGS = {
|
|
31
|
+
"audioset": {
|
|
32
|
+
"target_length": 1024,
|
|
33
|
+
"freqm": 48,
|
|
34
|
+
"timem": 192,
|
|
35
|
+
"mixup": 0.5,
|
|
36
|
+
"mean": -4.2677393,
|
|
37
|
+
"std": 4.5689974,
|
|
38
|
+
"noise": False,
|
|
39
|
+
"input_duration": 10.0,
|
|
40
|
+
"num_classes": 527,
|
|
41
|
+
},
|
|
42
|
+
"esc50": {
|
|
43
|
+
"target_length": 512,
|
|
44
|
+
"freqm": 24,
|
|
45
|
+
"timem": 96,
|
|
46
|
+
"mixup": 0.0,
|
|
47
|
+
"mean": -6.6268077,
|
|
48
|
+
"std": 5.358466,
|
|
49
|
+
"noise": False,
|
|
50
|
+
"input_duration": 5.0,
|
|
51
|
+
"num_classes": 50,
|
|
52
|
+
},
|
|
53
|
+
"speechcommands": {
|
|
54
|
+
"target_length": 128,
|
|
55
|
+
"freqm": 48,
|
|
56
|
+
"timem": 48,
|
|
57
|
+
"mixup": 0.6,
|
|
58
|
+
"mean": -6.845978,
|
|
59
|
+
"std": 5.5654526,
|
|
60
|
+
"noise": True,
|
|
61
|
+
"input_duration": 1.0,
|
|
62
|
+
"num_classes": 35,
|
|
63
|
+
},
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _seed_tensor(seed: tf.Tensor | int | None) -> tf.Tensor:
|
|
68
|
+
if seed is None:
|
|
69
|
+
return tf.constant([0, 0], dtype=tf.int32)
|
|
70
|
+
|
|
71
|
+
seed = tf.cast(tf.convert_to_tensor(seed), tf.int32)
|
|
72
|
+
if seed.shape.rank == 0:
|
|
73
|
+
return tf.stack([seed, tf.constant(0, dtype=tf.int32)])
|
|
74
|
+
if seed.shape.rank == 1 and seed.shape[0] == 1:
|
|
75
|
+
return tf.stack([seed[0], tf.constant(0, dtype=tf.int32)])
|
|
76
|
+
return seed[:2]
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _uniform_int(
|
|
80
|
+
seed: tf.Tensor, minval: tf.Tensor, maxval_exclusive: tf.Tensor
|
|
81
|
+
) -> tf.Tensor:
|
|
82
|
+
minval = tf.cast(minval, tf.int32)
|
|
83
|
+
maxval_exclusive = tf.cast(maxval_exclusive, tf.int32)
|
|
84
|
+
return tf.cond(
|
|
85
|
+
maxval_exclusive <= minval,
|
|
86
|
+
lambda: minval,
|
|
87
|
+
lambda: tf.random.stateless_uniform(
|
|
88
|
+
[],
|
|
89
|
+
seed=seed,
|
|
90
|
+
minval=minval,
|
|
91
|
+
maxval=maxval_exclusive,
|
|
92
|
+
dtype=tf.int32,
|
|
93
|
+
),
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _sample_beta(seed: tf.Tensor, alpha: float = 10.0) -> tf.Tensor:
|
|
98
|
+
first_seed, second_seed = tf.unstack(tf.random.split(seed, 2))
|
|
99
|
+
first = tf.random.stateless_gamma(
|
|
100
|
+
[], seed=first_seed, alpha=alpha, dtype=tf.float32
|
|
101
|
+
)
|
|
102
|
+
second = tf.random.stateless_gamma(
|
|
103
|
+
[], seed=second_seed, alpha=alpha, dtype=tf.float32
|
|
104
|
+
)
|
|
105
|
+
return first / (first + second + tf.constant(1e-6, dtype=tf.float32))
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def ast_frontend(num_mel_bins: int = 128) -> FrontendConfig:
|
|
109
|
+
return FrontendConfig(
|
|
110
|
+
name="ast_kaldi_fbank",
|
|
111
|
+
stft=STFTConfig(
|
|
112
|
+
sample_rate=16000,
|
|
113
|
+
n_fft=512,
|
|
114
|
+
win_length=400,
|
|
115
|
+
hop_length=160,
|
|
116
|
+
window="hann",
|
|
117
|
+
window_periodic=False,
|
|
118
|
+
center=False,
|
|
119
|
+
power=2.0,
|
|
120
|
+
),
|
|
121
|
+
mel=MelConfig(
|
|
122
|
+
n_mels=num_mel_bins,
|
|
123
|
+
f_min=20.0,
|
|
124
|
+
f_max=None,
|
|
125
|
+
mel_scale="htk",
|
|
126
|
+
mel_norm="none",
|
|
127
|
+
filterbank_impl="kaldi_compatible",
|
|
128
|
+
),
|
|
129
|
+
log=LogCompressionConfig(kind="log"),
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def ast_pad_or_crop_fbank(
|
|
134
|
+
fbank: tf.Tensor, target_length: int | tf.Tensor
|
|
135
|
+
) -> tf.Tensor:
|
|
136
|
+
fbank = tf.convert_to_tensor(fbank)
|
|
137
|
+
target = tf.cast(target_length, tf.int32)
|
|
138
|
+
current = tf.shape(fbank)[0]
|
|
139
|
+
cropped = fbank[:target]
|
|
140
|
+
pad = tf.maximum(target - current, 0)
|
|
141
|
+
return tf.pad(cropped, [[0, pad], [0, 0], [0, 0]])
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def ast_normalize_fbank(fbank: tf.Tensor, mean: float, std: float) -> tf.Tensor:
|
|
145
|
+
fbank = tf.cast(tf.convert_to_tensor(fbank), tf.float32)
|
|
146
|
+
return (fbank - tf.cast(mean, tf.float32)) / (tf.cast(std, tf.float32) * 2.0)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _sample_mask_span(
|
|
150
|
+
axis_size: tf.Tensor,
|
|
151
|
+
max_width: int,
|
|
152
|
+
width_seed: tf.Tensor,
|
|
153
|
+
start_seed: tf.Tensor,
|
|
154
|
+
) -> tuple[tf.Tensor, tf.Tensor]:
|
|
155
|
+
limit = tf.minimum(tf.cast(max_width, tf.int32), tf.cast(axis_size, tf.int32))
|
|
156
|
+
width = _uniform_int(width_seed, tf.constant(0, tf.int32), limit)
|
|
157
|
+
max_start = tf.maximum(tf.cast(axis_size, tf.int32) - width, 0)
|
|
158
|
+
start = _uniform_int(start_seed, tf.constant(0, tf.int32), max_start + 1)
|
|
159
|
+
return start, width
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def ast_frequency_mask(
|
|
163
|
+
fbank: tf.Tensor,
|
|
164
|
+
*,
|
|
165
|
+
max_width: int,
|
|
166
|
+
seed: tf.Tensor | int | None = None,
|
|
167
|
+
start: int | tf.Tensor | None = None,
|
|
168
|
+
width: int | tf.Tensor | None = None,
|
|
169
|
+
) -> tf.Tensor:
|
|
170
|
+
fbank = tf.convert_to_tensor(fbank)
|
|
171
|
+
if max_width <= 0:
|
|
172
|
+
return fbank
|
|
173
|
+
|
|
174
|
+
seed = _seed_tensor(seed)
|
|
175
|
+
width_seed, start_seed = tf.unstack(tf.random.split(seed, 2))
|
|
176
|
+
freq = tf.shape(fbank)[1]
|
|
177
|
+
if start is None or width is None:
|
|
178
|
+
sampled_start, sampled_width = _sample_mask_span(
|
|
179
|
+
freq,
|
|
180
|
+
max_width,
|
|
181
|
+
width_seed,
|
|
182
|
+
start_seed,
|
|
183
|
+
)
|
|
184
|
+
else:
|
|
185
|
+
sampled_start = tf.cast(start, tf.int32)
|
|
186
|
+
sampled_width = tf.cast(width, tf.int32)
|
|
187
|
+
|
|
188
|
+
positions = tf.range(freq)
|
|
189
|
+
mask = tf.logical_and(
|
|
190
|
+
positions >= sampled_start,
|
|
191
|
+
positions < sampled_start + sampled_width,
|
|
192
|
+
)
|
|
193
|
+
return tf.where(
|
|
194
|
+
mask[tf.newaxis, :, tf.newaxis], tf.zeros([], dtype=fbank.dtype), fbank
|
|
195
|
+
)
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def ast_time_mask(
|
|
199
|
+
fbank: tf.Tensor,
|
|
200
|
+
*,
|
|
201
|
+
max_width: int,
|
|
202
|
+
seed: tf.Tensor | int | None = None,
|
|
203
|
+
start: int | tf.Tensor | None = None,
|
|
204
|
+
width: int | tf.Tensor | None = None,
|
|
205
|
+
) -> tf.Tensor:
|
|
206
|
+
fbank = tf.convert_to_tensor(fbank)
|
|
207
|
+
if max_width <= 0:
|
|
208
|
+
return fbank
|
|
209
|
+
|
|
210
|
+
seed = _seed_tensor(seed)
|
|
211
|
+
width_seed, start_seed = tf.unstack(tf.random.split(seed, 2))
|
|
212
|
+
time = tf.shape(fbank)[0]
|
|
213
|
+
if start is None or width is None:
|
|
214
|
+
sampled_start, sampled_width = _sample_mask_span(
|
|
215
|
+
time,
|
|
216
|
+
max_width,
|
|
217
|
+
width_seed,
|
|
218
|
+
start_seed,
|
|
219
|
+
)
|
|
220
|
+
else:
|
|
221
|
+
sampled_start = tf.cast(start, tf.int32)
|
|
222
|
+
sampled_width = tf.cast(width, tf.int32)
|
|
223
|
+
|
|
224
|
+
positions = tf.range(time)
|
|
225
|
+
mask = tf.logical_and(
|
|
226
|
+
positions >= sampled_start,
|
|
227
|
+
positions < sampled_start + sampled_width,
|
|
228
|
+
)
|
|
229
|
+
return tf.where(
|
|
230
|
+
mask[:, tf.newaxis, tf.newaxis], tf.zeros([], dtype=fbank.dtype), fbank
|
|
231
|
+
)
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def ast_specaugment(
|
|
235
|
+
fbank: tf.Tensor,
|
|
236
|
+
*,
|
|
237
|
+
freqm: int,
|
|
238
|
+
timem: int,
|
|
239
|
+
seed: tf.Tensor | int | None = None,
|
|
240
|
+
) -> tf.Tensor:
|
|
241
|
+
seed = _seed_tensor(seed)
|
|
242
|
+
freq_seed, time_seed = tf.unstack(tf.random.split(seed, 2))
|
|
243
|
+
fbank = ast_frequency_mask(fbank, max_width=freqm, seed=freq_seed)
|
|
244
|
+
return ast_time_mask(fbank, max_width=timem, seed=time_seed)
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def ast_add_noise_and_roll(
|
|
248
|
+
fbank: tf.Tensor,
|
|
249
|
+
*,
|
|
250
|
+
seed: tf.Tensor | int | None = None,
|
|
251
|
+
noise_scale: float | tf.Tensor | None = None,
|
|
252
|
+
roll_shift: int | tf.Tensor | None = None,
|
|
253
|
+
) -> tf.Tensor:
|
|
254
|
+
seed = _seed_tensor(seed)
|
|
255
|
+
scale_seed, noise_seed, roll_seed = tf.unstack(tf.random.split(seed, 3))
|
|
256
|
+
fbank = tf.convert_to_tensor(fbank)
|
|
257
|
+
|
|
258
|
+
if noise_scale is None:
|
|
259
|
+
scale = tf.random.stateless_uniform([], seed=scale_seed, minval=0.0, maxval=0.1)
|
|
260
|
+
else:
|
|
261
|
+
scale = tf.cast(noise_scale, tf.float32)
|
|
262
|
+
noise = tf.random.stateless_uniform(
|
|
263
|
+
tf.shape(fbank), seed=noise_seed, dtype=tf.float32
|
|
264
|
+
)
|
|
265
|
+
fbank = fbank + tf.cast(noise * scale, fbank.dtype)
|
|
266
|
+
|
|
267
|
+
if roll_shift is None:
|
|
268
|
+
shift = _uniform_int(
|
|
269
|
+
roll_seed, tf.constant(-10, tf.int32), tf.constant(10, tf.int32)
|
|
270
|
+
)
|
|
271
|
+
else:
|
|
272
|
+
shift = tf.cast(roll_shift, tf.int32)
|
|
273
|
+
return tf.roll(fbank, shift=shift, axis=0)
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def _metadata_for_recipe(recipe_name: str) -> dict[str, Any]:
|
|
277
|
+
recipe = AST_RECIPE_CONFIGS[recipe_name]
|
|
278
|
+
return {
|
|
279
|
+
"preset_version": 2,
|
|
280
|
+
"model_family": "ast",
|
|
281
|
+
"recipe": recipe_name,
|
|
282
|
+
"frontend_contract": "ast-kaldi-fbank-v1",
|
|
283
|
+
"compatibility_status": "frontend_golden",
|
|
284
|
+
"reference_repo": "YuanGongND/ast",
|
|
285
|
+
"reference_commit": AST_REFERENCE_COMMIT,
|
|
286
|
+
"ast": {
|
|
287
|
+
key: recipe[key]
|
|
288
|
+
for key in (
|
|
289
|
+
"target_length",
|
|
290
|
+
"freqm",
|
|
291
|
+
"timem",
|
|
292
|
+
"mixup",
|
|
293
|
+
"mean",
|
|
294
|
+
"std",
|
|
295
|
+
"noise",
|
|
296
|
+
)
|
|
297
|
+
},
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def _preprocess() -> AudioPreprocessConfig:
|
|
302
|
+
return AudioPreprocessConfig(
|
|
303
|
+
target_sample_rate=16000,
|
|
304
|
+
channel_strategy="mono_mean",
|
|
305
|
+
normalize_waveform="none",
|
|
306
|
+
)
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
def _segment(duration: float) -> SegmentStrategyConfig:
|
|
310
|
+
return SegmentStrategyConfig(
|
|
311
|
+
clip_duration=duration,
|
|
312
|
+
train_mode="full",
|
|
313
|
+
eval_mode="full",
|
|
314
|
+
pad_mode="zero",
|
|
315
|
+
pad_position="right",
|
|
316
|
+
)
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def _recipe_preset(name: str, recipe_name: str) -> AudioPreset:
|
|
320
|
+
recipe = AST_RECIPE_CONFIGS[recipe_name]
|
|
321
|
+
return AudioPreset(
|
|
322
|
+
name=name,
|
|
323
|
+
input_duration=recipe["input_duration"],
|
|
324
|
+
target_sample_rate=16000,
|
|
325
|
+
preprocess=_preprocess(),
|
|
326
|
+
segment=_segment(recipe["input_duration"]),
|
|
327
|
+
frontend=ast_frontend(),
|
|
328
|
+
label_transform=LabelTransformConfig(
|
|
329
|
+
mode="multi_hot",
|
|
330
|
+
num_classes=recipe["num_classes"],
|
|
331
|
+
),
|
|
332
|
+
layout="btf",
|
|
333
|
+
static_shape=(recipe["target_length"], 128),
|
|
334
|
+
train_augment={"ast": _metadata_for_recipe(recipe_name)["ast"]},
|
|
335
|
+
metadata=_metadata_for_recipe(recipe_name),
|
|
336
|
+
)
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def ast_audioset_16k_10s_fbank128() -> AudioPreset:
|
|
340
|
+
return _recipe_preset("ast_audioset_16k_10s_fbank128", "audioset")
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
def ast_esc50_16k_5s_fbank128() -> AudioPreset:
|
|
344
|
+
return _recipe_preset("ast_esc50_16k_5s_fbank128", "esc50")
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
def ast_speechcommands_16k_1s_fbank128() -> AudioPreset:
|
|
348
|
+
return _recipe_preset("ast_speechcommands_16k_1s_fbank128", "speechcommands")
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def _ast_config(
|
|
352
|
+
metadata: Mapping[str, Any] | None, train_augment: Mapping[str, Any] | None
|
|
353
|
+
) -> dict:
|
|
354
|
+
ast = {}
|
|
355
|
+
if isinstance(metadata, Mapping) and isinstance(metadata.get("ast"), Mapping):
|
|
356
|
+
ast.update(metadata["ast"])
|
|
357
|
+
if isinstance(train_augment, Mapping) and isinstance(
|
|
358
|
+
train_augment.get("ast"), Mapping
|
|
359
|
+
):
|
|
360
|
+
ast.update(train_augment["ast"])
|
|
361
|
+
return ast
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
def _target_length(
|
|
365
|
+
layout: str, static_shape: tuple[int | None, ...] | None, ast: Mapping[str, Any]
|
|
366
|
+
) -> int:
|
|
367
|
+
if "target_length" in ast:
|
|
368
|
+
return int(ast["target_length"])
|
|
369
|
+
if static_shape is None:
|
|
370
|
+
raise ValueError(
|
|
371
|
+
"AST pipeline requires static_shape or metadata['ast']['target_length']"
|
|
372
|
+
)
|
|
373
|
+
if layout == "btf":
|
|
374
|
+
return int(static_shape[0])
|
|
375
|
+
if layout == "bft":
|
|
376
|
+
return int(static_shape[1])
|
|
377
|
+
if layout == "bcft":
|
|
378
|
+
return int(static_shape[2])
|
|
379
|
+
if layout == "btfc":
|
|
380
|
+
return int(static_shape[0])
|
|
381
|
+
raise ValueError(f"Unsupported AST layout: {layout!r}")
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
def _transform_sample_label(
|
|
385
|
+
result: dict, label_config: LabelTransformConfig | None
|
|
386
|
+
) -> dict:
|
|
387
|
+
if label_config is None or LABEL not in result:
|
|
388
|
+
return result
|
|
389
|
+
label, metadata = transform_label(
|
|
390
|
+
result[LABEL],
|
|
391
|
+
label_config,
|
|
392
|
+
metadata=result.get(METADATA),
|
|
393
|
+
)
|
|
394
|
+
result = dict(result)
|
|
395
|
+
result[LABEL] = label
|
|
396
|
+
result[METADATA] = metadata
|
|
397
|
+
return result
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
def _prepare_waveform_for_ast_mix(waveform: tf.Tensor) -> tf.Tensor:
|
|
401
|
+
waveform = tf.cast(tf.convert_to_tensor(waveform), tf.float32)
|
|
402
|
+
if waveform.shape.rank == 1:
|
|
403
|
+
waveform = waveform[:, tf.newaxis]
|
|
404
|
+
return waveform - tf.reduce_mean(waveform)
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
def ast_mix_waveforms(
|
|
408
|
+
waveform: tf.Tensor,
|
|
409
|
+
partner_waveform: tf.Tensor,
|
|
410
|
+
mix_lambda: tf.Tensor | float,
|
|
411
|
+
) -> tf.Tensor:
|
|
412
|
+
waveform = _prepare_waveform_for_ast_mix(waveform)
|
|
413
|
+
partner = _prepare_waveform_for_ast_mix(partner_waveform)
|
|
414
|
+
target = tf.shape(waveform)[0]
|
|
415
|
+
partner_len = tf.shape(partner)[0]
|
|
416
|
+
|
|
417
|
+
def _pad_partner() -> tf.Tensor:
|
|
418
|
+
return tf.pad(partner, [[0, target - partner_len], [0, 0]])
|
|
419
|
+
|
|
420
|
+
partner = tf.cond(partner_len < target, _pad_partner, lambda: partner[:target])
|
|
421
|
+
lam = tf.cast(mix_lambda, tf.float32)
|
|
422
|
+
mixed = lam * waveform + (1.0 - lam) * partner
|
|
423
|
+
return mixed - tf.reduce_mean(mixed)
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
def _maybe_mix_waveform_and_label(
|
|
427
|
+
result: dict,
|
|
428
|
+
*,
|
|
429
|
+
label_config: LabelTransformConfig | None,
|
|
430
|
+
seed: tf.Tensor,
|
|
431
|
+
) -> dict:
|
|
432
|
+
if AST_MIX_WAVEFORM not in result:
|
|
433
|
+
return _transform_sample_label(result, label_config)
|
|
434
|
+
|
|
435
|
+
lam = result.get(AST_MIX_LAMBDA)
|
|
436
|
+
if lam is None:
|
|
437
|
+
lam = _sample_beta(seed, alpha=10.0)
|
|
438
|
+
lam = tf.cast(lam, tf.float32)
|
|
439
|
+
|
|
440
|
+
mixed = ast_mix_waveforms(result[WAVEFORM], result[AST_MIX_WAVEFORM], lam)
|
|
441
|
+
result = dict(result)
|
|
442
|
+
result[WAVEFORM] = mixed
|
|
443
|
+
|
|
444
|
+
if label_config is not None and LABEL in result:
|
|
445
|
+
label, metadata = transform_label(
|
|
446
|
+
result[LABEL],
|
|
447
|
+
label_config,
|
|
448
|
+
metadata=result.get(METADATA),
|
|
449
|
+
)
|
|
450
|
+
if AST_MIX_LABEL in result:
|
|
451
|
+
partner_label, _partner_metadata = transform_label(
|
|
452
|
+
result[AST_MIX_LABEL],
|
|
453
|
+
label_config,
|
|
454
|
+
metadata=None,
|
|
455
|
+
)
|
|
456
|
+
label = lam * tf.cast(label, tf.float32) + (1.0 - lam) * tf.cast(
|
|
457
|
+
partner_label,
|
|
458
|
+
tf.float32,
|
|
459
|
+
)
|
|
460
|
+
result[LABEL] = label
|
|
461
|
+
result[METADATA] = metadata
|
|
462
|
+
return result
|
|
463
|
+
|
|
464
|
+
|
|
465
|
+
def _make_ast_feature_stage(
|
|
466
|
+
*,
|
|
467
|
+
frontend: FrontendConfig,
|
|
468
|
+
layout: str,
|
|
469
|
+
dtype: str,
|
|
470
|
+
output_key: str,
|
|
471
|
+
static_shape: tuple[int | None, ...] | None,
|
|
472
|
+
target_length: int,
|
|
473
|
+
norm_mean: float,
|
|
474
|
+
norm_std: float,
|
|
475
|
+
freqm: int,
|
|
476
|
+
timem: int,
|
|
477
|
+
noise: bool,
|
|
478
|
+
segment_config: SegmentStrategyConfig | None,
|
|
479
|
+
label_config: LabelTransformConfig | None,
|
|
480
|
+
is_training: bool,
|
|
481
|
+
):
|
|
482
|
+
def stage(
|
|
483
|
+
sample: dict, seed: tf.Tensor | int | None = None, num_classes=None
|
|
484
|
+
) -> dict:
|
|
485
|
+
del num_classes
|
|
486
|
+
seed = _seed_tensor(seed)
|
|
487
|
+
segment_seed, mix_seed, spec_seed, noise_seed = tf.unstack(
|
|
488
|
+
tf.random.split(seed, 4)
|
|
489
|
+
)
|
|
490
|
+
result = dict(sample)
|
|
491
|
+
|
|
492
|
+
if segment_config is not None:
|
|
493
|
+
segmented = segment_waveform(
|
|
494
|
+
result[WAVEFORM],
|
|
495
|
+
result[SAMPLE_RATE],
|
|
496
|
+
segment_config,
|
|
497
|
+
is_training=is_training,
|
|
498
|
+
seed=segment_seed,
|
|
499
|
+
)
|
|
500
|
+
metadata = dict(result.get(METADATA, {}))
|
|
501
|
+
metadata.update(segmented[METADATA])
|
|
502
|
+
result[WAVEFORM] = segmented["audio"]
|
|
503
|
+
result[METADATA] = metadata
|
|
504
|
+
|
|
505
|
+
if is_training:
|
|
506
|
+
result = _maybe_mix_waveform_and_label(
|
|
507
|
+
result,
|
|
508
|
+
label_config=label_config,
|
|
509
|
+
seed=mix_seed,
|
|
510
|
+
)
|
|
511
|
+
|
|
512
|
+
features = ast_kaldi_fbank(result[WAVEFORM], frontend)
|
|
513
|
+
features = ast_pad_or_crop_fbank(features, target_length)
|
|
514
|
+
if is_training:
|
|
515
|
+
features = ast_specaugment(
|
|
516
|
+
features, freqm=freqm, timem=timem, seed=spec_seed
|
|
517
|
+
)
|
|
518
|
+
features = ast_normalize_fbank(features, norm_mean, norm_std)
|
|
519
|
+
if is_training and noise:
|
|
520
|
+
features = ast_add_noise_and_roll(features, seed=noise_seed)
|
|
521
|
+
|
|
522
|
+
features = convert_audio_layout(features, layout, output_kind="features")
|
|
523
|
+
features = tf.cast(features, tf.as_dtype(dtype))
|
|
524
|
+
if static_shape is not None:
|
|
525
|
+
features.set_shape(static_shape)
|
|
526
|
+
|
|
527
|
+
result[output_key] = features
|
|
528
|
+
if not is_training:
|
|
529
|
+
result = _transform_sample_label(result, label_config)
|
|
530
|
+
return result
|
|
531
|
+
|
|
532
|
+
return stage
|
|
533
|
+
|
|
534
|
+
|
|
535
|
+
def ast_pipeline(
|
|
536
|
+
preproc_kwargs: dict | None = None,
|
|
537
|
+
aug_kwargs: dict | None = None,
|
|
538
|
+
laug_kwargs: dict | None = None,
|
|
539
|
+
postproc_kwargs: dict | None = None,
|
|
540
|
+
preprocess: dict | None = None,
|
|
541
|
+
segment: dict | None = None,
|
|
542
|
+
frontend: dict | FrontendConfig | None = None,
|
|
543
|
+
layout: str = "btf",
|
|
544
|
+
dtype: str = "float32",
|
|
545
|
+
output_key: str | None = None,
|
|
546
|
+
static_shape: tuple[int | None, ...] | None = None,
|
|
547
|
+
label_transform: dict | LabelTransformConfig | None = None,
|
|
548
|
+
train_augment: dict | None = None,
|
|
549
|
+
metadata: dict | None = None,
|
|
550
|
+
**kwargs,
|
|
551
|
+
):
|
|
552
|
+
del aug_kwargs, laug_kwargs, kwargs
|
|
553
|
+
preproc_kwargs = preproc_kwargs or {}
|
|
554
|
+
postproc_kwargs = postproc_kwargs or {}
|
|
555
|
+
if preprocess is not None:
|
|
556
|
+
preproc_kwargs.setdefault("config", preprocess)
|
|
557
|
+
|
|
558
|
+
is_training = postproc_kwargs.get("is_training", False)
|
|
559
|
+
frontend_config = FrontendConfig.from_dict(frontend or ast_frontend())
|
|
560
|
+
segment_config = (
|
|
561
|
+
SegmentStrategyConfig.from_dict(segment) if segment is not None else None
|
|
562
|
+
)
|
|
563
|
+
label_config = (
|
|
564
|
+
LabelTransformConfig.from_dict(label_transform)
|
|
565
|
+
if label_transform is not None
|
|
566
|
+
else None
|
|
567
|
+
)
|
|
568
|
+
ast = _ast_config(metadata, train_augment)
|
|
569
|
+
target_length = _target_length(layout, static_shape, ast)
|
|
570
|
+
output_key = output_key or FEATURES
|
|
571
|
+
|
|
572
|
+
feature_stage = _make_ast_feature_stage(
|
|
573
|
+
frontend=frontend_config,
|
|
574
|
+
layout=layout,
|
|
575
|
+
dtype=dtype,
|
|
576
|
+
output_key=output_key,
|
|
577
|
+
static_shape=static_shape,
|
|
578
|
+
target_length=target_length,
|
|
579
|
+
norm_mean=float(ast.get("mean", -4.2677393)),
|
|
580
|
+
norm_std=float(ast.get("std", 4.5689974)),
|
|
581
|
+
freqm=int(ast.get("freqm", 0)),
|
|
582
|
+
timem=int(ast.get("timem", 0)),
|
|
583
|
+
noise=bool(ast.get("noise", False)),
|
|
584
|
+
segment_config=segment_config,
|
|
585
|
+
label_config=label_config,
|
|
586
|
+
is_training=is_training,
|
|
587
|
+
)
|
|
588
|
+
|
|
589
|
+
def identity_sample(sample, *args, **_kwargs):
|
|
590
|
+
return sample
|
|
591
|
+
|
|
592
|
+
def identity_batch(batch, num_classes=None, seed=None, **_kwargs):
|
|
593
|
+
del num_classes, seed
|
|
594
|
+
return batch
|
|
595
|
+
|
|
596
|
+
def train_postprocess(sample, num_classes=None):
|
|
597
|
+
del num_classes
|
|
598
|
+
return _transform_sample_label(dict(sample), label_config)
|
|
599
|
+
|
|
600
|
+
return (
|
|
601
|
+
make_preprocessing(**preproc_kwargs),
|
|
602
|
+
feature_stage if is_training else identity_sample,
|
|
603
|
+
identity_batch,
|
|
604
|
+
train_postprocess if is_training else feature_stage,
|
|
605
|
+
)
|
|
606
|
+
|
|
607
|
+
|
|
608
|
+
__all__ = [
|
|
609
|
+
"AST_MIX_LABEL",
|
|
610
|
+
"AST_MIX_LAMBDA",
|
|
611
|
+
"AST_MIX_WAVEFORM",
|
|
612
|
+
"AST_RECIPE_CONFIGS",
|
|
613
|
+
"AST_REFERENCE_COMMIT",
|
|
614
|
+
"ast_add_noise_and_roll",
|
|
615
|
+
"ast_audioset_16k_10s_fbank128",
|
|
616
|
+
"ast_esc50_16k_5s_fbank128",
|
|
617
|
+
"ast_frequency_mask",
|
|
618
|
+
"ast_frontend",
|
|
619
|
+
"ast_mix_waveforms",
|
|
620
|
+
"ast_normalize_fbank",
|
|
621
|
+
"ast_pad_or_crop_fbank",
|
|
622
|
+
"ast_pipeline",
|
|
623
|
+
"ast_specaugment",
|
|
624
|
+
"ast_speechcommands_16k_1s_fbank128",
|
|
625
|
+
"ast_time_mask",
|
|
626
|
+
]
|