justdata 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. justdata/__init__.py +3 -0
  2. justdata/acoustic/__init__.py +381 -0
  3. justdata/acoustic/adapters.py +329 -0
  4. justdata/acoustic/augment/__init__.py +119 -0
  5. justdata/acoustic/augment/batch.py +813 -0
  6. justdata/acoustic/augment/configs.py +204 -0
  7. justdata/acoustic/augment/patchout.py +331 -0
  8. justdata/acoustic/augment/spectrogram.py +806 -0
  9. justdata/acoustic/augment/waveform.py +879 -0
  10. justdata/acoustic/batching.py +27 -0
  11. justdata/acoustic/channel.py +53 -0
  12. justdata/acoustic/compat/__init__.py +95 -0
  13. justdata/acoustic/compat/ast.py +626 -0
  14. justdata/acoustic/compat/ced.py +152 -0
  15. justdata/acoustic/compat/efficientat.py +243 -0
  16. justdata/acoustic/compat/panns.py +115 -0
  17. justdata/acoustic/compat/passt.py +191 -0
  18. justdata/acoustic/configs.py +519 -0
  19. justdata/acoustic/corruptions/__init__.py +33 -0
  20. justdata/acoustic/corruptions/codec.py +92 -0
  21. justdata/acoustic/corruptions/datasets.py +155 -0
  22. justdata/acoustic/corruptions/device.py +39 -0
  23. justdata/acoustic/corruptions/dynamics.py +55 -0
  24. justdata/acoustic/corruptions/filters.py +105 -0
  25. justdata/acoustic/corruptions/noise.py +173 -0
  26. justdata/acoustic/corruptions/registry.py +230 -0
  27. justdata/acoustic/corruptions/reverb.py +49 -0
  28. justdata/acoustic/corruptions/timefreq.py +127 -0
  29. justdata/acoustic/datasets.py +52 -0
  30. justdata/acoustic/dcase2025.py +814 -0
  31. justdata/acoustic/decoding.py +73 -0
  32. justdata/acoustic/eval_views.py +54 -0
  33. justdata/acoustic/evals.py +71 -0
  34. justdata/acoustic/filters.py +18 -0
  35. justdata/acoustic/frontends/__init__.py +37 -0
  36. justdata/acoustic/frontends/ast.py +116 -0
  37. justdata/acoustic/frontends/kaldi.py +34 -0
  38. justdata/acoustic/frontends/log.py +94 -0
  39. justdata/acoustic/frontends/mel.py +173 -0
  40. justdata/acoustic/frontends/mfcc.py +26 -0
  41. justdata/acoustic/frontends/pcen.py +26 -0
  42. justdata/acoustic/frontends/stft.py +174 -0
  43. justdata/acoustic/jax.py +23 -0
  44. justdata/acoustic/labels.py +169 -0
  45. justdata/acoustic/layouts.py +86 -0
  46. justdata/acoustic/metadata.py +152 -0
  47. justdata/acoustic/normalization.py +196 -0
  48. justdata/acoustic/pipelines.py +112 -0
  49. justdata/acoustic/postprocessing.py +328 -0
  50. justdata/acoustic/preprocessing.py +175 -0
  51. justdata/acoustic/presets.py +257 -0
  52. justdata/acoustic/registry.py +381 -0
  53. justdata/acoustic/resampling.py +218 -0
  54. justdata/acoustic/schema.py +51 -0
  55. justdata/acoustic/segment.py +369 -0
  56. justdata/acoustic/sources.py +503 -0
  57. justdata/acoustic/stages.py +59 -0
  58. justdata/acoustic/stats.py +50 -0
  59. justdata/acoustic/tasks.py +375 -0
  60. justdata/audio/__init__.py +20 -0
  61. justdata/core/__init__.py +74 -0
  62. justdata/core/adapters.py +46 -0
  63. justdata/core/augmentations.py +34 -0
  64. justdata/core/filters.py +94 -0
  65. justdata/core/finalization.py +173 -0
  66. justdata/core/label_mixing.py +129 -0
  67. justdata/core/loader.py +384 -0
  68. justdata/core/metadata.py +250 -0
  69. justdata/core/presets.py +155 -0
  70. justdata/core/registry.py +229 -0
  71. justdata/core/sources.py +87 -0
  72. justdata/core/stats.py +227 -0
  73. justdata/corruptions.py +52 -0
  74. justdata/py.typed +0 -0
  75. justdata/vision/__init__.py +17 -0
  76. justdata/vision/augmentations/__init__.py +31 -0
  77. justdata/vision/augmentations/auto.py +480 -0
  78. justdata/vision/augmentations/color.py +163 -0
  79. justdata/vision/augmentations/composed.py +156 -0
  80. justdata/vision/augmentations/geometric.py +302 -0
  81. justdata/vision/augmentations/mixing.py +232 -0
  82. justdata/vision/augmentations/registry.py +142 -0
  83. justdata/vision/corruptions/__init__.py +16 -0
  84. justdata/vision/corruptions/blur.py +65 -0
  85. justdata/vision/corruptions/digital.py +26 -0
  86. justdata/vision/corruptions/noise.py +20 -0
  87. justdata/vision/corruptions/registry.py +113 -0
  88. justdata/vision/corruptions/weather.py +100 -0
  89. justdata/vision/datasets.py +34 -0
  90. justdata/vision/encodings/__init__.py +1 -0
  91. justdata/vision/encodings/heatmaps.py +196 -0
  92. justdata/vision/minic.py +69 -0
  93. justdata/vision/pipelines.py +62 -0
  94. justdata/vision/presets.py +509 -0
  95. justdata/vision/sources.py +1080 -0
  96. justdata/vision/stages.py +271 -0
  97. justdata/vision/tasks/__init__.py +1 -0
  98. justdata/vision/tasks/classification.py +321 -0
  99. justdata/vision/tasks/depth.py +25 -0
  100. justdata/vision/tasks/segmentation.py +194 -0
  101. justdata/vision/transforms.py +113 -0
  102. justdata/vision/utils.py +741 -0
  103. justdata-1.0.0.dist-info/METADATA +860 -0
  104. justdata-1.0.0.dist-info/RECORD +106 -0
  105. justdata-1.0.0.dist-info/WHEEL +4 -0
  106. justdata-1.0.0.dist-info/licenses/LICENSE +23 -0
@@ -0,0 +1,626 @@
1
+ from __future__ import annotations
2
+
3
+ from typing import Any, Mapping
4
+
5
+ import tensorflow as tf
6
+
7
+ from justdata.acoustic.configs import (
8
+ AudioPreprocessConfig,
9
+ AudioPreset,
10
+ FrontendConfig,
11
+ LabelTransformConfig,
12
+ LogCompressionConfig,
13
+ MelConfig,
14
+ STFTConfig,
15
+ SegmentStrategyConfig,
16
+ )
17
+ from justdata.acoustic.labels import transform_label
18
+ from justdata.acoustic.layouts import convert_audio_layout
19
+ from justdata.acoustic.preprocessing import make_preprocessing
20
+ from justdata.acoustic.schema import FEATURES, LABEL, METADATA, SAMPLE_RATE, WAVEFORM
21
+ from justdata.acoustic.segment import segment_waveform
22
+ from justdata.acoustic.frontends.ast import ast_kaldi_fbank
23
+
24
+
25
+ AST_REFERENCE_COMMIT = "31088be8a3f6ef96416145c4b8d43c81f99eba7a"
26
+ AST_MIX_WAVEFORM = "ast_mix_waveform"
27
+ AST_MIX_LABEL = "ast_mix_label"
28
+ AST_MIX_LAMBDA = "ast_mix_lambda"
29
+
30
+ AST_RECIPE_CONFIGS = {
31
+ "audioset": {
32
+ "target_length": 1024,
33
+ "freqm": 48,
34
+ "timem": 192,
35
+ "mixup": 0.5,
36
+ "mean": -4.2677393,
37
+ "std": 4.5689974,
38
+ "noise": False,
39
+ "input_duration": 10.0,
40
+ "num_classes": 527,
41
+ },
42
+ "esc50": {
43
+ "target_length": 512,
44
+ "freqm": 24,
45
+ "timem": 96,
46
+ "mixup": 0.0,
47
+ "mean": -6.6268077,
48
+ "std": 5.358466,
49
+ "noise": False,
50
+ "input_duration": 5.0,
51
+ "num_classes": 50,
52
+ },
53
+ "speechcommands": {
54
+ "target_length": 128,
55
+ "freqm": 48,
56
+ "timem": 48,
57
+ "mixup": 0.6,
58
+ "mean": -6.845978,
59
+ "std": 5.5654526,
60
+ "noise": True,
61
+ "input_duration": 1.0,
62
+ "num_classes": 35,
63
+ },
64
+ }
65
+
66
+
67
+ def _seed_tensor(seed: tf.Tensor | int | None) -> tf.Tensor:
68
+ if seed is None:
69
+ return tf.constant([0, 0], dtype=tf.int32)
70
+
71
+ seed = tf.cast(tf.convert_to_tensor(seed), tf.int32)
72
+ if seed.shape.rank == 0:
73
+ return tf.stack([seed, tf.constant(0, dtype=tf.int32)])
74
+ if seed.shape.rank == 1 and seed.shape[0] == 1:
75
+ return tf.stack([seed[0], tf.constant(0, dtype=tf.int32)])
76
+ return seed[:2]
77
+
78
+
79
+ def _uniform_int(
80
+ seed: tf.Tensor, minval: tf.Tensor, maxval_exclusive: tf.Tensor
81
+ ) -> tf.Tensor:
82
+ minval = tf.cast(minval, tf.int32)
83
+ maxval_exclusive = tf.cast(maxval_exclusive, tf.int32)
84
+ return tf.cond(
85
+ maxval_exclusive <= minval,
86
+ lambda: minval,
87
+ lambda: tf.random.stateless_uniform(
88
+ [],
89
+ seed=seed,
90
+ minval=minval,
91
+ maxval=maxval_exclusive,
92
+ dtype=tf.int32,
93
+ ),
94
+ )
95
+
96
+
97
+ def _sample_beta(seed: tf.Tensor, alpha: float = 10.0) -> tf.Tensor:
98
+ first_seed, second_seed = tf.unstack(tf.random.split(seed, 2))
99
+ first = tf.random.stateless_gamma(
100
+ [], seed=first_seed, alpha=alpha, dtype=tf.float32
101
+ )
102
+ second = tf.random.stateless_gamma(
103
+ [], seed=second_seed, alpha=alpha, dtype=tf.float32
104
+ )
105
+ return first / (first + second + tf.constant(1e-6, dtype=tf.float32))
106
+
107
+
108
+ def ast_frontend(num_mel_bins: int = 128) -> FrontendConfig:
109
+ return FrontendConfig(
110
+ name="ast_kaldi_fbank",
111
+ stft=STFTConfig(
112
+ sample_rate=16000,
113
+ n_fft=512,
114
+ win_length=400,
115
+ hop_length=160,
116
+ window="hann",
117
+ window_periodic=False,
118
+ center=False,
119
+ power=2.0,
120
+ ),
121
+ mel=MelConfig(
122
+ n_mels=num_mel_bins,
123
+ f_min=20.0,
124
+ f_max=None,
125
+ mel_scale="htk",
126
+ mel_norm="none",
127
+ filterbank_impl="kaldi_compatible",
128
+ ),
129
+ log=LogCompressionConfig(kind="log"),
130
+ )
131
+
132
+
133
+ def ast_pad_or_crop_fbank(
134
+ fbank: tf.Tensor, target_length: int | tf.Tensor
135
+ ) -> tf.Tensor:
136
+ fbank = tf.convert_to_tensor(fbank)
137
+ target = tf.cast(target_length, tf.int32)
138
+ current = tf.shape(fbank)[0]
139
+ cropped = fbank[:target]
140
+ pad = tf.maximum(target - current, 0)
141
+ return tf.pad(cropped, [[0, pad], [0, 0], [0, 0]])
142
+
143
+
144
+ def ast_normalize_fbank(fbank: tf.Tensor, mean: float, std: float) -> tf.Tensor:
145
+ fbank = tf.cast(tf.convert_to_tensor(fbank), tf.float32)
146
+ return (fbank - tf.cast(mean, tf.float32)) / (tf.cast(std, tf.float32) * 2.0)
147
+
148
+
149
+ def _sample_mask_span(
150
+ axis_size: tf.Tensor,
151
+ max_width: int,
152
+ width_seed: tf.Tensor,
153
+ start_seed: tf.Tensor,
154
+ ) -> tuple[tf.Tensor, tf.Tensor]:
155
+ limit = tf.minimum(tf.cast(max_width, tf.int32), tf.cast(axis_size, tf.int32))
156
+ width = _uniform_int(width_seed, tf.constant(0, tf.int32), limit)
157
+ max_start = tf.maximum(tf.cast(axis_size, tf.int32) - width, 0)
158
+ start = _uniform_int(start_seed, tf.constant(0, tf.int32), max_start + 1)
159
+ return start, width
160
+
161
+
162
+ def ast_frequency_mask(
163
+ fbank: tf.Tensor,
164
+ *,
165
+ max_width: int,
166
+ seed: tf.Tensor | int | None = None,
167
+ start: int | tf.Tensor | None = None,
168
+ width: int | tf.Tensor | None = None,
169
+ ) -> tf.Tensor:
170
+ fbank = tf.convert_to_tensor(fbank)
171
+ if max_width <= 0:
172
+ return fbank
173
+
174
+ seed = _seed_tensor(seed)
175
+ width_seed, start_seed = tf.unstack(tf.random.split(seed, 2))
176
+ freq = tf.shape(fbank)[1]
177
+ if start is None or width is None:
178
+ sampled_start, sampled_width = _sample_mask_span(
179
+ freq,
180
+ max_width,
181
+ width_seed,
182
+ start_seed,
183
+ )
184
+ else:
185
+ sampled_start = tf.cast(start, tf.int32)
186
+ sampled_width = tf.cast(width, tf.int32)
187
+
188
+ positions = tf.range(freq)
189
+ mask = tf.logical_and(
190
+ positions >= sampled_start,
191
+ positions < sampled_start + sampled_width,
192
+ )
193
+ return tf.where(
194
+ mask[tf.newaxis, :, tf.newaxis], tf.zeros([], dtype=fbank.dtype), fbank
195
+ )
196
+
197
+
198
+ def ast_time_mask(
199
+ fbank: tf.Tensor,
200
+ *,
201
+ max_width: int,
202
+ seed: tf.Tensor | int | None = None,
203
+ start: int | tf.Tensor | None = None,
204
+ width: int | tf.Tensor | None = None,
205
+ ) -> tf.Tensor:
206
+ fbank = tf.convert_to_tensor(fbank)
207
+ if max_width <= 0:
208
+ return fbank
209
+
210
+ seed = _seed_tensor(seed)
211
+ width_seed, start_seed = tf.unstack(tf.random.split(seed, 2))
212
+ time = tf.shape(fbank)[0]
213
+ if start is None or width is None:
214
+ sampled_start, sampled_width = _sample_mask_span(
215
+ time,
216
+ max_width,
217
+ width_seed,
218
+ start_seed,
219
+ )
220
+ else:
221
+ sampled_start = tf.cast(start, tf.int32)
222
+ sampled_width = tf.cast(width, tf.int32)
223
+
224
+ positions = tf.range(time)
225
+ mask = tf.logical_and(
226
+ positions >= sampled_start,
227
+ positions < sampled_start + sampled_width,
228
+ )
229
+ return tf.where(
230
+ mask[:, tf.newaxis, tf.newaxis], tf.zeros([], dtype=fbank.dtype), fbank
231
+ )
232
+
233
+
234
+ def ast_specaugment(
235
+ fbank: tf.Tensor,
236
+ *,
237
+ freqm: int,
238
+ timem: int,
239
+ seed: tf.Tensor | int | None = None,
240
+ ) -> tf.Tensor:
241
+ seed = _seed_tensor(seed)
242
+ freq_seed, time_seed = tf.unstack(tf.random.split(seed, 2))
243
+ fbank = ast_frequency_mask(fbank, max_width=freqm, seed=freq_seed)
244
+ return ast_time_mask(fbank, max_width=timem, seed=time_seed)
245
+
246
+
247
+ def ast_add_noise_and_roll(
248
+ fbank: tf.Tensor,
249
+ *,
250
+ seed: tf.Tensor | int | None = None,
251
+ noise_scale: float | tf.Tensor | None = None,
252
+ roll_shift: int | tf.Tensor | None = None,
253
+ ) -> tf.Tensor:
254
+ seed = _seed_tensor(seed)
255
+ scale_seed, noise_seed, roll_seed = tf.unstack(tf.random.split(seed, 3))
256
+ fbank = tf.convert_to_tensor(fbank)
257
+
258
+ if noise_scale is None:
259
+ scale = tf.random.stateless_uniform([], seed=scale_seed, minval=0.0, maxval=0.1)
260
+ else:
261
+ scale = tf.cast(noise_scale, tf.float32)
262
+ noise = tf.random.stateless_uniform(
263
+ tf.shape(fbank), seed=noise_seed, dtype=tf.float32
264
+ )
265
+ fbank = fbank + tf.cast(noise * scale, fbank.dtype)
266
+
267
+ if roll_shift is None:
268
+ shift = _uniform_int(
269
+ roll_seed, tf.constant(-10, tf.int32), tf.constant(10, tf.int32)
270
+ )
271
+ else:
272
+ shift = tf.cast(roll_shift, tf.int32)
273
+ return tf.roll(fbank, shift=shift, axis=0)
274
+
275
+
276
+ def _metadata_for_recipe(recipe_name: str) -> dict[str, Any]:
277
+ recipe = AST_RECIPE_CONFIGS[recipe_name]
278
+ return {
279
+ "preset_version": 2,
280
+ "model_family": "ast",
281
+ "recipe": recipe_name,
282
+ "frontend_contract": "ast-kaldi-fbank-v1",
283
+ "compatibility_status": "frontend_golden",
284
+ "reference_repo": "YuanGongND/ast",
285
+ "reference_commit": AST_REFERENCE_COMMIT,
286
+ "ast": {
287
+ key: recipe[key]
288
+ for key in (
289
+ "target_length",
290
+ "freqm",
291
+ "timem",
292
+ "mixup",
293
+ "mean",
294
+ "std",
295
+ "noise",
296
+ )
297
+ },
298
+ }
299
+
300
+
301
+ def _preprocess() -> AudioPreprocessConfig:
302
+ return AudioPreprocessConfig(
303
+ target_sample_rate=16000,
304
+ channel_strategy="mono_mean",
305
+ normalize_waveform="none",
306
+ )
307
+
308
+
309
+ def _segment(duration: float) -> SegmentStrategyConfig:
310
+ return SegmentStrategyConfig(
311
+ clip_duration=duration,
312
+ train_mode="full",
313
+ eval_mode="full",
314
+ pad_mode="zero",
315
+ pad_position="right",
316
+ )
317
+
318
+
319
+ def _recipe_preset(name: str, recipe_name: str) -> AudioPreset:
320
+ recipe = AST_RECIPE_CONFIGS[recipe_name]
321
+ return AudioPreset(
322
+ name=name,
323
+ input_duration=recipe["input_duration"],
324
+ target_sample_rate=16000,
325
+ preprocess=_preprocess(),
326
+ segment=_segment(recipe["input_duration"]),
327
+ frontend=ast_frontend(),
328
+ label_transform=LabelTransformConfig(
329
+ mode="multi_hot",
330
+ num_classes=recipe["num_classes"],
331
+ ),
332
+ layout="btf",
333
+ static_shape=(recipe["target_length"], 128),
334
+ train_augment={"ast": _metadata_for_recipe(recipe_name)["ast"]},
335
+ metadata=_metadata_for_recipe(recipe_name),
336
+ )
337
+
338
+
339
+ def ast_audioset_16k_10s_fbank128() -> AudioPreset:
340
+ return _recipe_preset("ast_audioset_16k_10s_fbank128", "audioset")
341
+
342
+
343
+ def ast_esc50_16k_5s_fbank128() -> AudioPreset:
344
+ return _recipe_preset("ast_esc50_16k_5s_fbank128", "esc50")
345
+
346
+
347
+ def ast_speechcommands_16k_1s_fbank128() -> AudioPreset:
348
+ return _recipe_preset("ast_speechcommands_16k_1s_fbank128", "speechcommands")
349
+
350
+
351
+ def _ast_config(
352
+ metadata: Mapping[str, Any] | None, train_augment: Mapping[str, Any] | None
353
+ ) -> dict:
354
+ ast = {}
355
+ if isinstance(metadata, Mapping) and isinstance(metadata.get("ast"), Mapping):
356
+ ast.update(metadata["ast"])
357
+ if isinstance(train_augment, Mapping) and isinstance(
358
+ train_augment.get("ast"), Mapping
359
+ ):
360
+ ast.update(train_augment["ast"])
361
+ return ast
362
+
363
+
364
+ def _target_length(
365
+ layout: str, static_shape: tuple[int | None, ...] | None, ast: Mapping[str, Any]
366
+ ) -> int:
367
+ if "target_length" in ast:
368
+ return int(ast["target_length"])
369
+ if static_shape is None:
370
+ raise ValueError(
371
+ "AST pipeline requires static_shape or metadata['ast']['target_length']"
372
+ )
373
+ if layout == "btf":
374
+ return int(static_shape[0])
375
+ if layout == "bft":
376
+ return int(static_shape[1])
377
+ if layout == "bcft":
378
+ return int(static_shape[2])
379
+ if layout == "btfc":
380
+ return int(static_shape[0])
381
+ raise ValueError(f"Unsupported AST layout: {layout!r}")
382
+
383
+
384
+ def _transform_sample_label(
385
+ result: dict, label_config: LabelTransformConfig | None
386
+ ) -> dict:
387
+ if label_config is None or LABEL not in result:
388
+ return result
389
+ label, metadata = transform_label(
390
+ result[LABEL],
391
+ label_config,
392
+ metadata=result.get(METADATA),
393
+ )
394
+ result = dict(result)
395
+ result[LABEL] = label
396
+ result[METADATA] = metadata
397
+ return result
398
+
399
+
400
+ def _prepare_waveform_for_ast_mix(waveform: tf.Tensor) -> tf.Tensor:
401
+ waveform = tf.cast(tf.convert_to_tensor(waveform), tf.float32)
402
+ if waveform.shape.rank == 1:
403
+ waveform = waveform[:, tf.newaxis]
404
+ return waveform - tf.reduce_mean(waveform)
405
+
406
+
407
+ def ast_mix_waveforms(
408
+ waveform: tf.Tensor,
409
+ partner_waveform: tf.Tensor,
410
+ mix_lambda: tf.Tensor | float,
411
+ ) -> tf.Tensor:
412
+ waveform = _prepare_waveform_for_ast_mix(waveform)
413
+ partner = _prepare_waveform_for_ast_mix(partner_waveform)
414
+ target = tf.shape(waveform)[0]
415
+ partner_len = tf.shape(partner)[0]
416
+
417
+ def _pad_partner() -> tf.Tensor:
418
+ return tf.pad(partner, [[0, target - partner_len], [0, 0]])
419
+
420
+ partner = tf.cond(partner_len < target, _pad_partner, lambda: partner[:target])
421
+ lam = tf.cast(mix_lambda, tf.float32)
422
+ mixed = lam * waveform + (1.0 - lam) * partner
423
+ return mixed - tf.reduce_mean(mixed)
424
+
425
+
426
+ def _maybe_mix_waveform_and_label(
427
+ result: dict,
428
+ *,
429
+ label_config: LabelTransformConfig | None,
430
+ seed: tf.Tensor,
431
+ ) -> dict:
432
+ if AST_MIX_WAVEFORM not in result:
433
+ return _transform_sample_label(result, label_config)
434
+
435
+ lam = result.get(AST_MIX_LAMBDA)
436
+ if lam is None:
437
+ lam = _sample_beta(seed, alpha=10.0)
438
+ lam = tf.cast(lam, tf.float32)
439
+
440
+ mixed = ast_mix_waveforms(result[WAVEFORM], result[AST_MIX_WAVEFORM], lam)
441
+ result = dict(result)
442
+ result[WAVEFORM] = mixed
443
+
444
+ if label_config is not None and LABEL in result:
445
+ label, metadata = transform_label(
446
+ result[LABEL],
447
+ label_config,
448
+ metadata=result.get(METADATA),
449
+ )
450
+ if AST_MIX_LABEL in result:
451
+ partner_label, _partner_metadata = transform_label(
452
+ result[AST_MIX_LABEL],
453
+ label_config,
454
+ metadata=None,
455
+ )
456
+ label = lam * tf.cast(label, tf.float32) + (1.0 - lam) * tf.cast(
457
+ partner_label,
458
+ tf.float32,
459
+ )
460
+ result[LABEL] = label
461
+ result[METADATA] = metadata
462
+ return result
463
+
464
+
465
+ def _make_ast_feature_stage(
466
+ *,
467
+ frontend: FrontendConfig,
468
+ layout: str,
469
+ dtype: str,
470
+ output_key: str,
471
+ static_shape: tuple[int | None, ...] | None,
472
+ target_length: int,
473
+ norm_mean: float,
474
+ norm_std: float,
475
+ freqm: int,
476
+ timem: int,
477
+ noise: bool,
478
+ segment_config: SegmentStrategyConfig | None,
479
+ label_config: LabelTransformConfig | None,
480
+ is_training: bool,
481
+ ):
482
+ def stage(
483
+ sample: dict, seed: tf.Tensor | int | None = None, num_classes=None
484
+ ) -> dict:
485
+ del num_classes
486
+ seed = _seed_tensor(seed)
487
+ segment_seed, mix_seed, spec_seed, noise_seed = tf.unstack(
488
+ tf.random.split(seed, 4)
489
+ )
490
+ result = dict(sample)
491
+
492
+ if segment_config is not None:
493
+ segmented = segment_waveform(
494
+ result[WAVEFORM],
495
+ result[SAMPLE_RATE],
496
+ segment_config,
497
+ is_training=is_training,
498
+ seed=segment_seed,
499
+ )
500
+ metadata = dict(result.get(METADATA, {}))
501
+ metadata.update(segmented[METADATA])
502
+ result[WAVEFORM] = segmented["audio"]
503
+ result[METADATA] = metadata
504
+
505
+ if is_training:
506
+ result = _maybe_mix_waveform_and_label(
507
+ result,
508
+ label_config=label_config,
509
+ seed=mix_seed,
510
+ )
511
+
512
+ features = ast_kaldi_fbank(result[WAVEFORM], frontend)
513
+ features = ast_pad_or_crop_fbank(features, target_length)
514
+ if is_training:
515
+ features = ast_specaugment(
516
+ features, freqm=freqm, timem=timem, seed=spec_seed
517
+ )
518
+ features = ast_normalize_fbank(features, norm_mean, norm_std)
519
+ if is_training and noise:
520
+ features = ast_add_noise_and_roll(features, seed=noise_seed)
521
+
522
+ features = convert_audio_layout(features, layout, output_kind="features")
523
+ features = tf.cast(features, tf.as_dtype(dtype))
524
+ if static_shape is not None:
525
+ features.set_shape(static_shape)
526
+
527
+ result[output_key] = features
528
+ if not is_training:
529
+ result = _transform_sample_label(result, label_config)
530
+ return result
531
+
532
+ return stage
533
+
534
+
535
+ def ast_pipeline(
536
+ preproc_kwargs: dict | None = None,
537
+ aug_kwargs: dict | None = None,
538
+ laug_kwargs: dict | None = None,
539
+ postproc_kwargs: dict | None = None,
540
+ preprocess: dict | None = None,
541
+ segment: dict | None = None,
542
+ frontend: dict | FrontendConfig | None = None,
543
+ layout: str = "btf",
544
+ dtype: str = "float32",
545
+ output_key: str | None = None,
546
+ static_shape: tuple[int | None, ...] | None = None,
547
+ label_transform: dict | LabelTransformConfig | None = None,
548
+ train_augment: dict | None = None,
549
+ metadata: dict | None = None,
550
+ **kwargs,
551
+ ):
552
+ del aug_kwargs, laug_kwargs, kwargs
553
+ preproc_kwargs = preproc_kwargs or {}
554
+ postproc_kwargs = postproc_kwargs or {}
555
+ if preprocess is not None:
556
+ preproc_kwargs.setdefault("config", preprocess)
557
+
558
+ is_training = postproc_kwargs.get("is_training", False)
559
+ frontend_config = FrontendConfig.from_dict(frontend or ast_frontend())
560
+ segment_config = (
561
+ SegmentStrategyConfig.from_dict(segment) if segment is not None else None
562
+ )
563
+ label_config = (
564
+ LabelTransformConfig.from_dict(label_transform)
565
+ if label_transform is not None
566
+ else None
567
+ )
568
+ ast = _ast_config(metadata, train_augment)
569
+ target_length = _target_length(layout, static_shape, ast)
570
+ output_key = output_key or FEATURES
571
+
572
+ feature_stage = _make_ast_feature_stage(
573
+ frontend=frontend_config,
574
+ layout=layout,
575
+ dtype=dtype,
576
+ output_key=output_key,
577
+ static_shape=static_shape,
578
+ target_length=target_length,
579
+ norm_mean=float(ast.get("mean", -4.2677393)),
580
+ norm_std=float(ast.get("std", 4.5689974)),
581
+ freqm=int(ast.get("freqm", 0)),
582
+ timem=int(ast.get("timem", 0)),
583
+ noise=bool(ast.get("noise", False)),
584
+ segment_config=segment_config,
585
+ label_config=label_config,
586
+ is_training=is_training,
587
+ )
588
+
589
+ def identity_sample(sample, *args, **_kwargs):
590
+ return sample
591
+
592
+ def identity_batch(batch, num_classes=None, seed=None, **_kwargs):
593
+ del num_classes, seed
594
+ return batch
595
+
596
+ def train_postprocess(sample, num_classes=None):
597
+ del num_classes
598
+ return _transform_sample_label(dict(sample), label_config)
599
+
600
+ return (
601
+ make_preprocessing(**preproc_kwargs),
602
+ feature_stage if is_training else identity_sample,
603
+ identity_batch,
604
+ train_postprocess if is_training else feature_stage,
605
+ )
606
+
607
+
608
+ __all__ = [
609
+ "AST_MIX_LABEL",
610
+ "AST_MIX_LAMBDA",
611
+ "AST_MIX_WAVEFORM",
612
+ "AST_RECIPE_CONFIGS",
613
+ "AST_REFERENCE_COMMIT",
614
+ "ast_add_noise_and_roll",
615
+ "ast_audioset_16k_10s_fbank128",
616
+ "ast_esc50_16k_5s_fbank128",
617
+ "ast_frequency_mask",
618
+ "ast_frontend",
619
+ "ast_mix_waveforms",
620
+ "ast_normalize_fbank",
621
+ "ast_pad_or_crop_fbank",
622
+ "ast_pipeline",
623
+ "ast_specaugment",
624
+ "ast_speechcommands_16k_1s_fbank128",
625
+ "ast_time_mask",
626
+ ]