justdata 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- justdata/__init__.py +3 -0
- justdata/acoustic/__init__.py +381 -0
- justdata/acoustic/adapters.py +329 -0
- justdata/acoustic/augment/__init__.py +119 -0
- justdata/acoustic/augment/batch.py +813 -0
- justdata/acoustic/augment/configs.py +204 -0
- justdata/acoustic/augment/patchout.py +331 -0
- justdata/acoustic/augment/spectrogram.py +806 -0
- justdata/acoustic/augment/waveform.py +879 -0
- justdata/acoustic/batching.py +27 -0
- justdata/acoustic/channel.py +53 -0
- justdata/acoustic/compat/__init__.py +95 -0
- justdata/acoustic/compat/ast.py +626 -0
- justdata/acoustic/compat/ced.py +152 -0
- justdata/acoustic/compat/efficientat.py +243 -0
- justdata/acoustic/compat/panns.py +115 -0
- justdata/acoustic/compat/passt.py +191 -0
- justdata/acoustic/configs.py +519 -0
- justdata/acoustic/corruptions/__init__.py +33 -0
- justdata/acoustic/corruptions/codec.py +92 -0
- justdata/acoustic/corruptions/datasets.py +155 -0
- justdata/acoustic/corruptions/device.py +39 -0
- justdata/acoustic/corruptions/dynamics.py +55 -0
- justdata/acoustic/corruptions/filters.py +105 -0
- justdata/acoustic/corruptions/noise.py +173 -0
- justdata/acoustic/corruptions/registry.py +230 -0
- justdata/acoustic/corruptions/reverb.py +49 -0
- justdata/acoustic/corruptions/timefreq.py +127 -0
- justdata/acoustic/datasets.py +52 -0
- justdata/acoustic/dcase2025.py +814 -0
- justdata/acoustic/decoding.py +73 -0
- justdata/acoustic/eval_views.py +54 -0
- justdata/acoustic/evals.py +71 -0
- justdata/acoustic/filters.py +18 -0
- justdata/acoustic/frontends/__init__.py +37 -0
- justdata/acoustic/frontends/ast.py +116 -0
- justdata/acoustic/frontends/kaldi.py +34 -0
- justdata/acoustic/frontends/log.py +94 -0
- justdata/acoustic/frontends/mel.py +173 -0
- justdata/acoustic/frontends/mfcc.py +26 -0
- justdata/acoustic/frontends/pcen.py +26 -0
- justdata/acoustic/frontends/stft.py +174 -0
- justdata/acoustic/jax.py +23 -0
- justdata/acoustic/labels.py +169 -0
- justdata/acoustic/layouts.py +86 -0
- justdata/acoustic/metadata.py +152 -0
- justdata/acoustic/normalization.py +196 -0
- justdata/acoustic/pipelines.py +112 -0
- justdata/acoustic/postprocessing.py +328 -0
- justdata/acoustic/preprocessing.py +175 -0
- justdata/acoustic/presets.py +257 -0
- justdata/acoustic/registry.py +381 -0
- justdata/acoustic/resampling.py +218 -0
- justdata/acoustic/schema.py +51 -0
- justdata/acoustic/segment.py +369 -0
- justdata/acoustic/sources.py +503 -0
- justdata/acoustic/stages.py +59 -0
- justdata/acoustic/stats.py +50 -0
- justdata/acoustic/tasks.py +375 -0
- justdata/audio/__init__.py +20 -0
- justdata/core/__init__.py +74 -0
- justdata/core/adapters.py +46 -0
- justdata/core/augmentations.py +34 -0
- justdata/core/filters.py +94 -0
- justdata/core/finalization.py +173 -0
- justdata/core/label_mixing.py +129 -0
- justdata/core/loader.py +384 -0
- justdata/core/metadata.py +250 -0
- justdata/core/presets.py +155 -0
- justdata/core/registry.py +229 -0
- justdata/core/sources.py +87 -0
- justdata/core/stats.py +227 -0
- justdata/corruptions.py +52 -0
- justdata/py.typed +0 -0
- justdata/vision/__init__.py +17 -0
- justdata/vision/augmentations/__init__.py +31 -0
- justdata/vision/augmentations/auto.py +480 -0
- justdata/vision/augmentations/color.py +163 -0
- justdata/vision/augmentations/composed.py +156 -0
- justdata/vision/augmentations/geometric.py +302 -0
- justdata/vision/augmentations/mixing.py +232 -0
- justdata/vision/augmentations/registry.py +142 -0
- justdata/vision/corruptions/__init__.py +16 -0
- justdata/vision/corruptions/blur.py +65 -0
- justdata/vision/corruptions/digital.py +26 -0
- justdata/vision/corruptions/noise.py +20 -0
- justdata/vision/corruptions/registry.py +113 -0
- justdata/vision/corruptions/weather.py +100 -0
- justdata/vision/datasets.py +34 -0
- justdata/vision/encodings/__init__.py +1 -0
- justdata/vision/encodings/heatmaps.py +196 -0
- justdata/vision/minic.py +69 -0
- justdata/vision/pipelines.py +62 -0
- justdata/vision/presets.py +509 -0
- justdata/vision/sources.py +1080 -0
- justdata/vision/stages.py +271 -0
- justdata/vision/tasks/__init__.py +1 -0
- justdata/vision/tasks/classification.py +321 -0
- justdata/vision/tasks/depth.py +25 -0
- justdata/vision/tasks/segmentation.py +194 -0
- justdata/vision/transforms.py +113 -0
- justdata/vision/utils.py +741 -0
- justdata-1.0.0.dist-info/METADATA +860 -0
- justdata-1.0.0.dist-info/RECORD +106 -0
- justdata-1.0.0.dist-info/WHEEL +4 -0
- justdata-1.0.0.dist-info/licenses/LICENSE +23 -0
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections.abc import Callable
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
import tensorflow as tf
|
|
7
|
+
|
|
8
|
+
from justdata.acoustic.configs import FeatureNormConfig
|
|
9
|
+
from justdata.acoustic.registry import register_audio_normalization
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
_AXES_BY_RANK = {
|
|
13
|
+
2: {"time": 0, "channel": 1},
|
|
14
|
+
3: {"time": 0, "frequency": 1, "channel": 2},
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _rank(x: tf.Tensor) -> int:
|
|
19
|
+
rank = x.shape.rank
|
|
20
|
+
if rank not in _AXES_BY_RANK:
|
|
21
|
+
raise ValueError("Feature normalization expects rank-2 or rank-3 tensors")
|
|
22
|
+
return rank
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _axes_from_names(x: tf.Tensor, names: tuple[str, ...]) -> list[int]:
|
|
26
|
+
rank = _rank(x)
|
|
27
|
+
if not names:
|
|
28
|
+
return list(range(rank))
|
|
29
|
+
axes: list[int] = []
|
|
30
|
+
for name in names:
|
|
31
|
+
if name not in _AXES_BY_RANK[rank]:
|
|
32
|
+
raise ValueError(
|
|
33
|
+
f"Unknown normalization axis {name!r} for rank-{rank} tensor"
|
|
34
|
+
)
|
|
35
|
+
axes.append(_AXES_BY_RANK[rank][name])
|
|
36
|
+
return axes
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _as_float_tensor(value: Any, field_name: str) -> tf.Tensor:
|
|
40
|
+
if isinstance(value, str):
|
|
41
|
+
raise ValueError(
|
|
42
|
+
f"{field_name} path references are not loaded by the built-in normalizer"
|
|
43
|
+
)
|
|
44
|
+
return tf.cast(tf.convert_to_tensor(value), tf.float32)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _broadcast_vector(
|
|
48
|
+
x: tf.Tensor, values: tuple[float, ...], field_name: str
|
|
49
|
+
) -> tf.Tensor:
|
|
50
|
+
rank = _rank(x)
|
|
51
|
+
tensor = _as_float_tensor(values, field_name)
|
|
52
|
+
length = len(values)
|
|
53
|
+
|
|
54
|
+
if length == 1:
|
|
55
|
+
return tf.reshape(tensor, [1] * rank)
|
|
56
|
+
|
|
57
|
+
static_shape = x.shape.as_list()
|
|
58
|
+
if rank == 2:
|
|
59
|
+
channels = static_shape[1]
|
|
60
|
+
if channels is not None and length != channels:
|
|
61
|
+
raise ValueError(
|
|
62
|
+
f"{field_name} length {length} does not match channel dimension {channels}"
|
|
63
|
+
)
|
|
64
|
+
return tf.reshape(tensor, [1, length])
|
|
65
|
+
|
|
66
|
+
frequency = static_shape[1]
|
|
67
|
+
channels = static_shape[2]
|
|
68
|
+
if frequency is not None and length == frequency:
|
|
69
|
+
return tf.reshape(tensor, [1, length, 1])
|
|
70
|
+
if channels is not None and length == channels:
|
|
71
|
+
return tf.reshape(tensor, [1, 1, length])
|
|
72
|
+
|
|
73
|
+
expected = [dim for dim in (frequency, channels) if dim is not None]
|
|
74
|
+
expected_text = " or ".join(str(dim) for dim in expected) or "frequency or channel"
|
|
75
|
+
raise ValueError(f"{field_name} length {length} must match {expected_text}")
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _broadcast_stat(
|
|
79
|
+
x: tf.Tensor,
|
|
80
|
+
value: tuple[float, ...] | float | str | None,
|
|
81
|
+
field_name: str,
|
|
82
|
+
default: float,
|
|
83
|
+
) -> tf.Tensor:
|
|
84
|
+
if value is None:
|
|
85
|
+
return tf.cast(default, tf.float32)
|
|
86
|
+
if isinstance(value, (int, float)):
|
|
87
|
+
return tf.cast(value, tf.float32)
|
|
88
|
+
if isinstance(value, tuple):
|
|
89
|
+
return _broadcast_vector(x, value, field_name)
|
|
90
|
+
return _as_float_tensor(value, field_name)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _mean_std(x: tf.Tensor, axes: list[int], eps: float) -> tf.Tensor:
|
|
94
|
+
mean, variance = tf.nn.moments(x, axes=axes, keepdims=True)
|
|
95
|
+
return (x - mean) / tf.sqrt(variance + tf.cast(eps, x.dtype))
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
@register_audio_normalization("none")
|
|
99
|
+
def normalize_none(
|
|
100
|
+
x: tf.Tensor, config: FeatureNormConfig | dict | None = None
|
|
101
|
+
) -> tf.Tensor:
|
|
102
|
+
return tf.convert_to_tensor(x)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
@register_audio_normalization("dataset_mean_std")
|
|
106
|
+
def normalize_dataset_mean_std(
|
|
107
|
+
x: tf.Tensor, config: FeatureNormConfig | dict | None = None
|
|
108
|
+
) -> tf.Tensor:
|
|
109
|
+
config = FeatureNormConfig.from_dict(config or {})
|
|
110
|
+
x = tf.cast(tf.convert_to_tensor(x), tf.float32)
|
|
111
|
+
mean = _broadcast_stat(x, config.mean, "mean", 0.0)
|
|
112
|
+
std = _broadcast_stat(x, config.std, "std", 1.0)
|
|
113
|
+
return (x - tf.cast(mean, x.dtype)) / (
|
|
114
|
+
tf.cast(std, x.dtype) + tf.cast(config.eps, x.dtype)
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
@register_audio_normalization("checkpoint_mean_std")
|
|
119
|
+
def normalize_checkpoint_mean_std(
|
|
120
|
+
x: tf.Tensor, config: FeatureNormConfig | dict | None = None
|
|
121
|
+
) -> tf.Tensor:
|
|
122
|
+
return normalize_dataset_mean_std(x, config)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
@register_audio_normalization("per_clip_mean_std")
|
|
126
|
+
def normalize_per_clip_mean_std(
|
|
127
|
+
x: tf.Tensor, config: FeatureNormConfig | dict | None = None
|
|
128
|
+
) -> tf.Tensor:
|
|
129
|
+
config = FeatureNormConfig.from_dict(config or {})
|
|
130
|
+
x = tf.cast(tf.convert_to_tensor(x), tf.float32)
|
|
131
|
+
return _mean_std(x, _axes_from_names(x, config.axes), config.eps)
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
@register_audio_normalization("per_frequency_mean_std")
|
|
135
|
+
def normalize_per_frequency_mean_std(
|
|
136
|
+
x: tf.Tensor, config: FeatureNormConfig | dict | None = None
|
|
137
|
+
) -> tf.Tensor:
|
|
138
|
+
config = FeatureNormConfig.from_dict(config or {})
|
|
139
|
+
x = tf.cast(tf.convert_to_tensor(x), tf.float32)
|
|
140
|
+
return _mean_std(x, [0], config.eps)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
@register_audio_normalization("kaldi_cmvn")
|
|
144
|
+
def normalize_kaldi_cmvn(
|
|
145
|
+
x: tf.Tensor, config: FeatureNormConfig | dict | None = None
|
|
146
|
+
) -> tf.Tensor:
|
|
147
|
+
config = FeatureNormConfig.from_dict(config or {})
|
|
148
|
+
x = tf.cast(tf.convert_to_tensor(x), tf.float32)
|
|
149
|
+
axes = _axes_from_names(x, config.axes or ("time",))
|
|
150
|
+
return _mean_std(x, axes, config.eps)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
@register_audio_normalization("affine")
|
|
154
|
+
def normalize_affine(
|
|
155
|
+
x: tf.Tensor, config: FeatureNormConfig | dict | None = None
|
|
156
|
+
) -> tf.Tensor:
|
|
157
|
+
config = FeatureNormConfig.from_dict(config or {})
|
|
158
|
+
x = tf.cast(tf.convert_to_tensor(x), tf.float32)
|
|
159
|
+
scale = _broadcast_stat(x, config.scale, "scale", 1.0)
|
|
160
|
+
bias = _broadcast_stat(x, config.bias, "bias", 0.0)
|
|
161
|
+
return x * tf.cast(scale, x.dtype) + tf.cast(bias, x.dtype)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
_NORMALIZERS: dict[
|
|
165
|
+
str, Callable[[tf.Tensor, FeatureNormConfig | dict | None], tf.Tensor]
|
|
166
|
+
] = {
|
|
167
|
+
"none": normalize_none,
|
|
168
|
+
"dataset_mean_std": normalize_dataset_mean_std,
|
|
169
|
+
"checkpoint_mean_std": normalize_checkpoint_mean_std,
|
|
170
|
+
"per_clip_mean_std": normalize_per_clip_mean_std,
|
|
171
|
+
"per_frequency_mean_std": normalize_per_frequency_mean_std,
|
|
172
|
+
"kaldi_cmvn": normalize_kaldi_cmvn,
|
|
173
|
+
"affine": normalize_affine,
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def apply_feature_normalization(
|
|
178
|
+
x: tf.Tensor,
|
|
179
|
+
config: FeatureNormConfig | dict | None,
|
|
180
|
+
) -> tf.Tensor:
|
|
181
|
+
config = FeatureNormConfig.from_dict(config or {})
|
|
182
|
+
if config.kind not in _NORMALIZERS:
|
|
183
|
+
raise ValueError(f"Unknown feature normalization kind: {config.kind!r}")
|
|
184
|
+
return _NORMALIZERS[config.kind](x, config)
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
__all__ = [
|
|
188
|
+
"apply_feature_normalization",
|
|
189
|
+
"normalize_affine",
|
|
190
|
+
"normalize_checkpoint_mean_std",
|
|
191
|
+
"normalize_dataset_mean_std",
|
|
192
|
+
"normalize_kaldi_cmvn",
|
|
193
|
+
"normalize_none",
|
|
194
|
+
"normalize_per_clip_mean_std",
|
|
195
|
+
"normalize_per_frequency_mean_std",
|
|
196
|
+
]
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
from justdata.core.registry import PipelineFuncs, register_pipeline
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def default_pipeline(
|
|
5
|
+
preproc_kwargs: dict = None,
|
|
6
|
+
aug_kwargs: dict = None,
|
|
7
|
+
laug_kwargs: dict = None,
|
|
8
|
+
postproc_kwargs: dict = None,
|
|
9
|
+
preprocess: dict = None,
|
|
10
|
+
segment: dict = None,
|
|
11
|
+
frontend: dict = None,
|
|
12
|
+
layout: str = None,
|
|
13
|
+
dtype: str = "float32",
|
|
14
|
+
output_key: str = None,
|
|
15
|
+
static_shape: tuple[int | None, ...] = None,
|
|
16
|
+
input_duration: float = None,
|
|
17
|
+
target_sample_rate: int = None,
|
|
18
|
+
label_transform: dict = None,
|
|
19
|
+
train_augment: dict = None,
|
|
20
|
+
waveform_augmentations=None,
|
|
21
|
+
spectrogram_augmentations=None,
|
|
22
|
+
batch_augmentations=None,
|
|
23
|
+
spectrogram_layout: str = None,
|
|
24
|
+
augment_eval: bool = False,
|
|
25
|
+
**kwargs,
|
|
26
|
+
) -> PipelineFuncs:
|
|
27
|
+
preproc_kwargs = preproc_kwargs or {}
|
|
28
|
+
aug_kwargs = aug_kwargs or {}
|
|
29
|
+
laug_kwargs = laug_kwargs or {}
|
|
30
|
+
postproc_kwargs = postproc_kwargs or {}
|
|
31
|
+
|
|
32
|
+
if preprocess is not None:
|
|
33
|
+
preproc_kwargs.setdefault("config", preprocess)
|
|
34
|
+
|
|
35
|
+
is_training = postproc_kwargs.get("is_training", False)
|
|
36
|
+
aug_kwargs.setdefault("is_training", is_training)
|
|
37
|
+
aug_kwargs.setdefault("augment_eval", augment_eval)
|
|
38
|
+
laug_kwargs.setdefault("is_training", is_training)
|
|
39
|
+
laug_kwargs.setdefault("augment_eval", augment_eval)
|
|
40
|
+
laug_kwargs.setdefault("model_layout", layout)
|
|
41
|
+
|
|
42
|
+
if train_augment is not None and (is_training or augment_eval):
|
|
43
|
+
aug_kwargs.setdefault("train_augment", train_augment)
|
|
44
|
+
laug_kwargs.setdefault("train_augment", train_augment)
|
|
45
|
+
if waveform_augmentations is not None:
|
|
46
|
+
aug_kwargs.setdefault("waveform_augmentations", waveform_augmentations)
|
|
47
|
+
if spectrogram_augmentations is not None:
|
|
48
|
+
laug_kwargs.setdefault("spectrogram_augmentations", spectrogram_augmentations)
|
|
49
|
+
if batch_augmentations is not None:
|
|
50
|
+
laug_kwargs.setdefault("batch_augmentations", batch_augmentations)
|
|
51
|
+
if (
|
|
52
|
+
train_augment is not None
|
|
53
|
+
and (is_training or augment_eval)
|
|
54
|
+
and "batch" in train_augment
|
|
55
|
+
):
|
|
56
|
+
laug_kwargs.setdefault("batch_augmentations", train_augment["batch"])
|
|
57
|
+
if spectrogram_layout is not None:
|
|
58
|
+
laug_kwargs.setdefault("spectrogram_layout", spectrogram_layout)
|
|
59
|
+
|
|
60
|
+
if segment is not None:
|
|
61
|
+
if is_training or augment_eval:
|
|
62
|
+
aug_kwargs.setdefault("segment_config", segment)
|
|
63
|
+
else:
|
|
64
|
+
postproc_kwargs.setdefault("segment_config", segment)
|
|
65
|
+
|
|
66
|
+
if frontend is not None:
|
|
67
|
+
postproc_kwargs.setdefault("frontend", frontend)
|
|
68
|
+
postproc_kwargs.setdefault("layout", layout)
|
|
69
|
+
postproc_kwargs.setdefault("dtype", dtype)
|
|
70
|
+
postproc_kwargs.setdefault("output_key", output_key)
|
|
71
|
+
postproc_kwargs.setdefault("static_shape", static_shape)
|
|
72
|
+
postproc_kwargs.setdefault("input_duration", input_duration)
|
|
73
|
+
postproc_kwargs.setdefault("target_sample_rate", target_sample_rate)
|
|
74
|
+
postproc_kwargs.setdefault("preprocess", preprocess)
|
|
75
|
+
|
|
76
|
+
if label_transform is not None:
|
|
77
|
+
postproc_kwargs.setdefault("label_transform", label_transform)
|
|
78
|
+
laug_kwargs.setdefault("label_transform", label_transform)
|
|
79
|
+
|
|
80
|
+
from justdata.acoustic.tasks import (
|
|
81
|
+
make_augmentations,
|
|
82
|
+
make_late_augmentations,
|
|
83
|
+
make_postprocessing,
|
|
84
|
+
make_preprocessing,
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
return (
|
|
88
|
+
make_preprocessing(**preproc_kwargs),
|
|
89
|
+
make_augmentations(**aug_kwargs),
|
|
90
|
+
make_late_augmentations(**laug_kwargs),
|
|
91
|
+
make_postprocessing(**postproc_kwargs),
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
register_pipeline("acoustic/default")(default_pipeline)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
@register_pipeline("acoustic/classification")
|
|
99
|
+
def classification_pipeline(**kwargs) -> PipelineFuncs:
|
|
100
|
+
return default_pipeline(**kwargs)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
@register_pipeline("acoustic/ast_classification")
|
|
104
|
+
def ast_classification_pipeline(**kwargs) -> PipelineFuncs:
|
|
105
|
+
from justdata.acoustic.compat.ast import ast_pipeline
|
|
106
|
+
|
|
107
|
+
return ast_pipeline(**kwargs)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
@register_pipeline("acoustic/identity")
|
|
111
|
+
def identity_pipeline(**kwargs) -> PipelineFuncs:
|
|
112
|
+
return default_pipeline(**kwargs)
|
|
@@ -0,0 +1,328 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections.abc import Mapping
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
import tensorflow as tf
|
|
7
|
+
|
|
8
|
+
from justdata.acoustic.configs import AudioPreset, FrontendConfig
|
|
9
|
+
from justdata.acoustic.frontends.stft import compute_num_frames
|
|
10
|
+
from justdata.acoustic.layouts import (
|
|
11
|
+
AudioLayout,
|
|
12
|
+
OutputKind,
|
|
13
|
+
convert_audio_layout,
|
|
14
|
+
unbatched_shape_for_layout,
|
|
15
|
+
)
|
|
16
|
+
from justdata.acoustic.registry import get_audio_frontend
|
|
17
|
+
from justdata.acoustic.schema import FEATURES, WAVEFORM
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def frontend_output_kind(frontend: FrontendConfig | dict) -> OutputKind:
|
|
21
|
+
config = FrontendConfig.from_dict(frontend)
|
|
22
|
+
return "waveform" if config.name == "raw_waveform" else "features"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def default_output_key(frontend: FrontendConfig | dict) -> str:
|
|
26
|
+
return WAVEFORM if frontend_output_kind(frontend) == "waveform" else FEATURES
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _get(config: Any, key: str, default: Any = None) -> Any:
|
|
30
|
+
if isinstance(config, Mapping):
|
|
31
|
+
return config.get(key, default)
|
|
32
|
+
return getattr(config, key, default)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _frontend_from_preset(preset: AudioPreset | Mapping[str, Any]) -> FrontendConfig:
|
|
36
|
+
return FrontendConfig.from_dict(_get(preset, "frontend"))
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _layout_from_preset(preset: AudioPreset | Mapping[str, Any]) -> AudioLayout:
|
|
40
|
+
return _get(preset, "layout")
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _output_key_from_preset(preset: AudioPreset | Mapping[str, Any]) -> str:
|
|
44
|
+
frontend = _frontend_from_preset(preset)
|
|
45
|
+
return _get(preset, "output_key") or default_output_key(frontend)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _duration_from_preset(preset: AudioPreset | Mapping[str, Any]) -> float | None:
|
|
49
|
+
duration = _get(preset, "input_duration")
|
|
50
|
+
if duration is not None:
|
|
51
|
+
return float(duration)
|
|
52
|
+
segment = _get(preset, "segment")
|
|
53
|
+
if segment is not None:
|
|
54
|
+
return float(_get(segment, "clip_duration"))
|
|
55
|
+
return None
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _sample_rate_from_preset(
|
|
59
|
+
preset: AudioPreset | Mapping[str, Any],
|
|
60
|
+
frontend: FrontendConfig,
|
|
61
|
+
) -> int | None:
|
|
62
|
+
sample_rate = _get(preset, "target_sample_rate")
|
|
63
|
+
if sample_rate is not None:
|
|
64
|
+
return int(sample_rate)
|
|
65
|
+
if frontend.stft is not None:
|
|
66
|
+
return frontend.stft.sample_rate
|
|
67
|
+
preprocess = _get(preset, "preprocess")
|
|
68
|
+
if preprocess is not None:
|
|
69
|
+
return int(_get(preprocess, "target_sample_rate"))
|
|
70
|
+
return None
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _channel_count_from_preset(preset: AudioPreset | Mapping[str, Any]) -> int | None:
|
|
74
|
+
preprocess = _get(preset, "preprocess")
|
|
75
|
+
if preprocess is None:
|
|
76
|
+
return 1
|
|
77
|
+
return None if _get(preprocess, "channel_strategy") == "keep" else 1
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _frequency_bins(frontend: FrontendConfig) -> int | None:
|
|
81
|
+
if frontend.name == "raw_waveform":
|
|
82
|
+
return None
|
|
83
|
+
if frontend.name == "stft_magnitude":
|
|
84
|
+
if frontend.stft is None:
|
|
85
|
+
return None
|
|
86
|
+
return (
|
|
87
|
+
frontend.stft.n_fft // 2 + 1
|
|
88
|
+
if frontend.stft.onesided
|
|
89
|
+
else frontend.stft.n_fft
|
|
90
|
+
)
|
|
91
|
+
if frontend.name == "mfcc":
|
|
92
|
+
return frontend.n_mfcc
|
|
93
|
+
if frontend.mel is not None:
|
|
94
|
+
return frontend.mel.n_mels
|
|
95
|
+
return None
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _fixed_eval_view_count(
|
|
99
|
+
preset: AudioPreset | Mapping[str, Any],
|
|
100
|
+
) -> int | None:
|
|
101
|
+
segment = _get(preset, "segment")
|
|
102
|
+
if segment is None:
|
|
103
|
+
return None
|
|
104
|
+
duration_policy = _get(segment, "duration_policy", "none")
|
|
105
|
+
if duration_policy == "sliding_windows":
|
|
106
|
+
raise ValueError(
|
|
107
|
+
"sliding segmentation has a data-dependent number of views and "
|
|
108
|
+
"cannot be assigned a fixed shape or batched"
|
|
109
|
+
)
|
|
110
|
+
if duration_policy != "none":
|
|
111
|
+
return None
|
|
112
|
+
eval_mode = _get(segment, "eval_mode")
|
|
113
|
+
if eval_mode == "sliding":
|
|
114
|
+
raise ValueError(
|
|
115
|
+
"sliding segmentation has a data-dependent number of views and "
|
|
116
|
+
"cannot be assigned a fixed shape or batched"
|
|
117
|
+
)
|
|
118
|
+
if eval_mode == "multi_crop":
|
|
119
|
+
return int(_get(segment, "num_views"))
|
|
120
|
+
return None
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def expected_audio_static_shape(
|
|
124
|
+
preset: AudioPreset | Mapping[str, Any],
|
|
125
|
+
) -> tuple[int | None, ...] | None:
|
|
126
|
+
view_count = _fixed_eval_view_count(preset)
|
|
127
|
+
explicit_shape = _get(preset, "static_shape")
|
|
128
|
+
if explicit_shape is not None:
|
|
129
|
+
shape = tuple(explicit_shape)
|
|
130
|
+
return (view_count,) + shape if view_count is not None else shape
|
|
131
|
+
|
|
132
|
+
frontend = _frontend_from_preset(preset)
|
|
133
|
+
layout = _layout_from_preset(preset)
|
|
134
|
+
output_kind = frontend_output_kind(frontend)
|
|
135
|
+
duration = _duration_from_preset(preset)
|
|
136
|
+
sample_rate = _sample_rate_from_preset(preset, frontend)
|
|
137
|
+
channels = _channel_count_from_preset(preset)
|
|
138
|
+
|
|
139
|
+
if duration is None or sample_rate is None:
|
|
140
|
+
return None
|
|
141
|
+
|
|
142
|
+
num_samples = round(duration * sample_rate)
|
|
143
|
+
if output_kind == "waveform":
|
|
144
|
+
time = num_samples
|
|
145
|
+
frequency = None
|
|
146
|
+
else:
|
|
147
|
+
if frontend.stft is None:
|
|
148
|
+
return None
|
|
149
|
+
time = compute_num_frames(
|
|
150
|
+
num_samples,
|
|
151
|
+
n_fft=frontend.stft.n_fft,
|
|
152
|
+
win_length=frontend.stft.win_length,
|
|
153
|
+
hop_length=frontend.stft.hop_length,
|
|
154
|
+
center=frontend.stft.center,
|
|
155
|
+
)
|
|
156
|
+
frequency = _frequency_bins(frontend)
|
|
157
|
+
|
|
158
|
+
shape = unbatched_shape_for_layout(
|
|
159
|
+
layout,
|
|
160
|
+
output_kind=output_kind,
|
|
161
|
+
time=time,
|
|
162
|
+
frequency=frequency,
|
|
163
|
+
channels=channels,
|
|
164
|
+
)
|
|
165
|
+
return (view_count,) + shape if view_count is not None else shape
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def set_static_audio_shape(
|
|
169
|
+
sample: dict,
|
|
170
|
+
preset: AudioPreset | Mapping[str, Any],
|
|
171
|
+
) -> dict:
|
|
172
|
+
output_key = _output_key_from_preset(preset)
|
|
173
|
+
if output_key not in sample:
|
|
174
|
+
return sample
|
|
175
|
+
|
|
176
|
+
expected_shape = expected_audio_static_shape(preset)
|
|
177
|
+
if expected_shape is None:
|
|
178
|
+
return sample
|
|
179
|
+
|
|
180
|
+
result = dict(sample)
|
|
181
|
+
tensor = tf.convert_to_tensor(result[output_key])
|
|
182
|
+
tensor.set_shape(expected_shape)
|
|
183
|
+
result[output_key] = tensor
|
|
184
|
+
return result
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def cast_audio_dtype(x: tf.Tensor, dtype: str) -> tf.Tensor:
|
|
188
|
+
return tf.cast(x, tf.as_dtype(dtype))
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _pad_or_crop_axis(x: tf.Tensor, target: tf.Tensor | int, axis: int) -> tf.Tensor:
|
|
192
|
+
x = tf.convert_to_tensor(x)
|
|
193
|
+
target = tf.cast(target, tf.int32)
|
|
194
|
+
rank = x.shape.rank
|
|
195
|
+
if rank is None:
|
|
196
|
+
raise ValueError("Padding requires a tensor with known rank")
|
|
197
|
+
if axis < 0:
|
|
198
|
+
axis += rank
|
|
199
|
+
|
|
200
|
+
current = tf.shape(x)[axis]
|
|
201
|
+
begin = tf.zeros([rank], dtype=tf.int32)
|
|
202
|
+
size = tf.shape(x)
|
|
203
|
+
size = tf.tensor_scatter_nd_update(size, [[axis]], [tf.minimum(current, target)])
|
|
204
|
+
cropped = tf.slice(x, begin, size)
|
|
205
|
+
|
|
206
|
+
pad_after = tf.maximum(target - current, 0)
|
|
207
|
+
paddings = tf.zeros([rank, 2], dtype=tf.int32)
|
|
208
|
+
paddings = tf.tensor_scatter_nd_update(paddings, [[axis, 1]], [pad_after])
|
|
209
|
+
return tf.pad(cropped, paddings)
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def pad_or_crop_time(
|
|
213
|
+
x: tf.Tensor, target_frames: int | tf.Tensor, *, time_axis: int = 0
|
|
214
|
+
) -> tf.Tensor:
|
|
215
|
+
return _pad_or_crop_axis(x, target_frames, time_axis)
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def pad_time_to_multiple(
|
|
219
|
+
x: tf.Tensor, multiple: int, *, time_axis: int = 0
|
|
220
|
+
) -> tf.Tensor:
|
|
221
|
+
if multiple <= 0:
|
|
222
|
+
raise ValueError("multiple must be positive")
|
|
223
|
+
current = tf.shape(x)[time_axis]
|
|
224
|
+
target = tf.cast(
|
|
225
|
+
tf.math.ceil(tf.cast(current, tf.float32) / float(multiple)) * float(multiple),
|
|
226
|
+
tf.int32,
|
|
227
|
+
)
|
|
228
|
+
return pad_or_crop_time(x, target, time_axis=time_axis)
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def pad_to_patch_multiple(
|
|
232
|
+
x: tf.Tensor,
|
|
233
|
+
*,
|
|
234
|
+
time_multiple: int = 1,
|
|
235
|
+
frequency_multiple: int = 1,
|
|
236
|
+
time_axis: int = 0,
|
|
237
|
+
frequency_axis: int = 1,
|
|
238
|
+
) -> tf.Tensor:
|
|
239
|
+
if time_multiple <= 0 or frequency_multiple <= 0:
|
|
240
|
+
raise ValueError("patch multiples must be positive")
|
|
241
|
+
x = pad_time_to_multiple(x, time_multiple, time_axis=time_axis)
|
|
242
|
+
return pad_time_to_multiple(x, frequency_multiple, time_axis=frequency_axis)
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def make_model_input_stage(
|
|
246
|
+
frontend: FrontendConfig | dict,
|
|
247
|
+
*,
|
|
248
|
+
layout: AudioLayout,
|
|
249
|
+
dtype: str = "float32",
|
|
250
|
+
output_key: str | None = None,
|
|
251
|
+
preset: AudioPreset | Mapping[str, Any] | None = None,
|
|
252
|
+
) -> Any:
|
|
253
|
+
import justdata.acoustic.frontends # noqa: F401
|
|
254
|
+
|
|
255
|
+
frontend_config = FrontendConfig.from_dict(frontend)
|
|
256
|
+
frontend_fn = get_audio_frontend(frontend_config.name)
|
|
257
|
+
output_kind = frontend_output_kind(frontend_config)
|
|
258
|
+
resolved_output_key = output_key or default_output_key(frontend_config)
|
|
259
|
+
|
|
260
|
+
if preset is not None:
|
|
261
|
+
_fixed_eval_view_count(preset)
|
|
262
|
+
|
|
263
|
+
def process_waveform(waveform: tf.Tensor) -> tf.Tensor:
|
|
264
|
+
features = frontend_fn(waveform, frontend_config)
|
|
265
|
+
features = convert_audio_layout(features, layout, output_kind=output_kind)
|
|
266
|
+
return cast_audio_dtype(features, dtype)
|
|
267
|
+
|
|
268
|
+
def stage(sample: dict, num_classes: int | None = None) -> dict:
|
|
269
|
+
waveform = tf.convert_to_tensor(sample[WAVEFORM])
|
|
270
|
+
if waveform.shape.rank == 2:
|
|
271
|
+
features = process_waveform(waveform)
|
|
272
|
+
elif waveform.shape.rank == 3:
|
|
273
|
+
features = tf.map_fn(
|
|
274
|
+
process_waveform,
|
|
275
|
+
waveform,
|
|
276
|
+
fn_output_signature=tf.TensorSpec(shape=None, dtype=tf.as_dtype(dtype)),
|
|
277
|
+
)
|
|
278
|
+
else:
|
|
279
|
+
raise ValueError(
|
|
280
|
+
"Model input stage expects waveform shape [T, C] or [V, T, C]"
|
|
281
|
+
)
|
|
282
|
+
|
|
283
|
+
result = dict(sample)
|
|
284
|
+
result[resolved_output_key] = features
|
|
285
|
+
if preset is not None:
|
|
286
|
+
result = set_static_audio_shape(result, preset)
|
|
287
|
+
return result
|
|
288
|
+
|
|
289
|
+
return stage
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def preset_info(
|
|
293
|
+
*,
|
|
294
|
+
frontend: FrontendConfig | dict,
|
|
295
|
+
layout: AudioLayout,
|
|
296
|
+
dtype: str,
|
|
297
|
+
output_key: str | None = None,
|
|
298
|
+
static_shape: tuple[int | None, ...] | None = None,
|
|
299
|
+
input_duration: float | None = None,
|
|
300
|
+
target_sample_rate: int | None = None,
|
|
301
|
+
preprocess: Any = None,
|
|
302
|
+
segment: Any = None,
|
|
303
|
+
) -> dict[str, Any]:
|
|
304
|
+
return {
|
|
305
|
+
"frontend": FrontendConfig.from_dict(frontend).to_dict(),
|
|
306
|
+
"layout": layout,
|
|
307
|
+
"dtype": dtype,
|
|
308
|
+
"output_key": output_key,
|
|
309
|
+
"static_shape": static_shape,
|
|
310
|
+
"input_duration": input_duration,
|
|
311
|
+
"target_sample_rate": target_sample_rate,
|
|
312
|
+
"preprocess": preprocess,
|
|
313
|
+
"segment": segment,
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
__all__ = [
|
|
318
|
+
"cast_audio_dtype",
|
|
319
|
+
"default_output_key",
|
|
320
|
+
"expected_audio_static_shape",
|
|
321
|
+
"frontend_output_kind",
|
|
322
|
+
"make_model_input_stage",
|
|
323
|
+
"pad_or_crop_time",
|
|
324
|
+
"pad_time_to_multiple",
|
|
325
|
+
"pad_to_patch_multiple",
|
|
326
|
+
"preset_info",
|
|
327
|
+
"set_static_audio_shape",
|
|
328
|
+
]
|