justdata 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- justdata/__init__.py +3 -0
- justdata/acoustic/__init__.py +381 -0
- justdata/acoustic/adapters.py +329 -0
- justdata/acoustic/augment/__init__.py +119 -0
- justdata/acoustic/augment/batch.py +813 -0
- justdata/acoustic/augment/configs.py +204 -0
- justdata/acoustic/augment/patchout.py +331 -0
- justdata/acoustic/augment/spectrogram.py +806 -0
- justdata/acoustic/augment/waveform.py +879 -0
- justdata/acoustic/batching.py +27 -0
- justdata/acoustic/channel.py +53 -0
- justdata/acoustic/compat/__init__.py +95 -0
- justdata/acoustic/compat/ast.py +626 -0
- justdata/acoustic/compat/ced.py +152 -0
- justdata/acoustic/compat/efficientat.py +243 -0
- justdata/acoustic/compat/panns.py +115 -0
- justdata/acoustic/compat/passt.py +191 -0
- justdata/acoustic/configs.py +519 -0
- justdata/acoustic/corruptions/__init__.py +33 -0
- justdata/acoustic/corruptions/codec.py +92 -0
- justdata/acoustic/corruptions/datasets.py +155 -0
- justdata/acoustic/corruptions/device.py +39 -0
- justdata/acoustic/corruptions/dynamics.py +55 -0
- justdata/acoustic/corruptions/filters.py +105 -0
- justdata/acoustic/corruptions/noise.py +173 -0
- justdata/acoustic/corruptions/registry.py +230 -0
- justdata/acoustic/corruptions/reverb.py +49 -0
- justdata/acoustic/corruptions/timefreq.py +127 -0
- justdata/acoustic/datasets.py +52 -0
- justdata/acoustic/dcase2025.py +814 -0
- justdata/acoustic/decoding.py +73 -0
- justdata/acoustic/eval_views.py +54 -0
- justdata/acoustic/evals.py +71 -0
- justdata/acoustic/filters.py +18 -0
- justdata/acoustic/frontends/__init__.py +37 -0
- justdata/acoustic/frontends/ast.py +116 -0
- justdata/acoustic/frontends/kaldi.py +34 -0
- justdata/acoustic/frontends/log.py +94 -0
- justdata/acoustic/frontends/mel.py +173 -0
- justdata/acoustic/frontends/mfcc.py +26 -0
- justdata/acoustic/frontends/pcen.py +26 -0
- justdata/acoustic/frontends/stft.py +174 -0
- justdata/acoustic/jax.py +23 -0
- justdata/acoustic/labels.py +169 -0
- justdata/acoustic/layouts.py +86 -0
- justdata/acoustic/metadata.py +152 -0
- justdata/acoustic/normalization.py +196 -0
- justdata/acoustic/pipelines.py +112 -0
- justdata/acoustic/postprocessing.py +328 -0
- justdata/acoustic/preprocessing.py +175 -0
- justdata/acoustic/presets.py +257 -0
- justdata/acoustic/registry.py +381 -0
- justdata/acoustic/resampling.py +218 -0
- justdata/acoustic/schema.py +51 -0
- justdata/acoustic/segment.py +369 -0
- justdata/acoustic/sources.py +503 -0
- justdata/acoustic/stages.py +59 -0
- justdata/acoustic/stats.py +50 -0
- justdata/acoustic/tasks.py +375 -0
- justdata/audio/__init__.py +20 -0
- justdata/core/__init__.py +74 -0
- justdata/core/adapters.py +46 -0
- justdata/core/augmentations.py +34 -0
- justdata/core/filters.py +94 -0
- justdata/core/finalization.py +173 -0
- justdata/core/label_mixing.py +129 -0
- justdata/core/loader.py +384 -0
- justdata/core/metadata.py +250 -0
- justdata/core/presets.py +155 -0
- justdata/core/registry.py +229 -0
- justdata/core/sources.py +87 -0
- justdata/core/stats.py +227 -0
- justdata/corruptions.py +52 -0
- justdata/py.typed +0 -0
- justdata/vision/__init__.py +17 -0
- justdata/vision/augmentations/__init__.py +31 -0
- justdata/vision/augmentations/auto.py +480 -0
- justdata/vision/augmentations/color.py +163 -0
- justdata/vision/augmentations/composed.py +156 -0
- justdata/vision/augmentations/geometric.py +302 -0
- justdata/vision/augmentations/mixing.py +232 -0
- justdata/vision/augmentations/registry.py +142 -0
- justdata/vision/corruptions/__init__.py +16 -0
- justdata/vision/corruptions/blur.py +65 -0
- justdata/vision/corruptions/digital.py +26 -0
- justdata/vision/corruptions/noise.py +20 -0
- justdata/vision/corruptions/registry.py +113 -0
- justdata/vision/corruptions/weather.py +100 -0
- justdata/vision/datasets.py +34 -0
- justdata/vision/encodings/__init__.py +1 -0
- justdata/vision/encodings/heatmaps.py +196 -0
- justdata/vision/minic.py +69 -0
- justdata/vision/pipelines.py +62 -0
- justdata/vision/presets.py +509 -0
- justdata/vision/sources.py +1080 -0
- justdata/vision/stages.py +271 -0
- justdata/vision/tasks/__init__.py +1 -0
- justdata/vision/tasks/classification.py +321 -0
- justdata/vision/tasks/depth.py +25 -0
- justdata/vision/tasks/segmentation.py +194 -0
- justdata/vision/transforms.py +113 -0
- justdata/vision/utils.py +741 -0
- justdata-1.0.0.dist-info/METADATA +860 -0
- justdata-1.0.0.dist-info/RECORD +106 -0
- justdata-1.0.0.dist-info/WHEEL +4 -0
- justdata-1.0.0.dist-info/licenses/LICENSE +23 -0
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
from typing import Union
|
|
5
|
+
|
|
6
|
+
import tensorflow as tf
|
|
7
|
+
|
|
8
|
+
from justdata.acoustic.registry import register_audio_decoder
|
|
9
|
+
from justdata.acoustic.schema import SAMPLE_RATE, WAVEFORM
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
PathLike = Union[str, bytes, os.PathLike, tf.Tensor]
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def decode_wav_bytes(
|
|
16
|
+
contents: tf.Tensor,
|
|
17
|
+
*,
|
|
18
|
+
desired_channels: int | None = None,
|
|
19
|
+
) -> tuple[tf.Tensor, tf.Tensor]:
|
|
20
|
+
"""Decode WAV bytes to float32 audio with shape [T, C]."""
|
|
21
|
+
channels = -1 if desired_channels is None else desired_channels
|
|
22
|
+
waveform, sample_rate = tf.audio.decode_wav(contents, desired_channels=channels)
|
|
23
|
+
return tf.cast(waveform, tf.float32), tf.cast(sample_rate, tf.int32)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def decode_wav_file(
|
|
27
|
+
path: PathLike,
|
|
28
|
+
*,
|
|
29
|
+
desired_channels: int | None = None,
|
|
30
|
+
) -> tuple[tf.Tensor, tf.Tensor]:
|
|
31
|
+
if isinstance(path, os.PathLike):
|
|
32
|
+
path = os.fspath(path)
|
|
33
|
+
contents = tf.io.read_file(path)
|
|
34
|
+
return decode_wav_bytes(contents, desired_channels=desired_channels)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def decode_audio_file(
|
|
38
|
+
path: PathLike,
|
|
39
|
+
*,
|
|
40
|
+
format_hint: str | None = None,
|
|
41
|
+
desired_channels: int | None = None,
|
|
42
|
+
) -> tuple[tf.Tensor, tf.Tensor]:
|
|
43
|
+
if format_hint is not None and format_hint.lower() not in {"wav", ".wav"}:
|
|
44
|
+
raise ValueError(
|
|
45
|
+
"Only WAV decoding is available in the TensorFlow decoder path. "
|
|
46
|
+
"Install an optional backend and decode before adaptation for other formats."
|
|
47
|
+
)
|
|
48
|
+
return decode_wav_file(path, desired_channels=desired_channels)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@register_audio_decoder("wav")
|
|
52
|
+
def decode_wav_sample(
|
|
53
|
+
sample: dict,
|
|
54
|
+
*,
|
|
55
|
+
path_key: str = "path",
|
|
56
|
+
desired_channels: int | None = None,
|
|
57
|
+
) -> dict:
|
|
58
|
+
waveform, sample_rate = decode_wav_file(
|
|
59
|
+
sample[path_key],
|
|
60
|
+
desired_channels=desired_channels,
|
|
61
|
+
)
|
|
62
|
+
result = dict(sample)
|
|
63
|
+
result[WAVEFORM] = waveform
|
|
64
|
+
result[SAMPLE_RATE] = sample_rate
|
|
65
|
+
return result
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
__all__ = [
|
|
69
|
+
"decode_audio_file",
|
|
70
|
+
"decode_wav_bytes",
|
|
71
|
+
"decode_wav_file",
|
|
72
|
+
"decode_wav_sample",
|
|
73
|
+
]
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections.abc import Callable
|
|
4
|
+
|
|
5
|
+
import tensorflow as tf
|
|
6
|
+
|
|
7
|
+
from justdata.acoustic.configs import SegmentStrategyConfig
|
|
8
|
+
from justdata.acoustic.registry import register_audio_eval_view_strategy
|
|
9
|
+
from justdata.acoustic.segment import segment_waveform
|
|
10
|
+
from justdata.acoustic.schema import METADATA, SAMPLE_RATE, WAVEFORM
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@register_audio_eval_view_strategy("center_crop")
|
|
14
|
+
@register_audio_eval_view_strategy("full")
|
|
15
|
+
@register_audio_eval_view_strategy("sliding")
|
|
16
|
+
@register_audio_eval_view_strategy("multi_crop")
|
|
17
|
+
def generate_eval_views(
|
|
18
|
+
audio: tf.Tensor,
|
|
19
|
+
sample_rate: tf.Tensor | int,
|
|
20
|
+
config: SegmentStrategyConfig,
|
|
21
|
+
) -> dict:
|
|
22
|
+
return segment_waveform(
|
|
23
|
+
audio,
|
|
24
|
+
sample_rate,
|
|
25
|
+
config,
|
|
26
|
+
is_training=False,
|
|
27
|
+
seed=None,
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def make_eval_views(
|
|
32
|
+
config: SegmentStrategyConfig,
|
|
33
|
+
*,
|
|
34
|
+
audio_key: str = WAVEFORM,
|
|
35
|
+
) -> Callable[[dict], dict]:
|
|
36
|
+
config = SegmentStrategyConfig.from_dict(config)
|
|
37
|
+
|
|
38
|
+
def eval_views(sample: dict) -> dict:
|
|
39
|
+
views = generate_eval_views(sample[audio_key], sample[SAMPLE_RATE], config)
|
|
40
|
+
metadata = dict(sample.get(METADATA, {}))
|
|
41
|
+
metadata.update(views[METADATA])
|
|
42
|
+
|
|
43
|
+
result = dict(sample)
|
|
44
|
+
result[audio_key] = views["audio"]
|
|
45
|
+
result[METADATA] = metadata
|
|
46
|
+
return result
|
|
47
|
+
|
|
48
|
+
return eval_views
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
__all__ = [
|
|
52
|
+
"generate_eval_views",
|
|
53
|
+
"make_eval_views",
|
|
54
|
+
]
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections.abc import Sequence
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
import numpy as np
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def _labels(values: Any) -> np.ndarray:
|
|
10
|
+
values = np.asarray(values)
|
|
11
|
+
if values.ndim > 1:
|
|
12
|
+
return np.argmax(values, axis=-1)
|
|
13
|
+
return values.astype(np.int64)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _predictions(values: Any) -> np.ndarray:
|
|
17
|
+
values = np.asarray(values)
|
|
18
|
+
if values.ndim > 1:
|
|
19
|
+
return np.argmax(values, axis=-1)
|
|
20
|
+
return values.astype(np.int64)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def macro_classwise_accuracy(y_true, y_pred, num_classes: int = 10) -> float:
|
|
24
|
+
true = _labels(y_true)
|
|
25
|
+
pred = _predictions(y_pred)
|
|
26
|
+
|
|
27
|
+
class_acc = []
|
|
28
|
+
for class_id in range(num_classes):
|
|
29
|
+
mask = true == class_id
|
|
30
|
+
if np.any(mask):
|
|
31
|
+
class_acc.append(float(np.mean(pred[mask] == class_id)))
|
|
32
|
+
if not class_acc:
|
|
33
|
+
return 0.0
|
|
34
|
+
return float(np.mean(class_acc))
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _group_accuracy(y_true, y_pred, groups: Sequence[Any]) -> dict[Any, float]:
|
|
38
|
+
true = _labels(y_true)
|
|
39
|
+
pred = _predictions(y_pred)
|
|
40
|
+
groups = np.asarray(groups)
|
|
41
|
+
result = {}
|
|
42
|
+
for group in sorted(set(groups.tolist()), key=str):
|
|
43
|
+
mask = groups == group
|
|
44
|
+
result[group] = float(np.mean(pred[mask] == true[mask]))
|
|
45
|
+
return result
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def per_device_accuracy(y_true, y_pred, devices) -> dict[Any, float]:
|
|
49
|
+
return _group_accuracy(y_true, y_pred, devices)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def per_scene_accuracy(y_true, y_pred, scenes) -> dict[Any, float]:
|
|
53
|
+
return _group_accuracy(y_true, y_pred, scenes)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def confusion_matrix(y_true, y_pred, num_classes: int = 10) -> np.ndarray:
|
|
57
|
+
true = _labels(y_true)
|
|
58
|
+
pred = _predictions(y_pred)
|
|
59
|
+
matrix = np.zeros((num_classes, num_classes), dtype=np.int64)
|
|
60
|
+
for target, prediction in zip(true, pred):
|
|
61
|
+
if 0 <= target < num_classes and 0 <= prediction < num_classes:
|
|
62
|
+
matrix[int(target), int(prediction)] += 1
|
|
63
|
+
return matrix
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
__all__ = [
|
|
67
|
+
"confusion_matrix",
|
|
68
|
+
"macro_classwise_accuracy",
|
|
69
|
+
"per_device_accuracy",
|
|
70
|
+
"per_scene_accuracy",
|
|
71
|
+
]
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
from justdata.core.filters import (
|
|
2
|
+
filter_by_metadata,
|
|
3
|
+
groupby_metadata,
|
|
4
|
+
metadata_filter_predicate,
|
|
5
|
+
)
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
filter = filter_by_metadata
|
|
9
|
+
groupby = groupby_metadata
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
__all__ = [
|
|
13
|
+
"filter",
|
|
14
|
+
"filter_by_metadata",
|
|
15
|
+
"groupby",
|
|
16
|
+
"groupby_metadata",
|
|
17
|
+
"metadata_filter_predicate",
|
|
18
|
+
]
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
from justdata.acoustic.frontends.ast import ast_kaldi_fbank
|
|
2
|
+
from justdata.acoustic.frontends.kaldi import kaldi_fbank
|
|
3
|
+
from justdata.acoustic.frontends.log import compress_log, logmel, pcen_compression
|
|
4
|
+
from justdata.acoustic.frontends.mel import (
|
|
5
|
+
mel_power,
|
|
6
|
+
mel_spectrogram,
|
|
7
|
+
mel_weight_matrix,
|
|
8
|
+
)
|
|
9
|
+
from justdata.acoustic.frontends.mfcc import mfcc
|
|
10
|
+
from justdata.acoustic.frontends.pcen import pcen_mel
|
|
11
|
+
from justdata.acoustic.frontends.stft import (
|
|
12
|
+
compute_num_frames,
|
|
13
|
+
ensure_waveform_tc,
|
|
14
|
+
raw_waveform,
|
|
15
|
+
reflect_or_constant_pad,
|
|
16
|
+
stft_magnitude,
|
|
17
|
+
stft_power_spectrogram,
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
__all__ = [
|
|
21
|
+
"compress_log",
|
|
22
|
+
"ast_kaldi_fbank",
|
|
23
|
+
"compute_num_frames",
|
|
24
|
+
"ensure_waveform_tc",
|
|
25
|
+
"kaldi_fbank",
|
|
26
|
+
"logmel",
|
|
27
|
+
"mel_power",
|
|
28
|
+
"mel_spectrogram",
|
|
29
|
+
"mel_weight_matrix",
|
|
30
|
+
"mfcc",
|
|
31
|
+
"pcen_compression",
|
|
32
|
+
"pcen_mel",
|
|
33
|
+
"raw_waveform",
|
|
34
|
+
"reflect_or_constant_pad",
|
|
35
|
+
"stft_magnitude",
|
|
36
|
+
"stft_power_spectrogram",
|
|
37
|
+
]
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import tensorflow as tf
|
|
4
|
+
|
|
5
|
+
from justdata.acoustic.configs import FrontendConfig
|
|
6
|
+
from justdata.acoustic.frontends.stft import ensure_waveform_tc
|
|
7
|
+
from justdata.acoustic.registry import register_audio_frontend
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
_KALDI_EPSILON = tf.constant(1.1920928955078125e-7, dtype=tf.float32)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _next_power_of_two(value: int) -> int:
|
|
14
|
+
return 1 if value == 0 else 2 ** (value - 1).bit_length()
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _kaldi_hz_to_mel(frequencies: tf.Tensor) -> tf.Tensor:
|
|
18
|
+
return 1127.0 * tf.math.log1p(frequencies / 700.0)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _kaldi_mel_to_hz(mels: tf.Tensor) -> tf.Tensor:
|
|
22
|
+
return 700.0 * (tf.exp(mels / 1127.0) - 1.0)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _ast_mel_banks(
|
|
26
|
+
*,
|
|
27
|
+
num_mel_bins: int,
|
|
28
|
+
padded_window_size: int,
|
|
29
|
+
sample_rate: int,
|
|
30
|
+
low_freq: float = 20.0,
|
|
31
|
+
high_freq: float = 0.0,
|
|
32
|
+
) -> tf.Tensor:
|
|
33
|
+
nyquist = 0.5 * float(sample_rate)
|
|
34
|
+
if high_freq <= 0.0:
|
|
35
|
+
high_freq += nyquist
|
|
36
|
+
|
|
37
|
+
fft_bin_width = float(sample_rate) / float(padded_window_size)
|
|
38
|
+
mel_low = 1127.0 * tf.math.log1p(tf.constant(low_freq, dtype=tf.float32) / 700.0)
|
|
39
|
+
mel_high = 1127.0 * tf.math.log1p(tf.constant(high_freq, dtype=tf.float32) / 700.0)
|
|
40
|
+
mel_delta = (mel_high - mel_low) / float(num_mel_bins + 1)
|
|
41
|
+
|
|
42
|
+
bins = tf.cast(tf.range(num_mel_bins), tf.float32)[:, tf.newaxis]
|
|
43
|
+
left_mel = mel_low + bins * mel_delta
|
|
44
|
+
center_mel = mel_low + (bins + 1.0) * mel_delta
|
|
45
|
+
right_mel = mel_low + (bins + 2.0) * mel_delta
|
|
46
|
+
|
|
47
|
+
num_fft_bins = padded_window_size // 2
|
|
48
|
+
mel = _kaldi_hz_to_mel(fft_bin_width * tf.cast(tf.range(num_fft_bins), tf.float32))[
|
|
49
|
+
tf.newaxis, :
|
|
50
|
+
]
|
|
51
|
+
|
|
52
|
+
up_slope = (mel - left_mel) / (center_mel - left_mel)
|
|
53
|
+
down_slope = (right_mel - mel) / (right_mel - center_mel)
|
|
54
|
+
weights = tf.maximum(
|
|
55
|
+
tf.zeros([], dtype=tf.float32), tf.minimum(up_slope, down_slope)
|
|
56
|
+
)
|
|
57
|
+
return tf.pad(weights, [[0, 0], [0, 1]])
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@register_audio_frontend("ast_kaldi_fbank")
|
|
61
|
+
def ast_kaldi_fbank(audio: tf.Tensor, config: FrontendConfig | dict) -> tf.Tensor:
|
|
62
|
+
"""AST's torchaudio.compliance.kaldi.fbank subset.
|
|
63
|
+
|
|
64
|
+
This matches the AST dataloader settings:
|
|
65
|
+
htk_compat=True, use_energy=False, window_type="hanning", dither=0.0,
|
|
66
|
+
frame_shift=10 ms, and 128 mel bins by default.
|
|
67
|
+
"""
|
|
68
|
+
|
|
69
|
+
config = FrontendConfig.from_dict(config)
|
|
70
|
+
if config.stft is None:
|
|
71
|
+
raise ValueError("ast_kaldi_fbank frontend requires config.stft")
|
|
72
|
+
if config.mel is None:
|
|
73
|
+
raise ValueError("ast_kaldi_fbank frontend requires config.mel")
|
|
74
|
+
|
|
75
|
+
stft = config.stft
|
|
76
|
+
mel = config.mel
|
|
77
|
+
waveform = ensure_waveform_tc(audio)[:, 0]
|
|
78
|
+
waveform = waveform - tf.reduce_mean(waveform)
|
|
79
|
+
|
|
80
|
+
padded_window_size = _next_power_of_two(stft.win_length)
|
|
81
|
+
if stft.n_fft != padded_window_size:
|
|
82
|
+
raise ValueError(
|
|
83
|
+
"AST fbank expects n_fft to be the next power of two win_length"
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
frames = tf.signal.frame(
|
|
87
|
+
waveform,
|
|
88
|
+
frame_length=stft.win_length,
|
|
89
|
+
frame_step=stft.hop_length,
|
|
90
|
+
pad_end=False,
|
|
91
|
+
)
|
|
92
|
+
frames = tf.cast(frames, tf.float32)
|
|
93
|
+
frames = frames - tf.reduce_mean(frames, axis=1, keepdims=True)
|
|
94
|
+
|
|
95
|
+
previous = tf.concat([frames[:, :1], frames[:, :-1]], axis=1)
|
|
96
|
+
frames = frames - 0.97 * previous
|
|
97
|
+
|
|
98
|
+
window = tf.signal.hann_window(stft.win_length, periodic=False, dtype=tf.float32)
|
|
99
|
+
frames = frames * window[tf.newaxis, :]
|
|
100
|
+
frames = tf.pad(frames, [[0, 0], [0, padded_window_size - stft.win_length]])
|
|
101
|
+
|
|
102
|
+
spectrum = tf.abs(tf.signal.rfft(frames))
|
|
103
|
+
spectrum = tf.square(spectrum)
|
|
104
|
+
weights = _ast_mel_banks(
|
|
105
|
+
num_mel_bins=mel.n_mels,
|
|
106
|
+
padded_window_size=padded_window_size,
|
|
107
|
+
sample_rate=stft.sample_rate,
|
|
108
|
+
low_freq=mel.f_min,
|
|
109
|
+
high_freq=0.0 if mel.f_max is None else mel.f_max,
|
|
110
|
+
)
|
|
111
|
+
mel_energies = tf.matmul(spectrum, weights, transpose_b=True)
|
|
112
|
+
mel_energies = tf.math.log(tf.maximum(mel_energies, _KALDI_EPSILON))
|
|
113
|
+
return mel_energies[:, :, tf.newaxis]
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
__all__ = ["ast_kaldi_fbank"]
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import replace
|
|
4
|
+
|
|
5
|
+
import tensorflow as tf
|
|
6
|
+
|
|
7
|
+
from justdata.acoustic.configs import FrontendConfig, LogCompressionConfig
|
|
8
|
+
from justdata.acoustic.frontends.log import compress_log
|
|
9
|
+
from justdata.acoustic.frontends.mel import mel_spectrogram
|
|
10
|
+
from justdata.acoustic.normalization import apply_feature_normalization
|
|
11
|
+
from justdata.acoustic.registry import register_audio_frontend
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@register_audio_frontend("kaldi_fbank")
|
|
15
|
+
def kaldi_fbank(audio: tf.Tensor, config: FrontendConfig | dict) -> tf.Tensor:
|
|
16
|
+
config = FrontendConfig.from_dict(config)
|
|
17
|
+
if config.mel is None:
|
|
18
|
+
raise ValueError("kaldi_fbank frontend requires config.mel")
|
|
19
|
+
|
|
20
|
+
kaldi_mel = replace(
|
|
21
|
+
config.mel,
|
|
22
|
+
filterbank_impl="kaldi_compatible",
|
|
23
|
+
mel_scale="htk",
|
|
24
|
+
mel_norm="none",
|
|
25
|
+
)
|
|
26
|
+
kaldi_config = replace(config, mel=kaldi_mel)
|
|
27
|
+
compression = config.log or LogCompressionConfig(kind="log")
|
|
28
|
+
features = compress_log(mel_spectrogram(audio, kaldi_config), compression)
|
|
29
|
+
return apply_feature_normalization(features, config.norm)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
__all__ = [
|
|
33
|
+
"kaldi_fbank",
|
|
34
|
+
]
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import tensorflow as tf
|
|
4
|
+
|
|
5
|
+
from justdata.acoustic.configs import FrontendConfig, LogCompressionConfig
|
|
6
|
+
from justdata.acoustic.frontends.mel import mel_spectrogram
|
|
7
|
+
from justdata.acoustic.normalization import apply_feature_normalization
|
|
8
|
+
from justdata.acoustic.registry import register_audio_frontend
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
_LOG10 = tf.math.log(tf.constant(10.0, dtype=tf.float32))
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _log10(x: tf.Tensor) -> tf.Tensor:
|
|
15
|
+
return tf.math.log(x) / tf.cast(_LOG10, x.dtype)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _example_axes(x: tf.Tensor) -> list[int]:
|
|
19
|
+
if x.shape.rank is not None and x.shape.rank >= 4:
|
|
20
|
+
return list(range(1, x.shape.rank))
|
|
21
|
+
return list(range(x.shape.rank or 0))
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _ref_value(x: tf.Tensor, ref: float | str) -> tf.Tensor:
|
|
25
|
+
if ref == "max":
|
|
26
|
+
return tf.maximum(
|
|
27
|
+
tf.reduce_max(x, axis=_example_axes(x), keepdims=True),
|
|
28
|
+
tf.cast(1e-30, x.dtype),
|
|
29
|
+
)
|
|
30
|
+
return tf.cast(ref, x.dtype)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def pcen_compression(x: tf.Tensor, config: LogCompressionConfig | dict) -> tf.Tensor:
|
|
34
|
+
config = LogCompressionConfig.from_dict(config)
|
|
35
|
+
x = tf.cast(tf.convert_to_tensor(x), tf.float32)
|
|
36
|
+
x = tf.maximum(x, tf.cast(config.amin, x.dtype))
|
|
37
|
+
smooth = tf.cast(config.smooth_coef, x.dtype)
|
|
38
|
+
|
|
39
|
+
first = x[0]
|
|
40
|
+
|
|
41
|
+
def _scan(prev: tf.Tensor, current: tf.Tensor) -> tf.Tensor:
|
|
42
|
+
return (1.0 - smooth) * prev + smooth * current
|
|
43
|
+
|
|
44
|
+
smoother = tf.concat(
|
|
45
|
+
[tf.expand_dims(first, 0), tf.scan(_scan, x[1:], initializer=first)], axis=0
|
|
46
|
+
)
|
|
47
|
+
pcen = tf.pow(
|
|
48
|
+
x / tf.pow(tf.cast(config.eps, x.dtype) + smoother, config.alpha)
|
|
49
|
+
+ config.delta,
|
|
50
|
+
config.r,
|
|
51
|
+
) - tf.pow(tf.cast(config.delta, x.dtype), config.r)
|
|
52
|
+
return tf.maximum(pcen, tf.zeros([], dtype=pcen.dtype))
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def compress_log(
|
|
56
|
+
x: tf.Tensor, config: LogCompressionConfig | dict | None = None
|
|
57
|
+
) -> tf.Tensor:
|
|
58
|
+
config = LogCompressionConfig.from_dict(config or {})
|
|
59
|
+
x = tf.cast(tf.convert_to_tensor(x), tf.float32)
|
|
60
|
+
floor = tf.maximum(x, tf.cast(config.amin, x.dtype))
|
|
61
|
+
|
|
62
|
+
if config.kind == "log":
|
|
63
|
+
return tf.math.log(floor + tf.cast(config.log_offset, x.dtype))
|
|
64
|
+
|
|
65
|
+
if config.kind == "log10":
|
|
66
|
+
return _log10(floor + tf.cast(config.log_offset, x.dtype))
|
|
67
|
+
|
|
68
|
+
if config.kind == "db":
|
|
69
|
+
ref = _ref_value(floor, config.ref)
|
|
70
|
+
db = 10.0 * _log10(floor / ref)
|
|
71
|
+
if config.top_db is not None:
|
|
72
|
+
peak = tf.reduce_max(db, axis=_example_axes(db), keepdims=True)
|
|
73
|
+
db = tf.maximum(db, peak - tf.cast(config.top_db, db.dtype))
|
|
74
|
+
return db
|
|
75
|
+
|
|
76
|
+
if config.kind == "pcen":
|
|
77
|
+
return pcen_compression(floor, config)
|
|
78
|
+
|
|
79
|
+
raise ValueError(f"Unknown log compression kind: {config.kind!r}")
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
@register_audio_frontend("logmel")
|
|
83
|
+
def logmel(audio: tf.Tensor, config: FrontendConfig | dict) -> tf.Tensor:
|
|
84
|
+
config = FrontendConfig.from_dict(config)
|
|
85
|
+
compression = config.log or LogCompressionConfig(kind="log")
|
|
86
|
+
features = compress_log(mel_spectrogram(audio, config), compression)
|
|
87
|
+
return apply_feature_normalization(features, config.norm)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
__all__ = [
|
|
91
|
+
"compress_log",
|
|
92
|
+
"logmel",
|
|
93
|
+
"pcen_compression",
|
|
94
|
+
]
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import tensorflow as tf
|
|
4
|
+
|
|
5
|
+
from justdata.acoustic.configs import FrontendConfig, MelConfig, STFTConfig
|
|
6
|
+
from justdata.acoustic.normalization import apply_feature_normalization
|
|
7
|
+
from justdata.acoustic.registry import register_audio_frontend
|
|
8
|
+
from justdata.acoustic.frontends.stft import stft_power_spectrogram
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _f_max(config: MelConfig, sample_rate: int) -> float:
|
|
12
|
+
f_max = float(sample_rate) / 2.0 if config.f_max is None else float(config.f_max)
|
|
13
|
+
if f_max > float(sample_rate) / 2.0:
|
|
14
|
+
raise ValueError("f_max must not exceed Nyquist frequency")
|
|
15
|
+
return f_max
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _kaldi_hz_to_mel(frequencies: tf.Tensor) -> tf.Tensor:
|
|
19
|
+
return 1127.0 * tf.math.log1p(frequencies / 700.0)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _kaldi_mel_to_hz(mels: tf.Tensor) -> tf.Tensor:
|
|
23
|
+
return 700.0 * (tf.exp(mels / 1127.0) - 1.0)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _kaldi_mel_weight_matrix(config: MelConfig, stft: STFTConfig) -> tf.Tensor:
|
|
27
|
+
num_bins = stft.n_fft // 2 + 1
|
|
28
|
+
f_max = _f_max(config, stft.sample_rate)
|
|
29
|
+
|
|
30
|
+
low_mel = _kaldi_hz_to_mel(tf.constant(float(config.f_min), dtype=tf.float32))
|
|
31
|
+
high_mel = _kaldi_hz_to_mel(tf.constant(f_max, dtype=tf.float32))
|
|
32
|
+
mel_points = tf.linspace(low_mel, high_mel, config.n_mels + 2)
|
|
33
|
+
hz_points = _kaldi_mel_to_hz(mel_points)
|
|
34
|
+
frequencies = tf.linspace(0.0, float(stft.sample_rate) / 2.0, num_bins)
|
|
35
|
+
|
|
36
|
+
lower = hz_points[:-2]
|
|
37
|
+
center = hz_points[1:-1]
|
|
38
|
+
upper = hz_points[2:]
|
|
39
|
+
frequencies = tf.expand_dims(frequencies, axis=1)
|
|
40
|
+
|
|
41
|
+
lower_slope = (frequencies - lower) / tf.maximum(center - lower, 1e-12)
|
|
42
|
+
upper_slope = (upper - frequencies) / tf.maximum(upper - center, 1e-12)
|
|
43
|
+
return tf.maximum(0.0, tf.minimum(lower_slope, upper_slope))
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _efficientat_kaldi_mel_weight_matrix(
|
|
47
|
+
config: MelConfig, stft: STFTConfig
|
|
48
|
+
) -> tf.Tensor:
|
|
49
|
+
num_bins = stft.n_fft // 2
|
|
50
|
+
f_max = _f_max(config, stft.sample_rate)
|
|
51
|
+
low_mel = _kaldi_hz_to_mel(tf.constant(float(config.f_min), tf.float32))
|
|
52
|
+
high_mel = _kaldi_hz_to_mel(tf.constant(f_max, tf.float32))
|
|
53
|
+
mel_points = tf.linspace(low_mel, high_mel, config.n_mels + 2)
|
|
54
|
+
hz_points = _kaldi_mel_to_hz(mel_points)
|
|
55
|
+
frequencies = (
|
|
56
|
+
tf.cast(tf.range(num_bins), tf.float32)
|
|
57
|
+
* float(stft.sample_rate)
|
|
58
|
+
/ float(stft.n_fft)
|
|
59
|
+
)
|
|
60
|
+
frequencies = tf.expand_dims(frequencies, 1)
|
|
61
|
+
lower = hz_points[:-2]
|
|
62
|
+
center = hz_points[1:-1]
|
|
63
|
+
upper = hz_points[2:]
|
|
64
|
+
lower_slope = (frequencies - lower) / tf.maximum(center - lower, 1e-12)
|
|
65
|
+
upper_slope = (upper - frequencies) / tf.maximum(upper - center, 1e-12)
|
|
66
|
+
weights = tf.maximum(0.0, tf.minimum(lower_slope, upper_slope))
|
|
67
|
+
return tf.pad(weights, [[0, 1], [0, 0]])
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _torchaudio_mel_weight_matrix(config: MelConfig, stft: STFTConfig) -> tf.Tensor:
|
|
71
|
+
if config.mel_scale != "htk" or config.mel_norm != "none":
|
|
72
|
+
raise ValueError(
|
|
73
|
+
"The TensorFlow torchaudio-compatible mel path supports HTK scale "
|
|
74
|
+
"without normalization"
|
|
75
|
+
)
|
|
76
|
+
num_bins = stft.n_fft // 2 + 1
|
|
77
|
+
f_max = _f_max(config, stft.sample_rate)
|
|
78
|
+
low_mel = _kaldi_hz_to_mel(tf.constant(float(config.f_min), tf.float32))
|
|
79
|
+
high_mel = _kaldi_hz_to_mel(tf.constant(f_max, tf.float32))
|
|
80
|
+
mel_points = tf.linspace(low_mel, high_mel, config.n_mels + 2)
|
|
81
|
+
hz_points = _kaldi_mel_to_hz(mel_points)
|
|
82
|
+
frequencies = tf.linspace(0.0, float(stft.sample_rate) / 2.0, num_bins)
|
|
83
|
+
frequencies = tf.expand_dims(frequencies, 1)
|
|
84
|
+
lower = hz_points[:-2]
|
|
85
|
+
center = hz_points[1:-1]
|
|
86
|
+
upper = hz_points[2:]
|
|
87
|
+
lower_slope = (frequencies - lower) / tf.maximum(center - lower, 1e-12)
|
|
88
|
+
upper_slope = (upper - frequencies) / tf.maximum(upper - center, 1e-12)
|
|
89
|
+
return tf.maximum(0.0, tf.minimum(lower_slope, upper_slope))
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def mel_weight_matrix(config: FrontendConfig | dict) -> tf.Tensor:
|
|
93
|
+
config = FrontendConfig.from_dict(config)
|
|
94
|
+
if config.stft is None:
|
|
95
|
+
raise ValueError("Mel frontends require config.stft")
|
|
96
|
+
if config.mel is None:
|
|
97
|
+
raise ValueError("Mel frontends require config.mel")
|
|
98
|
+
|
|
99
|
+
stft = config.stft
|
|
100
|
+
mel = config.mel
|
|
101
|
+
num_bins = stft.n_fft // 2 + 1
|
|
102
|
+
f_max = _f_max(mel, stft.sample_rate)
|
|
103
|
+
|
|
104
|
+
if mel.filterbank_impl == "tf":
|
|
105
|
+
if mel.mel_scale != "htk" or mel.mel_norm != "none":
|
|
106
|
+
raise ValueError(
|
|
107
|
+
"TensorFlow mel filters support HTK scale without Slaney normalization"
|
|
108
|
+
)
|
|
109
|
+
return tf.signal.linear_to_mel_weight_matrix(
|
|
110
|
+
num_mel_bins=mel.n_mels,
|
|
111
|
+
num_spectrogram_bins=num_bins,
|
|
112
|
+
sample_rate=stft.sample_rate,
|
|
113
|
+
lower_edge_hertz=mel.f_min,
|
|
114
|
+
upper_edge_hertz=f_max,
|
|
115
|
+
dtype=tf.float32,
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
if mel.filterbank_impl in {"librosa", "torchaudio_or_kaldi_compatible"}:
|
|
119
|
+
try:
|
|
120
|
+
import librosa
|
|
121
|
+
except ImportError as exc:
|
|
122
|
+
raise ImportError(
|
|
123
|
+
"librosa mel filters require the acoustic optional dependencies"
|
|
124
|
+
) from exc
|
|
125
|
+
weights = librosa.filters.mel(
|
|
126
|
+
sr=stft.sample_rate,
|
|
127
|
+
n_fft=stft.n_fft,
|
|
128
|
+
n_mels=mel.n_mels,
|
|
129
|
+
fmin=mel.f_min,
|
|
130
|
+
fmax=f_max,
|
|
131
|
+
htk=mel.mel_scale == "htk",
|
|
132
|
+
norm="slaney" if mel.mel_norm == "slaney" else None,
|
|
133
|
+
)
|
|
134
|
+
return tf.constant(weights.T, dtype=tf.float32)
|
|
135
|
+
|
|
136
|
+
if mel.filterbank_impl == "kaldi_compatible":
|
|
137
|
+
return _kaldi_mel_weight_matrix(mel, stft)
|
|
138
|
+
|
|
139
|
+
if mel.filterbank_impl == "efficientat_kaldi":
|
|
140
|
+
return _efficientat_kaldi_mel_weight_matrix(mel, stft)
|
|
141
|
+
|
|
142
|
+
if mel.filterbank_impl == "torchaudio":
|
|
143
|
+
return _torchaudio_mel_weight_matrix(mel, stft)
|
|
144
|
+
|
|
145
|
+
raise ValueError(f"Unknown mel filterbank implementation: {mel.filterbank_impl!r}")
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def mel_spectrogram(audio: tf.Tensor, config: FrontendConfig | dict) -> tf.Tensor:
|
|
149
|
+
config = FrontendConfig.from_dict(config)
|
|
150
|
+
if config.stft is None:
|
|
151
|
+
raise ValueError("Mel frontends require config.stft")
|
|
152
|
+
if config.mel is None:
|
|
153
|
+
raise ValueError("Mel frontends require config.mel")
|
|
154
|
+
if not config.stft.onesided:
|
|
155
|
+
raise ValueError("Mel frontends require onesided STFT bins")
|
|
156
|
+
|
|
157
|
+
spectrogram = stft_power_spectrogram(audio, config.stft)
|
|
158
|
+
weights = mel_weight_matrix(config)
|
|
159
|
+
return tf.einsum("tfc,fm->tmc", spectrogram, weights)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
@register_audio_frontend("mel_power")
|
|
163
|
+
def mel_power(audio: tf.Tensor, config: FrontendConfig | dict) -> tf.Tensor:
|
|
164
|
+
config = FrontendConfig.from_dict(config)
|
|
165
|
+
features = mel_spectrogram(audio, config)
|
|
166
|
+
return apply_feature_normalization(features, config.norm)
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
__all__ = [
|
|
170
|
+
"mel_power",
|
|
171
|
+
"mel_spectrogram",
|
|
172
|
+
"mel_weight_matrix",
|
|
173
|
+
]
|