justdata 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- justdata/__init__.py +3 -0
- justdata/acoustic/__init__.py +381 -0
- justdata/acoustic/adapters.py +329 -0
- justdata/acoustic/augment/__init__.py +119 -0
- justdata/acoustic/augment/batch.py +813 -0
- justdata/acoustic/augment/configs.py +204 -0
- justdata/acoustic/augment/patchout.py +331 -0
- justdata/acoustic/augment/spectrogram.py +806 -0
- justdata/acoustic/augment/waveform.py +879 -0
- justdata/acoustic/batching.py +27 -0
- justdata/acoustic/channel.py +53 -0
- justdata/acoustic/compat/__init__.py +95 -0
- justdata/acoustic/compat/ast.py +626 -0
- justdata/acoustic/compat/ced.py +152 -0
- justdata/acoustic/compat/efficientat.py +243 -0
- justdata/acoustic/compat/panns.py +115 -0
- justdata/acoustic/compat/passt.py +191 -0
- justdata/acoustic/configs.py +519 -0
- justdata/acoustic/corruptions/__init__.py +33 -0
- justdata/acoustic/corruptions/codec.py +92 -0
- justdata/acoustic/corruptions/datasets.py +155 -0
- justdata/acoustic/corruptions/device.py +39 -0
- justdata/acoustic/corruptions/dynamics.py +55 -0
- justdata/acoustic/corruptions/filters.py +105 -0
- justdata/acoustic/corruptions/noise.py +173 -0
- justdata/acoustic/corruptions/registry.py +230 -0
- justdata/acoustic/corruptions/reverb.py +49 -0
- justdata/acoustic/corruptions/timefreq.py +127 -0
- justdata/acoustic/datasets.py +52 -0
- justdata/acoustic/dcase2025.py +814 -0
- justdata/acoustic/decoding.py +73 -0
- justdata/acoustic/eval_views.py +54 -0
- justdata/acoustic/evals.py +71 -0
- justdata/acoustic/filters.py +18 -0
- justdata/acoustic/frontends/__init__.py +37 -0
- justdata/acoustic/frontends/ast.py +116 -0
- justdata/acoustic/frontends/kaldi.py +34 -0
- justdata/acoustic/frontends/log.py +94 -0
- justdata/acoustic/frontends/mel.py +173 -0
- justdata/acoustic/frontends/mfcc.py +26 -0
- justdata/acoustic/frontends/pcen.py +26 -0
- justdata/acoustic/frontends/stft.py +174 -0
- justdata/acoustic/jax.py +23 -0
- justdata/acoustic/labels.py +169 -0
- justdata/acoustic/layouts.py +86 -0
- justdata/acoustic/metadata.py +152 -0
- justdata/acoustic/normalization.py +196 -0
- justdata/acoustic/pipelines.py +112 -0
- justdata/acoustic/postprocessing.py +328 -0
- justdata/acoustic/preprocessing.py +175 -0
- justdata/acoustic/presets.py +257 -0
- justdata/acoustic/registry.py +381 -0
- justdata/acoustic/resampling.py +218 -0
- justdata/acoustic/schema.py +51 -0
- justdata/acoustic/segment.py +369 -0
- justdata/acoustic/sources.py +503 -0
- justdata/acoustic/stages.py +59 -0
- justdata/acoustic/stats.py +50 -0
- justdata/acoustic/tasks.py +375 -0
- justdata/audio/__init__.py +20 -0
- justdata/core/__init__.py +74 -0
- justdata/core/adapters.py +46 -0
- justdata/core/augmentations.py +34 -0
- justdata/core/filters.py +94 -0
- justdata/core/finalization.py +173 -0
- justdata/core/label_mixing.py +129 -0
- justdata/core/loader.py +384 -0
- justdata/core/metadata.py +250 -0
- justdata/core/presets.py +155 -0
- justdata/core/registry.py +229 -0
- justdata/core/sources.py +87 -0
- justdata/core/stats.py +227 -0
- justdata/corruptions.py +52 -0
- justdata/py.typed +0 -0
- justdata/vision/__init__.py +17 -0
- justdata/vision/augmentations/__init__.py +31 -0
- justdata/vision/augmentations/auto.py +480 -0
- justdata/vision/augmentations/color.py +163 -0
- justdata/vision/augmentations/composed.py +156 -0
- justdata/vision/augmentations/geometric.py +302 -0
- justdata/vision/augmentations/mixing.py +232 -0
- justdata/vision/augmentations/registry.py +142 -0
- justdata/vision/corruptions/__init__.py +16 -0
- justdata/vision/corruptions/blur.py +65 -0
- justdata/vision/corruptions/digital.py +26 -0
- justdata/vision/corruptions/noise.py +20 -0
- justdata/vision/corruptions/registry.py +113 -0
- justdata/vision/corruptions/weather.py +100 -0
- justdata/vision/datasets.py +34 -0
- justdata/vision/encodings/__init__.py +1 -0
- justdata/vision/encodings/heatmaps.py +196 -0
- justdata/vision/minic.py +69 -0
- justdata/vision/pipelines.py +62 -0
- justdata/vision/presets.py +509 -0
- justdata/vision/sources.py +1080 -0
- justdata/vision/stages.py +271 -0
- justdata/vision/tasks/__init__.py +1 -0
- justdata/vision/tasks/classification.py +321 -0
- justdata/vision/tasks/depth.py +25 -0
- justdata/vision/tasks/segmentation.py +194 -0
- justdata/vision/transforms.py +113 -0
- justdata/vision/utils.py +741 -0
- justdata-1.0.0.dist-info/METADATA +860 -0
- justdata-1.0.0.dist-info/RECORD +106 -0
- justdata-1.0.0.dist-info/WHEEL +4 -0
- justdata-1.0.0.dist-info/licenses/LICENSE +23 -0
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections.abc import Callable
|
|
4
|
+
from typing import Any, Literal
|
|
5
|
+
|
|
6
|
+
import tensorflow as tf
|
|
7
|
+
|
|
8
|
+
from justdata.core.metadata import apply_metadata_mode, attach_sidecar_writer
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
MetadataMode = Literal["full", "numeric_only", "none"]
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _pad_dataset(ds: tf.data.Dataset, batch_size: int) -> tf.data.Dataset:
|
|
15
|
+
def first_tensor(value):
|
|
16
|
+
if isinstance(value, dict):
|
|
17
|
+
for child in value.values():
|
|
18
|
+
found = first_tensor(child)
|
|
19
|
+
if found is not None:
|
|
20
|
+
return found
|
|
21
|
+
return None
|
|
22
|
+
return value
|
|
23
|
+
|
|
24
|
+
def get_batch_dim(batch):
|
|
25
|
+
for value in batch.values():
|
|
26
|
+
tensor = first_tensor(value)
|
|
27
|
+
if tensor is not None:
|
|
28
|
+
return tf.shape(tensor)[0]
|
|
29
|
+
return tf.constant(0, dtype=tf.int32)
|
|
30
|
+
|
|
31
|
+
def pad_value(value, pad_size):
|
|
32
|
+
if isinstance(value, dict):
|
|
33
|
+
return {key: pad_value(child, pad_size) for key, child in value.items()}
|
|
34
|
+
|
|
35
|
+
pad_shape = tf.concat([[pad_size], tf.shape(value)[1:]], axis=0)
|
|
36
|
+
if value.dtype == tf.string:
|
|
37
|
+
fill = tf.fill(pad_shape, tf.constant("", dtype=tf.string))
|
|
38
|
+
else:
|
|
39
|
+
fill = tf.zeros(pad_shape, dtype=value.dtype)
|
|
40
|
+
return tf.concat([value, fill], axis=0)
|
|
41
|
+
|
|
42
|
+
def pad_batch(batch):
|
|
43
|
+
current_size = get_batch_dim(batch)
|
|
44
|
+
pad_size = batch_size - current_size
|
|
45
|
+
padded = {key: pad_value(value, pad_size) for key, value in batch.items()}
|
|
46
|
+
padded["padding_mask"] = tf.concat(
|
|
47
|
+
[
|
|
48
|
+
tf.ones((current_size,), dtype=tf.bool),
|
|
49
|
+
tf.zeros((pad_size,), dtype=tf.bool),
|
|
50
|
+
],
|
|
51
|
+
axis=0,
|
|
52
|
+
)
|
|
53
|
+
return padded
|
|
54
|
+
|
|
55
|
+
return ds.map(
|
|
56
|
+
lambda batch: tf.cond(
|
|
57
|
+
get_batch_dim(batch) < batch_size,
|
|
58
|
+
lambda: pad_batch(batch),
|
|
59
|
+
lambda: batch | {"padding_mask": tf.ones((batch_size,), dtype=tf.bool)},
|
|
60
|
+
),
|
|
61
|
+
num_parallel_calls=tf.data.AUTOTUNE,
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def finalize_dataset(
|
|
66
|
+
ds: tf.data.Dataset,
|
|
67
|
+
*,
|
|
68
|
+
postprocess_fn: Callable,
|
|
69
|
+
num_classes: int | None,
|
|
70
|
+
batch_size: int,
|
|
71
|
+
post_postprocess_transform: Callable | None = None,
|
|
72
|
+
metadata_mode: MetadataMode = "full",
|
|
73
|
+
sidecar_metadata_path: str | None = None,
|
|
74
|
+
cache_model_inputs: bool = False,
|
|
75
|
+
model_input_cache_path: str = "",
|
|
76
|
+
drop_remainder: bool = False,
|
|
77
|
+
is_training: bool = False,
|
|
78
|
+
apply_late_augment: bool | None = None,
|
|
79
|
+
late_augment_fn: Callable | None = None,
|
|
80
|
+
rng: tf.random.Generator | None = None,
|
|
81
|
+
deterministic: bool = False,
|
|
82
|
+
shuffle_buffer: int | None = None,
|
|
83
|
+
shuffle_seed: int | None = None,
|
|
84
|
+
prefetch: bool = True,
|
|
85
|
+
as_numpy: bool = False,
|
|
86
|
+
) -> tuple[Any, int | None]:
|
|
87
|
+
"""Finalize preprocessed samples into padded, model-ready batches."""
|
|
88
|
+
if metadata_mode not in {"full", "numeric_only", "none"}:
|
|
89
|
+
raise ValueError(
|
|
90
|
+
"metadata_mode must be one of 'full', 'numeric_only', or 'none'."
|
|
91
|
+
)
|
|
92
|
+
if apply_late_augment is None:
|
|
93
|
+
apply_late_augment = is_training
|
|
94
|
+
if apply_late_augment and late_augment_fn is not None and rng is None:
|
|
95
|
+
raise ValueError("rng is required when late augmentation is enabled.")
|
|
96
|
+
|
|
97
|
+
map_deterministic = deterministic if is_training else None
|
|
98
|
+
|
|
99
|
+
def apply_postprocessing(dataset):
|
|
100
|
+
dataset = dataset.map(
|
|
101
|
+
lambda sample: postprocess_fn(sample, num_classes=num_classes),
|
|
102
|
+
num_parallel_calls=tf.data.AUTOTUNE,
|
|
103
|
+
deterministic=map_deterministic,
|
|
104
|
+
)
|
|
105
|
+
if post_postprocess_transform is not None:
|
|
106
|
+
dataset = post_postprocess_transform(dataset)
|
|
107
|
+
return dataset
|
|
108
|
+
|
|
109
|
+
def apply_metadata(dataset):
|
|
110
|
+
if metadata_mode == "numeric_only" and sidecar_metadata_path is not None:
|
|
111
|
+
dataset = attach_sidecar_writer(dataset, sidecar_metadata_path)
|
|
112
|
+
if metadata_mode != "full":
|
|
113
|
+
dataset = dataset.map(
|
|
114
|
+
lambda sample: apply_metadata_mode(sample, metadata_mode),
|
|
115
|
+
num_parallel_calls=tf.data.AUTOTUNE,
|
|
116
|
+
deterministic=map_deterministic,
|
|
117
|
+
)
|
|
118
|
+
return dataset
|
|
119
|
+
|
|
120
|
+
def apply_metadata_and_cache(dataset):
|
|
121
|
+
if cache_model_inputs and sidecar_metadata_path is not None:
|
|
122
|
+
dataset = dataset.cache(model_input_cache_path)
|
|
123
|
+
return apply_metadata(dataset)
|
|
124
|
+
dataset = apply_metadata(dataset)
|
|
125
|
+
if cache_model_inputs:
|
|
126
|
+
dataset = dataset.cache(model_input_cache_path)
|
|
127
|
+
return dataset
|
|
128
|
+
|
|
129
|
+
should_shuffle = is_training and shuffle_buffer is not None
|
|
130
|
+
if should_shuffle and not cache_model_inputs:
|
|
131
|
+
ds = ds.shuffle(shuffle_buffer, seed=shuffle_seed)
|
|
132
|
+
|
|
133
|
+
ds = apply_postprocessing(ds)
|
|
134
|
+
ds = apply_metadata_and_cache(ds)
|
|
135
|
+
|
|
136
|
+
if should_shuffle and cache_model_inputs:
|
|
137
|
+
ds = ds.shuffle(shuffle_buffer, seed=shuffle_seed)
|
|
138
|
+
|
|
139
|
+
ds = ds.batch(batch_size, drop_remainder=drop_remainder)
|
|
140
|
+
|
|
141
|
+
if apply_late_augment and late_augment_fn is not None:
|
|
142
|
+
ds = ds.map(
|
|
143
|
+
lambda batch: late_augment_fn(
|
|
144
|
+
batch,
|
|
145
|
+
num_classes=num_classes,
|
|
146
|
+
seed=rng.make_seeds(1)[:, 0],
|
|
147
|
+
),
|
|
148
|
+
num_parallel_calls=tf.data.AUTOTUNE,
|
|
149
|
+
deterministic=deterministic,
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
if drop_remainder:
|
|
153
|
+
ds = ds.map(
|
|
154
|
+
lambda batch: (
|
|
155
|
+
batch | {"padding_mask": tf.ones((batch_size,), dtype=tf.bool)}
|
|
156
|
+
),
|
|
157
|
+
num_parallel_calls=tf.data.AUTOTUNE,
|
|
158
|
+
deterministic=map_deterministic,
|
|
159
|
+
)
|
|
160
|
+
else:
|
|
161
|
+
ds = _pad_dataset(ds, batch_size)
|
|
162
|
+
|
|
163
|
+
if prefetch:
|
|
164
|
+
ds = ds.prefetch(tf.data.AUTOTUNE)
|
|
165
|
+
|
|
166
|
+
cardinality = int(tf.data.Dataset.cardinality(ds).numpy())
|
|
167
|
+
n_batches = cardinality if cardinality >= 0 else None
|
|
168
|
+
if as_numpy:
|
|
169
|
+
return ds.as_numpy_iterator(), n_batches
|
|
170
|
+
return ds, n_batches
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
__all__ = ["finalize_dataset"]
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Literal
|
|
4
|
+
|
|
5
|
+
import tensorflow as tf
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
LabelMixMode = Literal["single_label", "multi_label", "event_frames"]
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def apply_label_smoothing(
|
|
12
|
+
labels: tf.Tensor,
|
|
13
|
+
*,
|
|
14
|
+
num_classes: int,
|
|
15
|
+
label_smoothing: float,
|
|
16
|
+
) -> tf.Tensor:
|
|
17
|
+
labels = tf.cast(labels, tf.float32)
|
|
18
|
+
if label_smoothing <= 0.0:
|
|
19
|
+
return labels
|
|
20
|
+
off_value = label_smoothing / float(num_classes)
|
|
21
|
+
return labels * (1.0 - label_smoothing) + off_value
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def prepare_labels_for_mixing(
|
|
25
|
+
labels: tf.Tensor,
|
|
26
|
+
*,
|
|
27
|
+
label_mode: LabelMixMode,
|
|
28
|
+
num_classes: int | None = None,
|
|
29
|
+
label_smoothing: float = 0.0,
|
|
30
|
+
) -> tf.Tensor:
|
|
31
|
+
labels = tf.convert_to_tensor(labels)
|
|
32
|
+
|
|
33
|
+
if label_mode == "single_label":
|
|
34
|
+
if num_classes is None:
|
|
35
|
+
raise ValueError("`num_classes` is required for single-label mixing.")
|
|
36
|
+
|
|
37
|
+
is_dense = tf.logical_and(
|
|
38
|
+
tf.greater_equal(tf.rank(labels), 2),
|
|
39
|
+
tf.equal(tf.shape(labels)[-1], num_classes),
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
def dense() -> tf.Tensor:
|
|
43
|
+
return apply_label_smoothing(
|
|
44
|
+
labels,
|
|
45
|
+
num_classes=num_classes,
|
|
46
|
+
label_smoothing=label_smoothing,
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
def sparse() -> tf.Tensor:
|
|
50
|
+
off_value = label_smoothing / float(num_classes)
|
|
51
|
+
on_value = 1.0 - label_smoothing + off_value
|
|
52
|
+
return tf.one_hot(
|
|
53
|
+
tf.cast(tf.reshape(labels, [-1]), tf.int32),
|
|
54
|
+
num_classes,
|
|
55
|
+
on_value=on_value,
|
|
56
|
+
off_value=off_value,
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
return tf.cond(is_dense, dense, sparse)
|
|
60
|
+
|
|
61
|
+
if label_mode == "multi_label":
|
|
62
|
+
return tf.cast(labels, tf.float32)
|
|
63
|
+
|
|
64
|
+
if label_mode == "event_frames":
|
|
65
|
+
return tf.cast(labels, tf.float32)
|
|
66
|
+
|
|
67
|
+
raise ValueError(f"Unsupported label mixing mode: {label_mode!r}")
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def broadcast_mix_coefficient(coefficient: tf.Tensor, labels: tf.Tensor) -> tf.Tensor:
|
|
71
|
+
coefficient = tf.cast(tf.convert_to_tensor(coefficient), tf.float32)
|
|
72
|
+
rank_delta = tf.maximum(tf.rank(labels) - tf.rank(coefficient), 0)
|
|
73
|
+
target_shape = tf.concat(
|
|
74
|
+
[tf.shape(coefficient), tf.ones([rank_delta], dtype=tf.int32)],
|
|
75
|
+
axis=0,
|
|
76
|
+
)
|
|
77
|
+
return tf.reshape(coefficient, target_shape)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def blend_prepared_labels(
|
|
81
|
+
labels: tf.Tensor,
|
|
82
|
+
partner_labels: tf.Tensor,
|
|
83
|
+
coefficient: tf.Tensor,
|
|
84
|
+
*,
|
|
85
|
+
bce_target: bool = False,
|
|
86
|
+
) -> tf.Tensor:
|
|
87
|
+
labels = tf.cast(labels, tf.float32)
|
|
88
|
+
partner_labels = tf.cast(partner_labels, tf.float32)
|
|
89
|
+
if bce_target:
|
|
90
|
+
return tf.maximum(labels, partner_labels)
|
|
91
|
+
coefficient = broadcast_mix_coefficient(coefficient, labels)
|
|
92
|
+
return coefficient * labels + (1.0 - coefficient) * partner_labels
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def mix_labels(
|
|
96
|
+
labels: tf.Tensor,
|
|
97
|
+
partner_labels: tf.Tensor,
|
|
98
|
+
coefficient: tf.Tensor,
|
|
99
|
+
*,
|
|
100
|
+
label_mode: LabelMixMode,
|
|
101
|
+
num_classes: int | None = None,
|
|
102
|
+
label_smoothing: float = 0.0,
|
|
103
|
+
bce_target: bool = False,
|
|
104
|
+
) -> tf.Tensor:
|
|
105
|
+
labels = prepare_labels_for_mixing(
|
|
106
|
+
labels,
|
|
107
|
+
label_mode=label_mode,
|
|
108
|
+
num_classes=num_classes,
|
|
109
|
+
label_smoothing=label_smoothing,
|
|
110
|
+
)
|
|
111
|
+
partner_labels = prepare_labels_for_mixing(
|
|
112
|
+
partner_labels,
|
|
113
|
+
label_mode=label_mode,
|
|
114
|
+
num_classes=num_classes,
|
|
115
|
+
label_smoothing=label_smoothing,
|
|
116
|
+
)
|
|
117
|
+
return blend_prepared_labels(
|
|
118
|
+
labels, partner_labels, coefficient, bce_target=bce_target
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
__all__ = [
|
|
123
|
+
"LabelMixMode",
|
|
124
|
+
"apply_label_smoothing",
|
|
125
|
+
"blend_prepared_labels",
|
|
126
|
+
"broadcast_mix_coefficient",
|
|
127
|
+
"mix_labels",
|
|
128
|
+
"prepare_labels_for_mixing",
|
|
129
|
+
]
|
justdata/core/loader.py
ADDED
|
@@ -0,0 +1,384 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from functools import partial
|
|
3
|
+
from typing import Any, Dict, Literal, Optional, Union
|
|
4
|
+
|
|
5
|
+
import tensorflow as tf
|
|
6
|
+
from loguru import logger
|
|
7
|
+
|
|
8
|
+
from justdata.core.adapters import get_adapter
|
|
9
|
+
from justdata.core.finalization import finalize_dataset
|
|
10
|
+
from justdata.core.sources import get_source_loader
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _concatenate_tf_datasets(
|
|
14
|
+
dataset_list: list[tf.data.Dataset],
|
|
15
|
+
) -> Optional[tf.data.Dataset]:
|
|
16
|
+
"""Concatenates a list of tf.data.Dataset objects sequentially."""
|
|
17
|
+
if not dataset_list:
|
|
18
|
+
return None
|
|
19
|
+
|
|
20
|
+
concatenated_ds = dataset_list[0]
|
|
21
|
+
for i in range(1, len(dataset_list)):
|
|
22
|
+
concatenated_ds = concatenated_ds.concatenate(dataset_list[i])
|
|
23
|
+
|
|
24
|
+
return concatenated_ds
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def fetch_ds(
|
|
28
|
+
dataset_names: list[str],
|
|
29
|
+
splits_info: Union[list[str], Dict[str, list[str]]],
|
|
30
|
+
data_dir: Union[None, str, os.PathLike] = None,
|
|
31
|
+
*,
|
|
32
|
+
source_filter_fn=None,
|
|
33
|
+
) -> Optional[tf.data.Dataset]:
|
|
34
|
+
"""
|
|
35
|
+
Loads and concatenates multiple TensorFlow Datasets and their splits.
|
|
36
|
+
|
|
37
|
+
This function performs a two-level concatenation:
|
|
38
|
+
1. For each dataset name in `dataset_names`:
|
|
39
|
+
- If `splits_info` is a list, those splits are loaded.
|
|
40
|
+
- If `splits_info` is a dictionary, the splits for the current
|
|
41
|
+
dataset name are looked up in the dictionary. If not found or
|
|
42
|
+
empty, the dataset is skipped.
|
|
43
|
+
All successfully loaded splits for the current dataset are concatenated.
|
|
44
|
+
2. The resulting datasets (one for each successfully processed name in
|
|
45
|
+
`dataset_names`, now containing its specified splits) are then
|
|
46
|
+
concatenated into a single, final `tf.data.Dataset`.
|
|
47
|
+
|
|
48
|
+
Args:
|
|
49
|
+
dataset_names: A list of dataset names to attempt to load
|
|
50
|
+
(e.g., ['cifar10', 'mnist']). The order is preserved.
|
|
51
|
+
splits_info: Defines the splits to load.
|
|
52
|
+
- If a List[str] (e.g., ['train', 'test']): these splits are
|
|
53
|
+
loaded for every dataset in `dataset_names`. If the list is
|
|
54
|
+
empty, no data will be loaded for any dataset.
|
|
55
|
+
- If a Dict[str, List[str]] (e.g.,
|
|
56
|
+
{'cifar10': ['train'], 'mnist': ['train', 'test']}):
|
|
57
|
+
For each dataset name in `dataset_names` that is also a key
|
|
58
|
+
in this dict, the corresponding list of splits is loaded.
|
|
59
|
+
Datasets in `dataset_names` not found as keys, or with an
|
|
60
|
+
empty list of splits in the dict, will be skipped.
|
|
61
|
+
data_dir: Optional path to the directory where datasets are stored or
|
|
62
|
+
will be downloaded. If None, defaults to
|
|
63
|
+
`tfds.builder.DEFAULT_DATA_DIR`.
|
|
64
|
+
source_filter_fn: Optional predicate applied to raw source records after
|
|
65
|
+
split concatenation and before adapter mapping. It may
|
|
66
|
+
reference only fields exposed by the source loader.
|
|
67
|
+
|
|
68
|
+
Returns:
|
|
69
|
+
A `tf.data.Dataset` instance representing the concatenation of all
|
|
70
|
+
specified and successfully loaded datasets and splits. Returns `None`
|
|
71
|
+
if `dataset_names` is empty, if `splits_info` leads to no loadable
|
|
72
|
+
splits for any dataset, or if all loading attempts fail.
|
|
73
|
+
|
|
74
|
+
Note:
|
|
75
|
+
- `tfds.load` is called with `as_supervised=False` and
|
|
76
|
+
`shuffle_files=False`. These parameters are fixed internally.
|
|
77
|
+
- If loading splits for a particular dataset name fails (e.g., due
|
|
78
|
+
to an invalid name or unavailable data), a warning is logged,
|
|
79
|
+
and that dataset is skipped. The function will attempt to proceed
|
|
80
|
+
with other dataset names.
|
|
81
|
+
"""
|
|
82
|
+
if not dataset_names:
|
|
83
|
+
logger.info("Empty 'dataset_names' list provided. Nothing to load.")
|
|
84
|
+
return None
|
|
85
|
+
|
|
86
|
+
datasets_with_splits = []
|
|
87
|
+
|
|
88
|
+
for dataset_name in dataset_names:
|
|
89
|
+
splits_to_load: Optional[list[str]] = None
|
|
90
|
+
|
|
91
|
+
if isinstance(splits_info, list):
|
|
92
|
+
if not splits_info:
|
|
93
|
+
logger.warning(
|
|
94
|
+
f"Global 'splits_info' list is empty. "
|
|
95
|
+
f"Skipping dataset '{dataset_name}'."
|
|
96
|
+
)
|
|
97
|
+
continue
|
|
98
|
+
splits_to_load = splits_info
|
|
99
|
+
elif isinstance(splits_info, dict):
|
|
100
|
+
if dataset_name not in splits_info or not splits_info[dataset_name]:
|
|
101
|
+
logger.warning(
|
|
102
|
+
f"Dataset '{dataset_name}' not found in 'splits_info' "
|
|
103
|
+
f"dict, or its split list is empty. Skipping."
|
|
104
|
+
)
|
|
105
|
+
continue
|
|
106
|
+
splits_to_load = splits_info[dataset_name]
|
|
107
|
+
else:
|
|
108
|
+
logger.error(
|
|
109
|
+
"Invalid type for 'splits_info'. Must be List[str] or "
|
|
110
|
+
"Dict[str, List[str]]. Cannot proceed."
|
|
111
|
+
)
|
|
112
|
+
return None
|
|
113
|
+
|
|
114
|
+
try:
|
|
115
|
+
loaded_splits = get_source_loader(dataset_name)(
|
|
116
|
+
dataset_name,
|
|
117
|
+
splits_to_load,
|
|
118
|
+
data_dir,
|
|
119
|
+
)
|
|
120
|
+
except Exception as e:
|
|
121
|
+
logger.warning(
|
|
122
|
+
f"Warning: Failed to load splits for dataset '{dataset_name}'. "
|
|
123
|
+
f"Error: {e}. Skipping this dataset."
|
|
124
|
+
)
|
|
125
|
+
continue
|
|
126
|
+
|
|
127
|
+
if not loaded_splits:
|
|
128
|
+
logger.warning(
|
|
129
|
+
f"No splits data resulted for dataset '{dataset_name}' "
|
|
130
|
+
f"after attempting to load. Skipping."
|
|
131
|
+
)
|
|
132
|
+
continue
|
|
133
|
+
|
|
134
|
+
concatenated_splits = _concatenate_tf_datasets(loaded_splits)
|
|
135
|
+
|
|
136
|
+
if concatenated_splits:
|
|
137
|
+
if source_filter_fn is not None:
|
|
138
|
+
concatenated_splits = concatenated_splits.filter(source_filter_fn)
|
|
139
|
+
adapter = get_adapter(dataset_name)
|
|
140
|
+
concatenated_splits = concatenated_splits.map(
|
|
141
|
+
adapter, num_parallel_calls=tf.data.AUTOTUNE
|
|
142
|
+
)
|
|
143
|
+
datasets_with_splits.append(concatenated_splits)
|
|
144
|
+
|
|
145
|
+
if not datasets_with_splits:
|
|
146
|
+
logger.info(
|
|
147
|
+
"No datasets were successfully loaded and processed "
|
|
148
|
+
"based on the provided names and splits configuration."
|
|
149
|
+
)
|
|
150
|
+
return None
|
|
151
|
+
|
|
152
|
+
return _concatenate_tf_datasets(datasets_with_splits)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def load_ds(
|
|
156
|
+
dataset_names_arg: Union[str, list[str]],
|
|
157
|
+
splits_arg: Union[str, list[str], Dict[str, list[str]]],
|
|
158
|
+
dataset_type: str,
|
|
159
|
+
batch_size: int,
|
|
160
|
+
seed: int,
|
|
161
|
+
num_classes: Optional[int] = None,
|
|
162
|
+
pipeline: Optional[Any] = None,
|
|
163
|
+
preprocess_fn=None,
|
|
164
|
+
augment_fn=None,
|
|
165
|
+
late_augment_fn=None,
|
|
166
|
+
postprocess_fn=None,
|
|
167
|
+
shuffle_buffer: int = 10_000,
|
|
168
|
+
cache_dataset: bool = False,
|
|
169
|
+
cache_path: str = "",
|
|
170
|
+
cache_model_inputs: bool = False,
|
|
171
|
+
model_input_cache_path: str = "",
|
|
172
|
+
allow_train_model_input_cache: bool = False,
|
|
173
|
+
drop_remainder: bool = False,
|
|
174
|
+
data_dir: Union[None, str, os.PathLike] = None,
|
|
175
|
+
return_raw_ds: bool = False,
|
|
176
|
+
deterministic: bool = False,
|
|
177
|
+
as_numpy: bool = False,
|
|
178
|
+
metadata_mode: Literal["full", "numeric_only", "none"] | None = None,
|
|
179
|
+
sidecar_metadata_path: str | None = None,
|
|
180
|
+
filter_fn=None,
|
|
181
|
+
*,
|
|
182
|
+
source_filter_fn=None,
|
|
183
|
+
):
|
|
184
|
+
"""
|
|
185
|
+
Loads, preprocesses, and batches HuggingFace or TensorFlow Datasets.
|
|
186
|
+
|
|
187
|
+
Args:
|
|
188
|
+
dataset_names_arg: A single dataset name (str) or a list of dataset names.
|
|
189
|
+
splits_arg: Defines the splits to load.
|
|
190
|
+
- If str (e.g., "train"): This split is loaded for all datasets.
|
|
191
|
+
- If List[str] (e.g., ["train", "test"]): These splits are loaded
|
|
192
|
+
for all datasets.
|
|
193
|
+
- If Dict[str, List[str]] (e.g., {"cifar10": ["train"]}): Specifies
|
|
194
|
+
splits per dataset. See `fetch_ds` for details.
|
|
195
|
+
dataset_type: Type of dataset, typically "train", "validation", or "test".
|
|
196
|
+
Affects caching, shuffling, and augmentation.
|
|
197
|
+
batch_size: The batch size for the output dataset.
|
|
198
|
+
seed: Random seed for shuffling and augmentations.
|
|
199
|
+
num_classes: Number of classes (needed for one-hot encoding/mixup).
|
|
200
|
+
pipeline: A `DataPipeline` object from `justdata.core.registry`. If provided,
|
|
201
|
+
it builds the necessary functions based on `dataset_type`.
|
|
202
|
+
preprocess_fn: Preprocessing function (fallback if `pipeline` is None).
|
|
203
|
+
augment_fn: Augmentation function (fallback if `pipeline` is None).
|
|
204
|
+
late_augment_fn: Late augmentation function (fallback if `pipeline` is None).
|
|
205
|
+
postprocess_fn: Postprocessing function (fallback if `pipeline` is None).
|
|
206
|
+
shuffle_buffer: Size of the shuffle buffer.
|
|
207
|
+
cache_dataset: Whether to cache the dataset after preprocessing. Disabled
|
|
208
|
+
by default so omitted caching remains streaming-friendly.
|
|
209
|
+
cache_path: Filesystem path for the preprocessing cache. When
|
|
210
|
+
``cache_dataset=True``, an empty path uses memory and a nonempty path
|
|
211
|
+
uses the filesystem.
|
|
212
|
+
cache_model_inputs: Whether to cache samples after deterministic
|
|
213
|
+
postprocessing, before batching. Use ``model_input_cache_path`` for
|
|
214
|
+
local SSD caching of resized/model-ready tensors.
|
|
215
|
+
model_input_cache_path: Filesystem path for ``cache_model_inputs``.
|
|
216
|
+
If a filename is not provided, model inputs will be cached in memory.
|
|
217
|
+
allow_train_model_input_cache: Required opt-in for model-input caching
|
|
218
|
+
with training or evaluation augmentation. Only use this when
|
|
219
|
+
materializing the first sampled augmented views is intentional.
|
|
220
|
+
drop_remainder: Choose to drop or pad batches without the correct size,
|
|
221
|
+
data_dir: Optional path to the TFDS data directory.
|
|
222
|
+
return_raw_ds: If True, returns the dataset immediately after
|
|
223
|
+
preprocessing (and caching) but BEFORE standard
|
|
224
|
+
augmentation, postprocessing, or batching.
|
|
225
|
+
Returns (ds, tools_dict) where tools_dict contains
|
|
226
|
+
``finalize_fn``, ``postprocess_fn``, and ``rng``.
|
|
227
|
+
deterministic: If false, sacrifices determinism for performance.
|
|
228
|
+
as_numpy: If True, returns an iterator yielding NumPy arrays.
|
|
229
|
+
metadata_mode: Controls metadata in output batches. When omitted, uses
|
|
230
|
+
``pipeline.kwargs["metadata_mode"]`` if present, then
|
|
231
|
+
defaults to ``full``. ``numeric_only`` drops strings
|
|
232
|
+
from batches and ``none`` removes metadata.
|
|
233
|
+
sidecar_metadata_path: JSONL path for string metadata when
|
|
234
|
+
``metadata_mode="numeric_only"``.
|
|
235
|
+
filter_fn: Optional predicate applied after preprocessing and caching,
|
|
236
|
+
before augmentation, postprocessing, and batching.
|
|
237
|
+
source_filter_fn: Optional predicate applied to raw source records before
|
|
238
|
+
adapter mapping. Unlike ``filter_fn``, it may reference
|
|
239
|
+
only fields exposed by the source loader.
|
|
240
|
+
Returns:
|
|
241
|
+
``(dataset, n_batches)`` where ``dataset`` is a batched
|
|
242
|
+
``tf.data.Dataset`` or NumPy iterator and ``n_batches`` is an integer
|
|
243
|
+
when cardinality is known, otherwise ``None``. With
|
|
244
|
+
``return_raw_ds=True``, returns ``(dataset, tools_dict)`` instead.
|
|
245
|
+
|
|
246
|
+
Raises:
|
|
247
|
+
ValueError: If dataset loading fails and `fetch_ds` returns None.
|
|
248
|
+
TypeError: If `dataset_names_arg` or `splits_arg` have invalid types.
|
|
249
|
+
"""
|
|
250
|
+
rng = tf.random.Generator.from_seed(seed)
|
|
251
|
+
if metadata_mode is None:
|
|
252
|
+
metadata_mode = (
|
|
253
|
+
pipeline.kwargs.get("metadata_mode", "full")
|
|
254
|
+
if pipeline is not None
|
|
255
|
+
else "full"
|
|
256
|
+
)
|
|
257
|
+
if metadata_mode not in {"full", "numeric_only", "none"}:
|
|
258
|
+
raise ValueError(
|
|
259
|
+
"metadata_mode must be one of 'full', 'numeric_only', or 'none'."
|
|
260
|
+
)
|
|
261
|
+
|
|
262
|
+
dataset_names: list[str]
|
|
263
|
+
if isinstance(dataset_names_arg, str):
|
|
264
|
+
dataset_names = [dataset_names_arg]
|
|
265
|
+
elif isinstance(dataset_names_arg, list):
|
|
266
|
+
dataset_names = dataset_names_arg
|
|
267
|
+
else:
|
|
268
|
+
raise TypeError("dataset_names_arg must be a string or a list of strings.")
|
|
269
|
+
|
|
270
|
+
splits: Union[list[str], Dict[str, list[str]]]
|
|
271
|
+
if isinstance(splits_arg, str):
|
|
272
|
+
splits = [splits_arg]
|
|
273
|
+
elif isinstance(splits_arg, list):
|
|
274
|
+
splits = splits_arg
|
|
275
|
+
elif isinstance(splits_arg, dict):
|
|
276
|
+
splits = splits_arg
|
|
277
|
+
else:
|
|
278
|
+
raise TypeError(
|
|
279
|
+
"splits_arg must be a string, a list of strings, or a "
|
|
280
|
+
"dictionary mapping dataset names to lists of strings."
|
|
281
|
+
)
|
|
282
|
+
|
|
283
|
+
is_training = dataset_type == "train"
|
|
284
|
+
augment_eval = bool(
|
|
285
|
+
pipeline is not None and pipeline.kwargs.get("augment_eval", False)
|
|
286
|
+
)
|
|
287
|
+
apply_augmentation = is_training or augment_eval
|
|
288
|
+
if cache_model_inputs and apply_augmentation and not allow_train_model_input_cache:
|
|
289
|
+
raise ValueError(
|
|
290
|
+
"cache_model_inputs with training or evaluation augmentation requires "
|
|
291
|
+
"allow_train_model_input_cache=True."
|
|
292
|
+
)
|
|
293
|
+
if cache_dataset and cache_model_inputs and cache_path and model_input_cache_path:
|
|
294
|
+
preprocess_cache_path = os.path.abspath(os.fspath(cache_path))
|
|
295
|
+
model_cache_path = os.path.abspath(os.fspath(model_input_cache_path))
|
|
296
|
+
if preprocess_cache_path == model_cache_path:
|
|
297
|
+
raise ValueError(
|
|
298
|
+
"cache_path and model_input_cache_path must be different when "
|
|
299
|
+
"both cache stages are enabled."
|
|
300
|
+
)
|
|
301
|
+
|
|
302
|
+
if source_filter_fn is None:
|
|
303
|
+
ds = fetch_ds(dataset_names, splits, data_dir)
|
|
304
|
+
else:
|
|
305
|
+
ds = fetch_ds(
|
|
306
|
+
dataset_names,
|
|
307
|
+
splits,
|
|
308
|
+
data_dir,
|
|
309
|
+
source_filter_fn=source_filter_fn,
|
|
310
|
+
)
|
|
311
|
+
|
|
312
|
+
if ds is None:
|
|
313
|
+
raise ValueError(
|
|
314
|
+
"Dataset loading failed (fetch_ds returned None). "
|
|
315
|
+
"Check logs for details on dataset names, splits, or data issues."
|
|
316
|
+
)
|
|
317
|
+
|
|
318
|
+
if pipeline is not None:
|
|
319
|
+
preprocess_fn, augment_fn, late_augment_fn, postprocess_fn = pipeline.build(
|
|
320
|
+
is_training=is_training
|
|
321
|
+
)
|
|
322
|
+
|
|
323
|
+
def seeded_augment(sample):
|
|
324
|
+
seed = rng.make_seeds(1)[:, 0]
|
|
325
|
+
return augment_fn(sample, seed=seed)
|
|
326
|
+
|
|
327
|
+
ds = ds.map(
|
|
328
|
+
preprocess_fn,
|
|
329
|
+
num_parallel_calls=tf.data.AUTOTUNE,
|
|
330
|
+
deterministic=deterministic if is_training else None,
|
|
331
|
+
)
|
|
332
|
+
|
|
333
|
+
# An empty path caches in memory; a nonempty path caches on the filesystem.
|
|
334
|
+
# Large datasets should use an explicit disk path or remain uncached.
|
|
335
|
+
if cache_dataset:
|
|
336
|
+
ds = ds.cache(cache_path)
|
|
337
|
+
else:
|
|
338
|
+
logger.info(
|
|
339
|
+
f"Caching disabled for '{dataset_type}' dataset. "
|
|
340
|
+
"Samples will be re-read each epoch (streaming-friendly)."
|
|
341
|
+
)
|
|
342
|
+
|
|
343
|
+
if filter_fn is not None:
|
|
344
|
+
ds = ds.filter(filter_fn)
|
|
345
|
+
|
|
346
|
+
finalize_fn = partial(
|
|
347
|
+
finalize_dataset,
|
|
348
|
+
postprocess_fn=postprocess_fn,
|
|
349
|
+
num_classes=num_classes,
|
|
350
|
+
batch_size=batch_size,
|
|
351
|
+
metadata_mode=metadata_mode,
|
|
352
|
+
sidecar_metadata_path=sidecar_metadata_path,
|
|
353
|
+
cache_model_inputs=cache_model_inputs,
|
|
354
|
+
model_input_cache_path=model_input_cache_path,
|
|
355
|
+
drop_remainder=drop_remainder,
|
|
356
|
+
is_training=is_training,
|
|
357
|
+
apply_late_augment=apply_augmentation,
|
|
358
|
+
late_augment_fn=late_augment_fn,
|
|
359
|
+
rng=rng,
|
|
360
|
+
deterministic=deterministic,
|
|
361
|
+
shuffle_buffer=shuffle_buffer if is_training else None,
|
|
362
|
+
shuffle_seed=seed,
|
|
363
|
+
as_numpy=as_numpy,
|
|
364
|
+
)
|
|
365
|
+
|
|
366
|
+
if return_raw_ds:
|
|
367
|
+
# Return necessary components to build custom pipelines
|
|
368
|
+
return (
|
|
369
|
+
ds,
|
|
370
|
+
{
|
|
371
|
+
"finalize_fn": finalize_fn,
|
|
372
|
+
"postprocess_fn": postprocess_fn,
|
|
373
|
+
"rng": rng,
|
|
374
|
+
},
|
|
375
|
+
)
|
|
376
|
+
|
|
377
|
+
if apply_augmentation:
|
|
378
|
+
ds = ds.map(
|
|
379
|
+
seeded_augment,
|
|
380
|
+
num_parallel_calls=1 if deterministic else tf.data.AUTOTUNE,
|
|
381
|
+
deterministic=deterministic,
|
|
382
|
+
)
|
|
383
|
+
|
|
384
|
+
return finalize_fn(ds)
|