justdata 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. justdata/__init__.py +3 -0
  2. justdata/acoustic/__init__.py +381 -0
  3. justdata/acoustic/adapters.py +329 -0
  4. justdata/acoustic/augment/__init__.py +119 -0
  5. justdata/acoustic/augment/batch.py +813 -0
  6. justdata/acoustic/augment/configs.py +204 -0
  7. justdata/acoustic/augment/patchout.py +331 -0
  8. justdata/acoustic/augment/spectrogram.py +806 -0
  9. justdata/acoustic/augment/waveform.py +879 -0
  10. justdata/acoustic/batching.py +27 -0
  11. justdata/acoustic/channel.py +53 -0
  12. justdata/acoustic/compat/__init__.py +95 -0
  13. justdata/acoustic/compat/ast.py +626 -0
  14. justdata/acoustic/compat/ced.py +152 -0
  15. justdata/acoustic/compat/efficientat.py +243 -0
  16. justdata/acoustic/compat/panns.py +115 -0
  17. justdata/acoustic/compat/passt.py +191 -0
  18. justdata/acoustic/configs.py +519 -0
  19. justdata/acoustic/corruptions/__init__.py +33 -0
  20. justdata/acoustic/corruptions/codec.py +92 -0
  21. justdata/acoustic/corruptions/datasets.py +155 -0
  22. justdata/acoustic/corruptions/device.py +39 -0
  23. justdata/acoustic/corruptions/dynamics.py +55 -0
  24. justdata/acoustic/corruptions/filters.py +105 -0
  25. justdata/acoustic/corruptions/noise.py +173 -0
  26. justdata/acoustic/corruptions/registry.py +230 -0
  27. justdata/acoustic/corruptions/reverb.py +49 -0
  28. justdata/acoustic/corruptions/timefreq.py +127 -0
  29. justdata/acoustic/datasets.py +52 -0
  30. justdata/acoustic/dcase2025.py +814 -0
  31. justdata/acoustic/decoding.py +73 -0
  32. justdata/acoustic/eval_views.py +54 -0
  33. justdata/acoustic/evals.py +71 -0
  34. justdata/acoustic/filters.py +18 -0
  35. justdata/acoustic/frontends/__init__.py +37 -0
  36. justdata/acoustic/frontends/ast.py +116 -0
  37. justdata/acoustic/frontends/kaldi.py +34 -0
  38. justdata/acoustic/frontends/log.py +94 -0
  39. justdata/acoustic/frontends/mel.py +173 -0
  40. justdata/acoustic/frontends/mfcc.py +26 -0
  41. justdata/acoustic/frontends/pcen.py +26 -0
  42. justdata/acoustic/frontends/stft.py +174 -0
  43. justdata/acoustic/jax.py +23 -0
  44. justdata/acoustic/labels.py +169 -0
  45. justdata/acoustic/layouts.py +86 -0
  46. justdata/acoustic/metadata.py +152 -0
  47. justdata/acoustic/normalization.py +196 -0
  48. justdata/acoustic/pipelines.py +112 -0
  49. justdata/acoustic/postprocessing.py +328 -0
  50. justdata/acoustic/preprocessing.py +175 -0
  51. justdata/acoustic/presets.py +257 -0
  52. justdata/acoustic/registry.py +381 -0
  53. justdata/acoustic/resampling.py +218 -0
  54. justdata/acoustic/schema.py +51 -0
  55. justdata/acoustic/segment.py +369 -0
  56. justdata/acoustic/sources.py +503 -0
  57. justdata/acoustic/stages.py +59 -0
  58. justdata/acoustic/stats.py +50 -0
  59. justdata/acoustic/tasks.py +375 -0
  60. justdata/audio/__init__.py +20 -0
  61. justdata/core/__init__.py +74 -0
  62. justdata/core/adapters.py +46 -0
  63. justdata/core/augmentations.py +34 -0
  64. justdata/core/filters.py +94 -0
  65. justdata/core/finalization.py +173 -0
  66. justdata/core/label_mixing.py +129 -0
  67. justdata/core/loader.py +384 -0
  68. justdata/core/metadata.py +250 -0
  69. justdata/core/presets.py +155 -0
  70. justdata/core/registry.py +229 -0
  71. justdata/core/sources.py +87 -0
  72. justdata/core/stats.py +227 -0
  73. justdata/corruptions.py +52 -0
  74. justdata/py.typed +0 -0
  75. justdata/vision/__init__.py +17 -0
  76. justdata/vision/augmentations/__init__.py +31 -0
  77. justdata/vision/augmentations/auto.py +480 -0
  78. justdata/vision/augmentations/color.py +163 -0
  79. justdata/vision/augmentations/composed.py +156 -0
  80. justdata/vision/augmentations/geometric.py +302 -0
  81. justdata/vision/augmentations/mixing.py +232 -0
  82. justdata/vision/augmentations/registry.py +142 -0
  83. justdata/vision/corruptions/__init__.py +16 -0
  84. justdata/vision/corruptions/blur.py +65 -0
  85. justdata/vision/corruptions/digital.py +26 -0
  86. justdata/vision/corruptions/noise.py +20 -0
  87. justdata/vision/corruptions/registry.py +113 -0
  88. justdata/vision/corruptions/weather.py +100 -0
  89. justdata/vision/datasets.py +34 -0
  90. justdata/vision/encodings/__init__.py +1 -0
  91. justdata/vision/encodings/heatmaps.py +196 -0
  92. justdata/vision/minic.py +69 -0
  93. justdata/vision/pipelines.py +62 -0
  94. justdata/vision/presets.py +509 -0
  95. justdata/vision/sources.py +1080 -0
  96. justdata/vision/stages.py +271 -0
  97. justdata/vision/tasks/__init__.py +1 -0
  98. justdata/vision/tasks/classification.py +321 -0
  99. justdata/vision/tasks/depth.py +25 -0
  100. justdata/vision/tasks/segmentation.py +194 -0
  101. justdata/vision/transforms.py +113 -0
  102. justdata/vision/utils.py +741 -0
  103. justdata-1.0.0.dist-info/METADATA +860 -0
  104. justdata-1.0.0.dist-info/RECORD +106 -0
  105. justdata-1.0.0.dist-info/WHEEL +4 -0
  106. justdata-1.0.0.dist-info/licenses/LICENSE +23 -0
@@ -0,0 +1,173 @@
1
+ from __future__ import annotations
2
+
3
+ from collections.abc import Callable
4
+ from typing import Any, Literal
5
+
6
+ import tensorflow as tf
7
+
8
+ from justdata.core.metadata import apply_metadata_mode, attach_sidecar_writer
9
+
10
+
11
+ MetadataMode = Literal["full", "numeric_only", "none"]
12
+
13
+
14
+ def _pad_dataset(ds: tf.data.Dataset, batch_size: int) -> tf.data.Dataset:
15
+ def first_tensor(value):
16
+ if isinstance(value, dict):
17
+ for child in value.values():
18
+ found = first_tensor(child)
19
+ if found is not None:
20
+ return found
21
+ return None
22
+ return value
23
+
24
+ def get_batch_dim(batch):
25
+ for value in batch.values():
26
+ tensor = first_tensor(value)
27
+ if tensor is not None:
28
+ return tf.shape(tensor)[0]
29
+ return tf.constant(0, dtype=tf.int32)
30
+
31
+ def pad_value(value, pad_size):
32
+ if isinstance(value, dict):
33
+ return {key: pad_value(child, pad_size) for key, child in value.items()}
34
+
35
+ pad_shape = tf.concat([[pad_size], tf.shape(value)[1:]], axis=0)
36
+ if value.dtype == tf.string:
37
+ fill = tf.fill(pad_shape, tf.constant("", dtype=tf.string))
38
+ else:
39
+ fill = tf.zeros(pad_shape, dtype=value.dtype)
40
+ return tf.concat([value, fill], axis=0)
41
+
42
+ def pad_batch(batch):
43
+ current_size = get_batch_dim(batch)
44
+ pad_size = batch_size - current_size
45
+ padded = {key: pad_value(value, pad_size) for key, value in batch.items()}
46
+ padded["padding_mask"] = tf.concat(
47
+ [
48
+ tf.ones((current_size,), dtype=tf.bool),
49
+ tf.zeros((pad_size,), dtype=tf.bool),
50
+ ],
51
+ axis=0,
52
+ )
53
+ return padded
54
+
55
+ return ds.map(
56
+ lambda batch: tf.cond(
57
+ get_batch_dim(batch) < batch_size,
58
+ lambda: pad_batch(batch),
59
+ lambda: batch | {"padding_mask": tf.ones((batch_size,), dtype=tf.bool)},
60
+ ),
61
+ num_parallel_calls=tf.data.AUTOTUNE,
62
+ )
63
+
64
+
65
+ def finalize_dataset(
66
+ ds: tf.data.Dataset,
67
+ *,
68
+ postprocess_fn: Callable,
69
+ num_classes: int | None,
70
+ batch_size: int,
71
+ post_postprocess_transform: Callable | None = None,
72
+ metadata_mode: MetadataMode = "full",
73
+ sidecar_metadata_path: str | None = None,
74
+ cache_model_inputs: bool = False,
75
+ model_input_cache_path: str = "",
76
+ drop_remainder: bool = False,
77
+ is_training: bool = False,
78
+ apply_late_augment: bool | None = None,
79
+ late_augment_fn: Callable | None = None,
80
+ rng: tf.random.Generator | None = None,
81
+ deterministic: bool = False,
82
+ shuffle_buffer: int | None = None,
83
+ shuffle_seed: int | None = None,
84
+ prefetch: bool = True,
85
+ as_numpy: bool = False,
86
+ ) -> tuple[Any, int | None]:
87
+ """Finalize preprocessed samples into padded, model-ready batches."""
88
+ if metadata_mode not in {"full", "numeric_only", "none"}:
89
+ raise ValueError(
90
+ "metadata_mode must be one of 'full', 'numeric_only', or 'none'."
91
+ )
92
+ if apply_late_augment is None:
93
+ apply_late_augment = is_training
94
+ if apply_late_augment and late_augment_fn is not None and rng is None:
95
+ raise ValueError("rng is required when late augmentation is enabled.")
96
+
97
+ map_deterministic = deterministic if is_training else None
98
+
99
+ def apply_postprocessing(dataset):
100
+ dataset = dataset.map(
101
+ lambda sample: postprocess_fn(sample, num_classes=num_classes),
102
+ num_parallel_calls=tf.data.AUTOTUNE,
103
+ deterministic=map_deterministic,
104
+ )
105
+ if post_postprocess_transform is not None:
106
+ dataset = post_postprocess_transform(dataset)
107
+ return dataset
108
+
109
+ def apply_metadata(dataset):
110
+ if metadata_mode == "numeric_only" and sidecar_metadata_path is not None:
111
+ dataset = attach_sidecar_writer(dataset, sidecar_metadata_path)
112
+ if metadata_mode != "full":
113
+ dataset = dataset.map(
114
+ lambda sample: apply_metadata_mode(sample, metadata_mode),
115
+ num_parallel_calls=tf.data.AUTOTUNE,
116
+ deterministic=map_deterministic,
117
+ )
118
+ return dataset
119
+
120
+ def apply_metadata_and_cache(dataset):
121
+ if cache_model_inputs and sidecar_metadata_path is not None:
122
+ dataset = dataset.cache(model_input_cache_path)
123
+ return apply_metadata(dataset)
124
+ dataset = apply_metadata(dataset)
125
+ if cache_model_inputs:
126
+ dataset = dataset.cache(model_input_cache_path)
127
+ return dataset
128
+
129
+ should_shuffle = is_training and shuffle_buffer is not None
130
+ if should_shuffle and not cache_model_inputs:
131
+ ds = ds.shuffle(shuffle_buffer, seed=shuffle_seed)
132
+
133
+ ds = apply_postprocessing(ds)
134
+ ds = apply_metadata_and_cache(ds)
135
+
136
+ if should_shuffle and cache_model_inputs:
137
+ ds = ds.shuffle(shuffle_buffer, seed=shuffle_seed)
138
+
139
+ ds = ds.batch(batch_size, drop_remainder=drop_remainder)
140
+
141
+ if apply_late_augment and late_augment_fn is not None:
142
+ ds = ds.map(
143
+ lambda batch: late_augment_fn(
144
+ batch,
145
+ num_classes=num_classes,
146
+ seed=rng.make_seeds(1)[:, 0],
147
+ ),
148
+ num_parallel_calls=tf.data.AUTOTUNE,
149
+ deterministic=deterministic,
150
+ )
151
+
152
+ if drop_remainder:
153
+ ds = ds.map(
154
+ lambda batch: (
155
+ batch | {"padding_mask": tf.ones((batch_size,), dtype=tf.bool)}
156
+ ),
157
+ num_parallel_calls=tf.data.AUTOTUNE,
158
+ deterministic=map_deterministic,
159
+ )
160
+ else:
161
+ ds = _pad_dataset(ds, batch_size)
162
+
163
+ if prefetch:
164
+ ds = ds.prefetch(tf.data.AUTOTUNE)
165
+
166
+ cardinality = int(tf.data.Dataset.cardinality(ds).numpy())
167
+ n_batches = cardinality if cardinality >= 0 else None
168
+ if as_numpy:
169
+ return ds.as_numpy_iterator(), n_batches
170
+ return ds, n_batches
171
+
172
+
173
+ __all__ = ["finalize_dataset"]
@@ -0,0 +1,129 @@
1
+ from __future__ import annotations
2
+
3
+ from typing import Literal
4
+
5
+ import tensorflow as tf
6
+
7
+
8
+ LabelMixMode = Literal["single_label", "multi_label", "event_frames"]
9
+
10
+
11
+ def apply_label_smoothing(
12
+ labels: tf.Tensor,
13
+ *,
14
+ num_classes: int,
15
+ label_smoothing: float,
16
+ ) -> tf.Tensor:
17
+ labels = tf.cast(labels, tf.float32)
18
+ if label_smoothing <= 0.0:
19
+ return labels
20
+ off_value = label_smoothing / float(num_classes)
21
+ return labels * (1.0 - label_smoothing) + off_value
22
+
23
+
24
+ def prepare_labels_for_mixing(
25
+ labels: tf.Tensor,
26
+ *,
27
+ label_mode: LabelMixMode,
28
+ num_classes: int | None = None,
29
+ label_smoothing: float = 0.0,
30
+ ) -> tf.Tensor:
31
+ labels = tf.convert_to_tensor(labels)
32
+
33
+ if label_mode == "single_label":
34
+ if num_classes is None:
35
+ raise ValueError("`num_classes` is required for single-label mixing.")
36
+
37
+ is_dense = tf.logical_and(
38
+ tf.greater_equal(tf.rank(labels), 2),
39
+ tf.equal(tf.shape(labels)[-1], num_classes),
40
+ )
41
+
42
+ def dense() -> tf.Tensor:
43
+ return apply_label_smoothing(
44
+ labels,
45
+ num_classes=num_classes,
46
+ label_smoothing=label_smoothing,
47
+ )
48
+
49
+ def sparse() -> tf.Tensor:
50
+ off_value = label_smoothing / float(num_classes)
51
+ on_value = 1.0 - label_smoothing + off_value
52
+ return tf.one_hot(
53
+ tf.cast(tf.reshape(labels, [-1]), tf.int32),
54
+ num_classes,
55
+ on_value=on_value,
56
+ off_value=off_value,
57
+ )
58
+
59
+ return tf.cond(is_dense, dense, sparse)
60
+
61
+ if label_mode == "multi_label":
62
+ return tf.cast(labels, tf.float32)
63
+
64
+ if label_mode == "event_frames":
65
+ return tf.cast(labels, tf.float32)
66
+
67
+ raise ValueError(f"Unsupported label mixing mode: {label_mode!r}")
68
+
69
+
70
+ def broadcast_mix_coefficient(coefficient: tf.Tensor, labels: tf.Tensor) -> tf.Tensor:
71
+ coefficient = tf.cast(tf.convert_to_tensor(coefficient), tf.float32)
72
+ rank_delta = tf.maximum(tf.rank(labels) - tf.rank(coefficient), 0)
73
+ target_shape = tf.concat(
74
+ [tf.shape(coefficient), tf.ones([rank_delta], dtype=tf.int32)],
75
+ axis=0,
76
+ )
77
+ return tf.reshape(coefficient, target_shape)
78
+
79
+
80
+ def blend_prepared_labels(
81
+ labels: tf.Tensor,
82
+ partner_labels: tf.Tensor,
83
+ coefficient: tf.Tensor,
84
+ *,
85
+ bce_target: bool = False,
86
+ ) -> tf.Tensor:
87
+ labels = tf.cast(labels, tf.float32)
88
+ partner_labels = tf.cast(partner_labels, tf.float32)
89
+ if bce_target:
90
+ return tf.maximum(labels, partner_labels)
91
+ coefficient = broadcast_mix_coefficient(coefficient, labels)
92
+ return coefficient * labels + (1.0 - coefficient) * partner_labels
93
+
94
+
95
+ def mix_labels(
96
+ labels: tf.Tensor,
97
+ partner_labels: tf.Tensor,
98
+ coefficient: tf.Tensor,
99
+ *,
100
+ label_mode: LabelMixMode,
101
+ num_classes: int | None = None,
102
+ label_smoothing: float = 0.0,
103
+ bce_target: bool = False,
104
+ ) -> tf.Tensor:
105
+ labels = prepare_labels_for_mixing(
106
+ labels,
107
+ label_mode=label_mode,
108
+ num_classes=num_classes,
109
+ label_smoothing=label_smoothing,
110
+ )
111
+ partner_labels = prepare_labels_for_mixing(
112
+ partner_labels,
113
+ label_mode=label_mode,
114
+ num_classes=num_classes,
115
+ label_smoothing=label_smoothing,
116
+ )
117
+ return blend_prepared_labels(
118
+ labels, partner_labels, coefficient, bce_target=bce_target
119
+ )
120
+
121
+
122
+ __all__ = [
123
+ "LabelMixMode",
124
+ "apply_label_smoothing",
125
+ "blend_prepared_labels",
126
+ "broadcast_mix_coefficient",
127
+ "mix_labels",
128
+ "prepare_labels_for_mixing",
129
+ ]
@@ -0,0 +1,384 @@
1
+ import os
2
+ from functools import partial
3
+ from typing import Any, Dict, Literal, Optional, Union
4
+
5
+ import tensorflow as tf
6
+ from loguru import logger
7
+
8
+ from justdata.core.adapters import get_adapter
9
+ from justdata.core.finalization import finalize_dataset
10
+ from justdata.core.sources import get_source_loader
11
+
12
+
13
+ def _concatenate_tf_datasets(
14
+ dataset_list: list[tf.data.Dataset],
15
+ ) -> Optional[tf.data.Dataset]:
16
+ """Concatenates a list of tf.data.Dataset objects sequentially."""
17
+ if not dataset_list:
18
+ return None
19
+
20
+ concatenated_ds = dataset_list[0]
21
+ for i in range(1, len(dataset_list)):
22
+ concatenated_ds = concatenated_ds.concatenate(dataset_list[i])
23
+
24
+ return concatenated_ds
25
+
26
+
27
+ def fetch_ds(
28
+ dataset_names: list[str],
29
+ splits_info: Union[list[str], Dict[str, list[str]]],
30
+ data_dir: Union[None, str, os.PathLike] = None,
31
+ *,
32
+ source_filter_fn=None,
33
+ ) -> Optional[tf.data.Dataset]:
34
+ """
35
+ Loads and concatenates multiple TensorFlow Datasets and their splits.
36
+
37
+ This function performs a two-level concatenation:
38
+ 1. For each dataset name in `dataset_names`:
39
+ - If `splits_info` is a list, those splits are loaded.
40
+ - If `splits_info` is a dictionary, the splits for the current
41
+ dataset name are looked up in the dictionary. If not found or
42
+ empty, the dataset is skipped.
43
+ All successfully loaded splits for the current dataset are concatenated.
44
+ 2. The resulting datasets (one for each successfully processed name in
45
+ `dataset_names`, now containing its specified splits) are then
46
+ concatenated into a single, final `tf.data.Dataset`.
47
+
48
+ Args:
49
+ dataset_names: A list of dataset names to attempt to load
50
+ (e.g., ['cifar10', 'mnist']). The order is preserved.
51
+ splits_info: Defines the splits to load.
52
+ - If a List[str] (e.g., ['train', 'test']): these splits are
53
+ loaded for every dataset in `dataset_names`. If the list is
54
+ empty, no data will be loaded for any dataset.
55
+ - If a Dict[str, List[str]] (e.g.,
56
+ {'cifar10': ['train'], 'mnist': ['train', 'test']}):
57
+ For each dataset name in `dataset_names` that is also a key
58
+ in this dict, the corresponding list of splits is loaded.
59
+ Datasets in `dataset_names` not found as keys, or with an
60
+ empty list of splits in the dict, will be skipped.
61
+ data_dir: Optional path to the directory where datasets are stored or
62
+ will be downloaded. If None, defaults to
63
+ `tfds.builder.DEFAULT_DATA_DIR`.
64
+ source_filter_fn: Optional predicate applied to raw source records after
65
+ split concatenation and before adapter mapping. It may
66
+ reference only fields exposed by the source loader.
67
+
68
+ Returns:
69
+ A `tf.data.Dataset` instance representing the concatenation of all
70
+ specified and successfully loaded datasets and splits. Returns `None`
71
+ if `dataset_names` is empty, if `splits_info` leads to no loadable
72
+ splits for any dataset, or if all loading attempts fail.
73
+
74
+ Note:
75
+ - `tfds.load` is called with `as_supervised=False` and
76
+ `shuffle_files=False`. These parameters are fixed internally.
77
+ - If loading splits for a particular dataset name fails (e.g., due
78
+ to an invalid name or unavailable data), a warning is logged,
79
+ and that dataset is skipped. The function will attempt to proceed
80
+ with other dataset names.
81
+ """
82
+ if not dataset_names:
83
+ logger.info("Empty 'dataset_names' list provided. Nothing to load.")
84
+ return None
85
+
86
+ datasets_with_splits = []
87
+
88
+ for dataset_name in dataset_names:
89
+ splits_to_load: Optional[list[str]] = None
90
+
91
+ if isinstance(splits_info, list):
92
+ if not splits_info:
93
+ logger.warning(
94
+ f"Global 'splits_info' list is empty. "
95
+ f"Skipping dataset '{dataset_name}'."
96
+ )
97
+ continue
98
+ splits_to_load = splits_info
99
+ elif isinstance(splits_info, dict):
100
+ if dataset_name not in splits_info or not splits_info[dataset_name]:
101
+ logger.warning(
102
+ f"Dataset '{dataset_name}' not found in 'splits_info' "
103
+ f"dict, or its split list is empty. Skipping."
104
+ )
105
+ continue
106
+ splits_to_load = splits_info[dataset_name]
107
+ else:
108
+ logger.error(
109
+ "Invalid type for 'splits_info'. Must be List[str] or "
110
+ "Dict[str, List[str]]. Cannot proceed."
111
+ )
112
+ return None
113
+
114
+ try:
115
+ loaded_splits = get_source_loader(dataset_name)(
116
+ dataset_name,
117
+ splits_to_load,
118
+ data_dir,
119
+ )
120
+ except Exception as e:
121
+ logger.warning(
122
+ f"Warning: Failed to load splits for dataset '{dataset_name}'. "
123
+ f"Error: {e}. Skipping this dataset."
124
+ )
125
+ continue
126
+
127
+ if not loaded_splits:
128
+ logger.warning(
129
+ f"No splits data resulted for dataset '{dataset_name}' "
130
+ f"after attempting to load. Skipping."
131
+ )
132
+ continue
133
+
134
+ concatenated_splits = _concatenate_tf_datasets(loaded_splits)
135
+
136
+ if concatenated_splits:
137
+ if source_filter_fn is not None:
138
+ concatenated_splits = concatenated_splits.filter(source_filter_fn)
139
+ adapter = get_adapter(dataset_name)
140
+ concatenated_splits = concatenated_splits.map(
141
+ adapter, num_parallel_calls=tf.data.AUTOTUNE
142
+ )
143
+ datasets_with_splits.append(concatenated_splits)
144
+
145
+ if not datasets_with_splits:
146
+ logger.info(
147
+ "No datasets were successfully loaded and processed "
148
+ "based on the provided names and splits configuration."
149
+ )
150
+ return None
151
+
152
+ return _concatenate_tf_datasets(datasets_with_splits)
153
+
154
+
155
+ def load_ds(
156
+ dataset_names_arg: Union[str, list[str]],
157
+ splits_arg: Union[str, list[str], Dict[str, list[str]]],
158
+ dataset_type: str,
159
+ batch_size: int,
160
+ seed: int,
161
+ num_classes: Optional[int] = None,
162
+ pipeline: Optional[Any] = None,
163
+ preprocess_fn=None,
164
+ augment_fn=None,
165
+ late_augment_fn=None,
166
+ postprocess_fn=None,
167
+ shuffle_buffer: int = 10_000,
168
+ cache_dataset: bool = False,
169
+ cache_path: str = "",
170
+ cache_model_inputs: bool = False,
171
+ model_input_cache_path: str = "",
172
+ allow_train_model_input_cache: bool = False,
173
+ drop_remainder: bool = False,
174
+ data_dir: Union[None, str, os.PathLike] = None,
175
+ return_raw_ds: bool = False,
176
+ deterministic: bool = False,
177
+ as_numpy: bool = False,
178
+ metadata_mode: Literal["full", "numeric_only", "none"] | None = None,
179
+ sidecar_metadata_path: str | None = None,
180
+ filter_fn=None,
181
+ *,
182
+ source_filter_fn=None,
183
+ ):
184
+ """
185
+ Loads, preprocesses, and batches HuggingFace or TensorFlow Datasets.
186
+
187
+ Args:
188
+ dataset_names_arg: A single dataset name (str) or a list of dataset names.
189
+ splits_arg: Defines the splits to load.
190
+ - If str (e.g., "train"): This split is loaded for all datasets.
191
+ - If List[str] (e.g., ["train", "test"]): These splits are loaded
192
+ for all datasets.
193
+ - If Dict[str, List[str]] (e.g., {"cifar10": ["train"]}): Specifies
194
+ splits per dataset. See `fetch_ds` for details.
195
+ dataset_type: Type of dataset, typically "train", "validation", or "test".
196
+ Affects caching, shuffling, and augmentation.
197
+ batch_size: The batch size for the output dataset.
198
+ seed: Random seed for shuffling and augmentations.
199
+ num_classes: Number of classes (needed for one-hot encoding/mixup).
200
+ pipeline: A `DataPipeline` object from `justdata.core.registry`. If provided,
201
+ it builds the necessary functions based on `dataset_type`.
202
+ preprocess_fn: Preprocessing function (fallback if `pipeline` is None).
203
+ augment_fn: Augmentation function (fallback if `pipeline` is None).
204
+ late_augment_fn: Late augmentation function (fallback if `pipeline` is None).
205
+ postprocess_fn: Postprocessing function (fallback if `pipeline` is None).
206
+ shuffle_buffer: Size of the shuffle buffer.
207
+ cache_dataset: Whether to cache the dataset after preprocessing. Disabled
208
+ by default so omitted caching remains streaming-friendly.
209
+ cache_path: Filesystem path for the preprocessing cache. When
210
+ ``cache_dataset=True``, an empty path uses memory and a nonempty path
211
+ uses the filesystem.
212
+ cache_model_inputs: Whether to cache samples after deterministic
213
+ postprocessing, before batching. Use ``model_input_cache_path`` for
214
+ local SSD caching of resized/model-ready tensors.
215
+ model_input_cache_path: Filesystem path for ``cache_model_inputs``.
216
+ If a filename is not provided, model inputs will be cached in memory.
217
+ allow_train_model_input_cache: Required opt-in for model-input caching
218
+ with training or evaluation augmentation. Only use this when
219
+ materializing the first sampled augmented views is intentional.
220
+ drop_remainder: Choose to drop or pad batches without the correct size,
221
+ data_dir: Optional path to the TFDS data directory.
222
+ return_raw_ds: If True, returns the dataset immediately after
223
+ preprocessing (and caching) but BEFORE standard
224
+ augmentation, postprocessing, or batching.
225
+ Returns (ds, tools_dict) where tools_dict contains
226
+ ``finalize_fn``, ``postprocess_fn``, and ``rng``.
227
+ deterministic: If false, sacrifices determinism for performance.
228
+ as_numpy: If True, returns an iterator yielding NumPy arrays.
229
+ metadata_mode: Controls metadata in output batches. When omitted, uses
230
+ ``pipeline.kwargs["metadata_mode"]`` if present, then
231
+ defaults to ``full``. ``numeric_only`` drops strings
232
+ from batches and ``none`` removes metadata.
233
+ sidecar_metadata_path: JSONL path for string metadata when
234
+ ``metadata_mode="numeric_only"``.
235
+ filter_fn: Optional predicate applied after preprocessing and caching,
236
+ before augmentation, postprocessing, and batching.
237
+ source_filter_fn: Optional predicate applied to raw source records before
238
+ adapter mapping. Unlike ``filter_fn``, it may reference
239
+ only fields exposed by the source loader.
240
+ Returns:
241
+ ``(dataset, n_batches)`` where ``dataset`` is a batched
242
+ ``tf.data.Dataset`` or NumPy iterator and ``n_batches`` is an integer
243
+ when cardinality is known, otherwise ``None``. With
244
+ ``return_raw_ds=True``, returns ``(dataset, tools_dict)`` instead.
245
+
246
+ Raises:
247
+ ValueError: If dataset loading fails and `fetch_ds` returns None.
248
+ TypeError: If `dataset_names_arg` or `splits_arg` have invalid types.
249
+ """
250
+ rng = tf.random.Generator.from_seed(seed)
251
+ if metadata_mode is None:
252
+ metadata_mode = (
253
+ pipeline.kwargs.get("metadata_mode", "full")
254
+ if pipeline is not None
255
+ else "full"
256
+ )
257
+ if metadata_mode not in {"full", "numeric_only", "none"}:
258
+ raise ValueError(
259
+ "metadata_mode must be one of 'full', 'numeric_only', or 'none'."
260
+ )
261
+
262
+ dataset_names: list[str]
263
+ if isinstance(dataset_names_arg, str):
264
+ dataset_names = [dataset_names_arg]
265
+ elif isinstance(dataset_names_arg, list):
266
+ dataset_names = dataset_names_arg
267
+ else:
268
+ raise TypeError("dataset_names_arg must be a string or a list of strings.")
269
+
270
+ splits: Union[list[str], Dict[str, list[str]]]
271
+ if isinstance(splits_arg, str):
272
+ splits = [splits_arg]
273
+ elif isinstance(splits_arg, list):
274
+ splits = splits_arg
275
+ elif isinstance(splits_arg, dict):
276
+ splits = splits_arg
277
+ else:
278
+ raise TypeError(
279
+ "splits_arg must be a string, a list of strings, or a "
280
+ "dictionary mapping dataset names to lists of strings."
281
+ )
282
+
283
+ is_training = dataset_type == "train"
284
+ augment_eval = bool(
285
+ pipeline is not None and pipeline.kwargs.get("augment_eval", False)
286
+ )
287
+ apply_augmentation = is_training or augment_eval
288
+ if cache_model_inputs and apply_augmentation and not allow_train_model_input_cache:
289
+ raise ValueError(
290
+ "cache_model_inputs with training or evaluation augmentation requires "
291
+ "allow_train_model_input_cache=True."
292
+ )
293
+ if cache_dataset and cache_model_inputs and cache_path and model_input_cache_path:
294
+ preprocess_cache_path = os.path.abspath(os.fspath(cache_path))
295
+ model_cache_path = os.path.abspath(os.fspath(model_input_cache_path))
296
+ if preprocess_cache_path == model_cache_path:
297
+ raise ValueError(
298
+ "cache_path and model_input_cache_path must be different when "
299
+ "both cache stages are enabled."
300
+ )
301
+
302
+ if source_filter_fn is None:
303
+ ds = fetch_ds(dataset_names, splits, data_dir)
304
+ else:
305
+ ds = fetch_ds(
306
+ dataset_names,
307
+ splits,
308
+ data_dir,
309
+ source_filter_fn=source_filter_fn,
310
+ )
311
+
312
+ if ds is None:
313
+ raise ValueError(
314
+ "Dataset loading failed (fetch_ds returned None). "
315
+ "Check logs for details on dataset names, splits, or data issues."
316
+ )
317
+
318
+ if pipeline is not None:
319
+ preprocess_fn, augment_fn, late_augment_fn, postprocess_fn = pipeline.build(
320
+ is_training=is_training
321
+ )
322
+
323
+ def seeded_augment(sample):
324
+ seed = rng.make_seeds(1)[:, 0]
325
+ return augment_fn(sample, seed=seed)
326
+
327
+ ds = ds.map(
328
+ preprocess_fn,
329
+ num_parallel_calls=tf.data.AUTOTUNE,
330
+ deterministic=deterministic if is_training else None,
331
+ )
332
+
333
+ # An empty path caches in memory; a nonempty path caches on the filesystem.
334
+ # Large datasets should use an explicit disk path or remain uncached.
335
+ if cache_dataset:
336
+ ds = ds.cache(cache_path)
337
+ else:
338
+ logger.info(
339
+ f"Caching disabled for '{dataset_type}' dataset. "
340
+ "Samples will be re-read each epoch (streaming-friendly)."
341
+ )
342
+
343
+ if filter_fn is not None:
344
+ ds = ds.filter(filter_fn)
345
+
346
+ finalize_fn = partial(
347
+ finalize_dataset,
348
+ postprocess_fn=postprocess_fn,
349
+ num_classes=num_classes,
350
+ batch_size=batch_size,
351
+ metadata_mode=metadata_mode,
352
+ sidecar_metadata_path=sidecar_metadata_path,
353
+ cache_model_inputs=cache_model_inputs,
354
+ model_input_cache_path=model_input_cache_path,
355
+ drop_remainder=drop_remainder,
356
+ is_training=is_training,
357
+ apply_late_augment=apply_augmentation,
358
+ late_augment_fn=late_augment_fn,
359
+ rng=rng,
360
+ deterministic=deterministic,
361
+ shuffle_buffer=shuffle_buffer if is_training else None,
362
+ shuffle_seed=seed,
363
+ as_numpy=as_numpy,
364
+ )
365
+
366
+ if return_raw_ds:
367
+ # Return necessary components to build custom pipelines
368
+ return (
369
+ ds,
370
+ {
371
+ "finalize_fn": finalize_fn,
372
+ "postprocess_fn": postprocess_fn,
373
+ "rng": rng,
374
+ },
375
+ )
376
+
377
+ if apply_augmentation:
378
+ ds = ds.map(
379
+ seeded_augment,
380
+ num_parallel_calls=1 if deterministic else tf.data.AUTOTUNE,
381
+ deterministic=deterministic,
382
+ )
383
+
384
+ return finalize_fn(ds)