justdata 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. justdata/__init__.py +3 -0
  2. justdata/acoustic/__init__.py +381 -0
  3. justdata/acoustic/adapters.py +329 -0
  4. justdata/acoustic/augment/__init__.py +119 -0
  5. justdata/acoustic/augment/batch.py +813 -0
  6. justdata/acoustic/augment/configs.py +204 -0
  7. justdata/acoustic/augment/patchout.py +331 -0
  8. justdata/acoustic/augment/spectrogram.py +806 -0
  9. justdata/acoustic/augment/waveform.py +879 -0
  10. justdata/acoustic/batching.py +27 -0
  11. justdata/acoustic/channel.py +53 -0
  12. justdata/acoustic/compat/__init__.py +95 -0
  13. justdata/acoustic/compat/ast.py +626 -0
  14. justdata/acoustic/compat/ced.py +152 -0
  15. justdata/acoustic/compat/efficientat.py +243 -0
  16. justdata/acoustic/compat/panns.py +115 -0
  17. justdata/acoustic/compat/passt.py +191 -0
  18. justdata/acoustic/configs.py +519 -0
  19. justdata/acoustic/corruptions/__init__.py +33 -0
  20. justdata/acoustic/corruptions/codec.py +92 -0
  21. justdata/acoustic/corruptions/datasets.py +155 -0
  22. justdata/acoustic/corruptions/device.py +39 -0
  23. justdata/acoustic/corruptions/dynamics.py +55 -0
  24. justdata/acoustic/corruptions/filters.py +105 -0
  25. justdata/acoustic/corruptions/noise.py +173 -0
  26. justdata/acoustic/corruptions/registry.py +230 -0
  27. justdata/acoustic/corruptions/reverb.py +49 -0
  28. justdata/acoustic/corruptions/timefreq.py +127 -0
  29. justdata/acoustic/datasets.py +52 -0
  30. justdata/acoustic/dcase2025.py +814 -0
  31. justdata/acoustic/decoding.py +73 -0
  32. justdata/acoustic/eval_views.py +54 -0
  33. justdata/acoustic/evals.py +71 -0
  34. justdata/acoustic/filters.py +18 -0
  35. justdata/acoustic/frontends/__init__.py +37 -0
  36. justdata/acoustic/frontends/ast.py +116 -0
  37. justdata/acoustic/frontends/kaldi.py +34 -0
  38. justdata/acoustic/frontends/log.py +94 -0
  39. justdata/acoustic/frontends/mel.py +173 -0
  40. justdata/acoustic/frontends/mfcc.py +26 -0
  41. justdata/acoustic/frontends/pcen.py +26 -0
  42. justdata/acoustic/frontends/stft.py +174 -0
  43. justdata/acoustic/jax.py +23 -0
  44. justdata/acoustic/labels.py +169 -0
  45. justdata/acoustic/layouts.py +86 -0
  46. justdata/acoustic/metadata.py +152 -0
  47. justdata/acoustic/normalization.py +196 -0
  48. justdata/acoustic/pipelines.py +112 -0
  49. justdata/acoustic/postprocessing.py +328 -0
  50. justdata/acoustic/preprocessing.py +175 -0
  51. justdata/acoustic/presets.py +257 -0
  52. justdata/acoustic/registry.py +381 -0
  53. justdata/acoustic/resampling.py +218 -0
  54. justdata/acoustic/schema.py +51 -0
  55. justdata/acoustic/segment.py +369 -0
  56. justdata/acoustic/sources.py +503 -0
  57. justdata/acoustic/stages.py +59 -0
  58. justdata/acoustic/stats.py +50 -0
  59. justdata/acoustic/tasks.py +375 -0
  60. justdata/audio/__init__.py +20 -0
  61. justdata/core/__init__.py +74 -0
  62. justdata/core/adapters.py +46 -0
  63. justdata/core/augmentations.py +34 -0
  64. justdata/core/filters.py +94 -0
  65. justdata/core/finalization.py +173 -0
  66. justdata/core/label_mixing.py +129 -0
  67. justdata/core/loader.py +384 -0
  68. justdata/core/metadata.py +250 -0
  69. justdata/core/presets.py +155 -0
  70. justdata/core/registry.py +229 -0
  71. justdata/core/sources.py +87 -0
  72. justdata/core/stats.py +227 -0
  73. justdata/corruptions.py +52 -0
  74. justdata/py.typed +0 -0
  75. justdata/vision/__init__.py +17 -0
  76. justdata/vision/augmentations/__init__.py +31 -0
  77. justdata/vision/augmentations/auto.py +480 -0
  78. justdata/vision/augmentations/color.py +163 -0
  79. justdata/vision/augmentations/composed.py +156 -0
  80. justdata/vision/augmentations/geometric.py +302 -0
  81. justdata/vision/augmentations/mixing.py +232 -0
  82. justdata/vision/augmentations/registry.py +142 -0
  83. justdata/vision/corruptions/__init__.py +16 -0
  84. justdata/vision/corruptions/blur.py +65 -0
  85. justdata/vision/corruptions/digital.py +26 -0
  86. justdata/vision/corruptions/noise.py +20 -0
  87. justdata/vision/corruptions/registry.py +113 -0
  88. justdata/vision/corruptions/weather.py +100 -0
  89. justdata/vision/datasets.py +34 -0
  90. justdata/vision/encodings/__init__.py +1 -0
  91. justdata/vision/encodings/heatmaps.py +196 -0
  92. justdata/vision/minic.py +69 -0
  93. justdata/vision/pipelines.py +62 -0
  94. justdata/vision/presets.py +509 -0
  95. justdata/vision/sources.py +1080 -0
  96. justdata/vision/stages.py +271 -0
  97. justdata/vision/tasks/__init__.py +1 -0
  98. justdata/vision/tasks/classification.py +321 -0
  99. justdata/vision/tasks/depth.py +25 -0
  100. justdata/vision/tasks/segmentation.py +194 -0
  101. justdata/vision/transforms.py +113 -0
  102. justdata/vision/utils.py +741 -0
  103. justdata-1.0.0.dist-info/METADATA +860 -0
  104. justdata-1.0.0.dist-info/RECORD +106 -0
  105. justdata-1.0.0.dist-info/WHEEL +4 -0
  106. justdata-1.0.0.dist-info/licenses/LICENSE +23 -0
@@ -0,0 +1,375 @@
1
+ from collections.abc import Mapping
2
+
3
+ import tensorflow as tf
4
+
5
+ from justdata.acoustic.registry import (
6
+ list_audio_spectrogram_augments,
7
+ list_audio_waveform_augments,
8
+ register_audio_batch_augment,
9
+ register_audio_corruption,
10
+ register_audio_decoder,
11
+ register_audio_postprocessor,
12
+ register_audio_resampler,
13
+ register_audio_spectrogram_augment,
14
+ register_audio_waveform_augment,
15
+ )
16
+ from justdata.acoustic.configs import (
17
+ AudioPreprocessConfig,
18
+ LabelTransformConfig,
19
+ SegmentStrategyConfig,
20
+ )
21
+ from justdata.acoustic.labels import transform_label
22
+ from justdata.acoustic.postprocessing import make_model_input_stage, preset_info
23
+ from justdata.acoustic.preprocessing import make_preprocessing as _make_preprocessing
24
+ from justdata.acoustic.schema import FEATURES, LABEL, METADATA
25
+ from justdata.acoustic.stages import make_segment_stage
26
+
27
+
28
+ def _identity_sample(sample, *args, **kwargs):
29
+ return sample
30
+
31
+
32
+ def _identity_batch(batch, num_classes=None, seed=None, **kwargs):
33
+ return batch
34
+
35
+
36
+ def _seed_tensor(seed):
37
+ if seed is None:
38
+ return None
39
+ seed = tf.cast(tf.convert_to_tensor(seed), tf.int32)
40
+ if seed.shape.rank == 0:
41
+ return tf.stack([seed, tf.constant(0, dtype=tf.int32)])
42
+ if seed.shape.rank == 1 and seed.shape[0] == 1:
43
+ return tf.stack([seed[0], tf.constant(0, dtype=tf.int32)])
44
+ return seed[:2]
45
+
46
+
47
+ @register_audio_decoder("identity")
48
+ @register_audio_resampler("identity")
49
+ @register_audio_waveform_augment("none")
50
+ @register_audio_spectrogram_augment("none")
51
+ @register_audio_batch_augment("none")
52
+ @register_audio_postprocessor("identity")
53
+ @register_audio_corruption("identity")
54
+ def identity(sample, *args, **kwargs):
55
+ return sample
56
+
57
+
58
+ def make_preprocessing(config: AudioPreprocessConfig | dict | None = None, **kwargs):
59
+ if config is None and not kwargs:
60
+ return _identity_sample
61
+ if config is None:
62
+ config = AudioPreprocessConfig(**kwargs)
63
+ else:
64
+ config = AudioPreprocessConfig.from_dict(config)
65
+ return _make_preprocessing(config)
66
+
67
+
68
+ def make_augmentations(
69
+ segment_config: SegmentStrategyConfig | dict | None = None,
70
+ waveform_augmentations=None,
71
+ spectrogram_augmentations=None,
72
+ train_augment: dict | None = None,
73
+ is_training: bool = True,
74
+ augment_eval: bool = False,
75
+ spectrogram_key: str = FEATURES,
76
+ spectrogram_layout: str | None = None,
77
+ **kwargs,
78
+ ):
79
+ import justdata.acoustic.augment # noqa: F401
80
+ from justdata.acoustic.augment import make_waveform_augmentation_stage
81
+
82
+ if spectrogram_augmentations is not None:
83
+ raise ValueError(
84
+ "spectrogram_augmentations require post-frontend features; "
85
+ "pass them to make_late_augmentations instead"
86
+ )
87
+
88
+ if waveform_augmentations is None and train_augment:
89
+ if "waveform" in train_augment:
90
+ waveform_augmentations = train_augment["waveform"]
91
+ else:
92
+ known_waveform_augments = set(list_audio_waveform_augments()) - {"none"}
93
+ waveform_augmentations = {
94
+ key: value
95
+ for key, value in train_augment.items()
96
+ if key in known_waveform_augments
97
+ }
98
+
99
+ stages = []
100
+ if segment_config is not None:
101
+ stages.append(make_segment_stage(segment_config, is_training=is_training))
102
+ if waveform_augmentations:
103
+ stages.append(
104
+ make_waveform_augmentation_stage(
105
+ waveform_augmentations,
106
+ is_training=is_training,
107
+ augment_eval=augment_eval,
108
+ )
109
+ )
110
+ if not stages:
111
+ return _identity_sample
112
+
113
+ def augment(sample, seed=None):
114
+ result = sample
115
+ if seed is None:
116
+ stage_seeds = [None] * len(stages)
117
+ else:
118
+ stage_seeds = tf.unstack(tf.random.split(_seed_tensor(seed), len(stages)))
119
+ for stage, stage_seed in zip(stages, stage_seeds):
120
+ result = stage(result, seed=stage_seed)
121
+ return result
122
+
123
+ return augment
124
+
125
+
126
+ def make_late_augmentations(**kwargs):
127
+ import justdata.acoustic.augment # noqa: F401
128
+ from justdata.acoustic.augment import (
129
+ make_batch_augmentation_stage,
130
+ make_spectrogram_augmentation_stage,
131
+ normalize_spectrogram_augment_specs,
132
+ )
133
+
134
+ known_batch_augments = {
135
+ "batch_mixstyle",
136
+ "cutmix",
137
+ "cutmix_spec",
138
+ "mixstyle",
139
+ "mixup",
140
+ "wavmix",
141
+ }
142
+ batch_augmentations = kwargs.pop("batch_augmentations", None)
143
+ train_augment = kwargs.pop("train_augment", None)
144
+ is_training = kwargs.pop("is_training", True)
145
+ augment_eval = kwargs.pop("augment_eval", False)
146
+ input_key = kwargs.pop("input_key", None)
147
+ input_kind = kwargs.pop("input_kind", None)
148
+ label_mode = kwargs.pop("label_mode", None)
149
+ label_transform = kwargs.pop("label_transform", None)
150
+ spectrogram_augmentations = kwargs.pop("spectrogram_augmentations", None)
151
+ spectrogram_key = kwargs.pop("spectrogram_key", FEATURES)
152
+ spectrogram_layout = kwargs.pop("spectrogram_layout", None)
153
+ model_layout = kwargs.pop("model_layout", None)
154
+
155
+ if spectrogram_augmentations is None and train_augment:
156
+ if "spectrogram" in train_augment:
157
+ spectrogram_augmentations = train_augment["spectrogram"]
158
+ else:
159
+ known_spectrogram_augments = set(list_audio_spectrogram_augments()) - {
160
+ "none"
161
+ }
162
+ spectrogram_augmentations = {
163
+ ("passt_patchout" if key == "patchout" else key): value
164
+ for key, value in train_augment.items()
165
+ if (
166
+ key in known_spectrogram_augments
167
+ and key not in known_batch_augments
168
+ )
169
+ or key == "patchout"
170
+ }
171
+
172
+ if batch_augmentations is None and train_augment:
173
+ if "batch" in train_augment:
174
+ batch_augmentations = train_augment["batch"]
175
+ else:
176
+ batch_augmentations = {
177
+ key: value
178
+ for key, value in train_augment.items()
179
+ if key in known_batch_augments
180
+ }
181
+
182
+ if batch_augmentations is None:
183
+ batch_augmentations = {
184
+ key: kwargs.pop(key) for key in tuple(kwargs) if key in known_batch_augments
185
+ }
186
+
187
+ spectrogram_specs = normalize_spectrogram_augment_specs(spectrogram_augmentations)
188
+ if not spectrogram_specs and not batch_augmentations:
189
+ return _identity_batch
190
+
191
+ sample_layouts = {"tf", "tfc", "cft"}
192
+ batch_to_sample_layout = {"btf": "tf", "btfc": "tfc", "bcft": "cft"}
193
+ layout_source = spectrogram_layout or model_layout
194
+ if not spectrogram_specs:
195
+ sample_spectrogram_layout = None
196
+ elif layout_source in sample_layouts or layout_source is None:
197
+ sample_spectrogram_layout = layout_source
198
+ elif layout_source in batch_to_sample_layout:
199
+ sample_spectrogram_layout = batch_to_sample_layout[layout_source]
200
+ else:
201
+ raise ValueError(
202
+ "Spectrogram augmentation layout must be one of 'tf', 'tfc', "
203
+ "'cft', 'btf', 'btfc', or 'bcft'"
204
+ )
205
+
206
+ batch_spectrogram_layout = model_layout or spectrogram_layout
207
+
208
+ label_config = (
209
+ LabelTransformConfig.from_dict(label_transform)
210
+ if label_transform is not None
211
+ else None
212
+ )
213
+ label_transform_dict = label_config.to_dict() if label_config is not None else None
214
+ spectrogram_stage = make_spectrogram_augmentation_stage(
215
+ spectrogram_specs,
216
+ is_training=is_training,
217
+ augment_eval=augment_eval,
218
+ feature_key=spectrogram_key,
219
+ layout=sample_spectrogram_layout,
220
+ )
221
+ batch_stage = make_batch_augmentation_stage(
222
+ batch_augmentations,
223
+ is_training=is_training,
224
+ augment_eval=augment_eval,
225
+ input_key=input_key,
226
+ input_kind=input_kind,
227
+ label_mode=label_mode,
228
+ label_transform=label_transform_dict,
229
+ spectrogram_layout=batch_spectrogram_layout,
230
+ )
231
+
232
+ def mapped_output_signature(batch):
233
+ signature = tf.nest.map_structure(
234
+ lambda value: tf.TensorSpec(value.shape[1:], value.dtype), batch
235
+ )
236
+ feature = batch[spectrogram_key]
237
+ signature[spectrogram_key] = tf.TensorSpec(
238
+ [None] * (feature.shape.rank - 1), feature.dtype
239
+ )
240
+
241
+ def patchout_debug_enabled(spec):
242
+ config = spec.get("config")
243
+ config_debug = (
244
+ config.get("debug", False) if isinstance(config, Mapping) else False
245
+ )
246
+ return spec["name"] == "passt_patchout" and bool(
247
+ spec.get("debug", config_debug)
248
+ )
249
+
250
+ patchout_debug = any(map(patchout_debug_enabled, spectrogram_specs))
251
+ if patchout_debug:
252
+ metadata_signature = dict(signature.get(METADATA, {}))
253
+ metadata_signature["patchout"] = {
254
+ "structured_frequency": tf.TensorSpec([None], tf.int32),
255
+ "structured_time": tf.TensorSpec([None], tf.int32),
256
+ "unstructured": tf.TensorSpec([None], tf.int32),
257
+ }
258
+ signature[METADATA] = metadata_signature
259
+ return signature
260
+
261
+ def late_augmentations(batch, num_classes=None, seed=None, **_kwargs):
262
+ effective_num_classes = num_classes
263
+ if effective_num_classes is None and label_config is not None:
264
+ effective_num_classes = label_config.num_classes
265
+
266
+ result = batch
267
+ if spectrogram_specs:
268
+ if spectrogram_key not in batch:
269
+ raise ValueError(
270
+ f"Spectrogram augmentations require batch key {spectrogram_key!r}"
271
+ )
272
+ base_seed = _seed_tensor(seed)
273
+ if base_seed is None:
274
+ base_seed = tf.constant([0, 0], dtype=tf.int32)
275
+ spectrogram_seed, batch_seed = tf.unstack(tf.random.split(base_seed, 2))
276
+ row_seeds = tf.random.split(
277
+ spectrogram_seed, tf.shape(batch[spectrogram_key])[0]
278
+ )
279
+ result = tf.map_fn(
280
+ lambda values: spectrogram_stage(values[0], seed=values[1]),
281
+ (batch, row_seeds),
282
+ fn_output_signature=mapped_output_signature(batch),
283
+ )
284
+ else:
285
+ batch_seed = seed
286
+
287
+ if batch_augmentations:
288
+ result = batch_stage(
289
+ result,
290
+ num_classes=effective_num_classes,
291
+ seed=batch_seed,
292
+ )
293
+ return result
294
+
295
+ return late_augmentations
296
+
297
+
298
+ def make_postprocessing(
299
+ segment_config: SegmentStrategyConfig | dict | None = None,
300
+ is_training: bool = False,
301
+ frontend: dict | None = None,
302
+ layout: str | None = None,
303
+ dtype: str = "float32",
304
+ output_key: str | None = None,
305
+ static_shape: tuple[int | None, ...] | None = None,
306
+ input_duration: float | None = None,
307
+ target_sample_rate: int | None = None,
308
+ preprocess: dict | None = None,
309
+ label_transform: LabelTransformConfig | dict | None = None,
310
+ **kwargs,
311
+ ):
312
+ original_segment_config = segment_config
313
+ if segment_config is None:
314
+ segment_keys = set(SegmentStrategyConfig.__dataclass_fields__)
315
+ if segment_keys.intersection(kwargs):
316
+ segment_config = {
317
+ key: kwargs.pop(key) for key in tuple(kwargs) if key in segment_keys
318
+ }
319
+ original_segment_config = segment_config
320
+ elif frontend is None:
321
+ return _identity_sample
322
+
323
+ stages = []
324
+ if segment_config is not None:
325
+ stages.append(make_segment_stage(segment_config, is_training=is_training))
326
+
327
+ if frontend is not None:
328
+ if layout is None:
329
+ raise ValueError("Acoustic frontend postprocessing requires a layout")
330
+ shape_preset = preset_info(
331
+ frontend=frontend,
332
+ layout=layout,
333
+ dtype=dtype,
334
+ output_key=output_key,
335
+ static_shape=static_shape,
336
+ input_duration=input_duration,
337
+ target_sample_rate=target_sample_rate,
338
+ preprocess=preprocess,
339
+ segment=original_segment_config,
340
+ )
341
+ stages.append(
342
+ make_model_input_stage(
343
+ frontend,
344
+ layout=layout,
345
+ dtype=dtype,
346
+ output_key=output_key,
347
+ preset=shape_preset,
348
+ )
349
+ )
350
+
351
+ label_config = (
352
+ LabelTransformConfig.from_dict(label_transform)
353
+ if label_transform is not None
354
+ else None
355
+ )
356
+
357
+ if not stages and label_config is None:
358
+ return _identity_sample
359
+
360
+ def postprocessing(sample, num_classes=None):
361
+ result = sample
362
+ for stage in stages:
363
+ result = stage(result)
364
+ if label_config is not None and LABEL in result:
365
+ label, metadata = transform_label(
366
+ result[LABEL],
367
+ label_config,
368
+ metadata=result.get(METADATA),
369
+ )
370
+ result = dict(result)
371
+ result[LABEL] = label
372
+ result[METADATA] = metadata
373
+ return result
374
+
375
+ return postprocessing
@@ -0,0 +1,20 @@
1
+ import sys
2
+ from types import ModuleType
3
+
4
+ from justdata import acoustic as _acoustic
5
+ from justdata.acoustic import * # noqa: F403
6
+
7
+
8
+ for _name in _acoustic.__all__:
9
+ _value = getattr(_acoustic, _name)
10
+ if isinstance(_value, ModuleType):
11
+ sys.modules.setdefault(f"{__name__}.{_name}", _value)
12
+
13
+ _acoustic_prefix = f"{_acoustic.__name__}."
14
+ for _canonical_name, _module in tuple(sys.modules.items()):
15
+ if _canonical_name.startswith(_acoustic_prefix):
16
+ _alias_name = f"{__name__}.{_canonical_name.removeprefix(_acoustic_prefix)}"
17
+ sys.modules.setdefault(_alias_name, _module)
18
+
19
+
20
+ __all__ = _acoustic.__all__
@@ -0,0 +1,74 @@
1
+ from justdata.core.adapters import get_adapter, register_adapter
2
+ from justdata.core.filters import (
3
+ filter_by_metadata,
4
+ groupby_metadata,
5
+ metadata_filter_predicate,
6
+ )
7
+ from justdata.core.finalization import finalize_dataset
8
+ from justdata.core.loader import fetch_ds, load_ds
9
+ from justdata.core.metadata import (
10
+ MetadataSidecar,
11
+ apply_metadata_mode,
12
+ numeric_metadata,
13
+ stable_int64_hash,
14
+ )
15
+ from justdata.core.presets import (
16
+ ResolvedPreset,
17
+ get_resolved_preset,
18
+ get_dataset_presets,
19
+ merge_with_presets,
20
+ register_preset,
21
+ )
22
+ from justdata.core.registry import (
23
+ DataPipeline,
24
+ DatasetInfo,
25
+ get_dataset_info,
26
+ get_pipeline,
27
+ get_pipeline_for_dataset,
28
+ get_task_for_dataset,
29
+ has_pipeline,
30
+ list_pipelines,
31
+ register_dataset,
32
+ register_pipeline,
33
+ )
34
+ from justdata.core.sources import (
35
+ get_source_loader,
36
+ register_default_source_loader,
37
+ register_source_loader,
38
+ )
39
+ from justdata.core.stats import compute_feature_stats, make_stats_iterator
40
+
41
+ __all__ = [
42
+ "DataPipeline",
43
+ "DatasetInfo",
44
+ "MetadataSidecar",
45
+ "ResolvedPreset",
46
+ "apply_metadata_mode",
47
+ "compute_feature_stats",
48
+ "filter_by_metadata",
49
+ "finalize_dataset",
50
+ "fetch_ds",
51
+ "get_adapter",
52
+ "get_dataset_info",
53
+ "get_dataset_presets",
54
+ "get_pipeline",
55
+ "get_pipeline_for_dataset",
56
+ "get_resolved_preset",
57
+ "get_source_loader",
58
+ "get_task_for_dataset",
59
+ "groupby_metadata",
60
+ "has_pipeline",
61
+ "list_pipelines",
62
+ "load_ds",
63
+ "make_stats_iterator",
64
+ "merge_with_presets",
65
+ "metadata_filter_predicate",
66
+ "numeric_metadata",
67
+ "register_adapter",
68
+ "register_dataset",
69
+ "register_default_source_loader",
70
+ "register_pipeline",
71
+ "register_preset",
72
+ "register_source_loader",
73
+ "stable_int64_hash",
74
+ ]
@@ -0,0 +1,46 @@
1
+ import threading
2
+ from typing import Any, Dict, Protocol
3
+
4
+
5
+ class DatasetAdapter(Protocol):
6
+ """Maps raw dataset samples to a canonical dict schema."""
7
+
8
+ def __call__(self, sample: Dict[str, Any]) -> Dict[str, Any]: ...
9
+
10
+
11
+ _ADAPTERS: Dict[str, DatasetAdapter] = {}
12
+ _ADAPTER_LOCK = threading.Lock()
13
+
14
+
15
+ def register_adapter(dataset_name: str):
16
+ def decorator(fn: DatasetAdapter):
17
+ with _ADAPTER_LOCK:
18
+ if dataset_name in _ADAPTERS:
19
+ raise ValueError(
20
+ f"Adapter for '{dataset_name}' already registered by "
21
+ f"{_ADAPTERS[dataset_name].__module__}.{_ADAPTERS[dataset_name].__qualname__}"
22
+ )
23
+ _ADAPTERS[dataset_name] = fn
24
+ return fn
25
+
26
+ return decorator
27
+
28
+
29
+ def get_adapter(dataset_name: str) -> DatasetAdapter:
30
+ with _ADAPTER_LOCK:
31
+ exact = _ADAPTERS.get(dataset_name)
32
+ entries = tuple(_ADAPTERS.items())
33
+
34
+ if exact is not None:
35
+ return exact
36
+
37
+ for key, adapter in sorted(entries, key=lambda kv: len(kv[0]), reverse=True):
38
+ if dataset_name.startswith(key):
39
+ return adapter
40
+
41
+ return _default_adapter
42
+
43
+
44
+ def _default_adapter(sample: dict) -> dict:
45
+ """Identity — assumes sample already has the expected keys."""
46
+ return sample
@@ -0,0 +1,34 @@
1
+ from __future__ import annotations
2
+
3
+ from dataclasses import dataclass
4
+ from typing import Literal
5
+
6
+
7
+ AugmentationDomain = Literal["waveform", "spectrogram", "image", "batch"]
8
+
9
+
10
+ @dataclass(frozen=True)
11
+ class AugmentationMetadata:
12
+ name: str
13
+ domain: AugmentationDomain
14
+ is_training_only: bool
15
+ requires_labels: bool
16
+
17
+
18
+ def attach_augmentation_metadata(fn, metadata: AugmentationMetadata):
19
+ registrations = tuple(getattr(fn, "augmentation_metadata", ()))
20
+ if metadata not in registrations:
21
+ registrations = registrations + (metadata,)
22
+ fn.augmentation_metadata = registrations
23
+ fn.name = metadata.name
24
+ fn.domain = metadata.domain
25
+ fn.is_training_only = metadata.is_training_only
26
+ fn.requires_labels = metadata.requires_labels
27
+ return fn
28
+
29
+
30
+ __all__ = [
31
+ "AugmentationDomain",
32
+ "AugmentationMetadata",
33
+ "attach_augmentation_metadata",
34
+ ]
@@ -0,0 +1,94 @@
1
+ from __future__ import annotations
2
+
3
+ from collections.abc import Iterable
4
+ from typing import Any
5
+
6
+ import tensorflow as tf
7
+
8
+ from justdata.core.metadata import get_metadata_value, python_value
9
+
10
+
11
+ def _criterion_parts(name: str) -> tuple[str, bool]:
12
+ if name.endswith("_in"):
13
+ return name[:-3], True
14
+ return name, False
15
+
16
+
17
+ def _as_option_tensor(value: Any, dtype: tf.DType) -> tf.Tensor:
18
+ if dtype == tf.string and not isinstance(value, (bytes, str)):
19
+ value = str(value)
20
+ return tf.convert_to_tensor(value, dtype=dtype)
21
+
22
+
23
+ def _matches(value: Any, expected: Any, *, membership: bool) -> tf.Tensor:
24
+ value = tf.convert_to_tensor(value)
25
+ if membership:
26
+ options = [_as_option_tensor(item, value.dtype) for item in expected]
27
+ if not options:
28
+ return tf.constant(False)
29
+ return tf.reduce_any(tf.equal(value, tf.stack(options)))
30
+ return tf.equal(value, _as_option_tensor(expected, value.dtype))
31
+
32
+
33
+ def metadata_filter_predicate(**criteria: Any):
34
+ def predicate(sample: dict) -> tf.Tensor:
35
+ checks = []
36
+ for criterion, expected in criteria.items():
37
+ key, membership = _criterion_parts(criterion)
38
+ value = get_metadata_value(sample, key)
39
+ if value is None:
40
+ checks.append(tf.constant(False))
41
+ else:
42
+ checks.append(_matches(value, expected, membership=membership))
43
+ if not checks:
44
+ return tf.constant(True)
45
+ return tf.reduce_all(tf.stack([tf.cast(check, tf.bool) for check in checks]))
46
+
47
+ return predicate
48
+
49
+
50
+ def filter_by_metadata(ds: tf.data.Dataset, **criteria: Any) -> tf.data.Dataset:
51
+ return ds.filter(metadata_filter_predicate(**criteria))
52
+
53
+
54
+ def _normalize_group_keys(keys: str | list[str] | tuple[str, ...]) -> tuple[str, ...]:
55
+ if isinstance(keys, str):
56
+ return (keys,)
57
+ if not isinstance(keys, Iterable):
58
+ raise TypeError("keys must be a string or a list of strings.")
59
+ return tuple(keys)
60
+
61
+
62
+ def _group_value(sample: dict, keys: tuple[str, ...]) -> Any:
63
+ values = tuple(python_value(get_metadata_value(sample, key)) for key in keys)
64
+ return values[0] if len(values) == 1 else values
65
+
66
+
67
+ def groupby_metadata(
68
+ ds: tf.data.Dataset,
69
+ keys: str | list[str] | tuple[str, ...],
70
+ ) -> dict[Any, tf.data.Dataset]:
71
+ normalized_keys = _normalize_group_keys(keys)
72
+ values = []
73
+ seen = set()
74
+ for sample in ds:
75
+ value = _group_value(sample, normalized_keys)
76
+ if value not in seen:
77
+ seen.add(value)
78
+ values.append(value)
79
+
80
+ groups = {}
81
+ for value in values:
82
+ if len(normalized_keys) == 1:
83
+ criteria = {normalized_keys[0]: value}
84
+ else:
85
+ criteria = dict(zip(normalized_keys, value))
86
+ groups[value] = filter_by_metadata(ds, **criteria)
87
+ return groups
88
+
89
+
90
+ __all__ = [
91
+ "filter_by_metadata",
92
+ "groupby_metadata",
93
+ "metadata_filter_predicate",
94
+ ]