justdata 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. justdata/__init__.py +3 -0
  2. justdata/acoustic/__init__.py +381 -0
  3. justdata/acoustic/adapters.py +329 -0
  4. justdata/acoustic/augment/__init__.py +119 -0
  5. justdata/acoustic/augment/batch.py +813 -0
  6. justdata/acoustic/augment/configs.py +204 -0
  7. justdata/acoustic/augment/patchout.py +331 -0
  8. justdata/acoustic/augment/spectrogram.py +806 -0
  9. justdata/acoustic/augment/waveform.py +879 -0
  10. justdata/acoustic/batching.py +27 -0
  11. justdata/acoustic/channel.py +53 -0
  12. justdata/acoustic/compat/__init__.py +95 -0
  13. justdata/acoustic/compat/ast.py +626 -0
  14. justdata/acoustic/compat/ced.py +152 -0
  15. justdata/acoustic/compat/efficientat.py +243 -0
  16. justdata/acoustic/compat/panns.py +115 -0
  17. justdata/acoustic/compat/passt.py +191 -0
  18. justdata/acoustic/configs.py +519 -0
  19. justdata/acoustic/corruptions/__init__.py +33 -0
  20. justdata/acoustic/corruptions/codec.py +92 -0
  21. justdata/acoustic/corruptions/datasets.py +155 -0
  22. justdata/acoustic/corruptions/device.py +39 -0
  23. justdata/acoustic/corruptions/dynamics.py +55 -0
  24. justdata/acoustic/corruptions/filters.py +105 -0
  25. justdata/acoustic/corruptions/noise.py +173 -0
  26. justdata/acoustic/corruptions/registry.py +230 -0
  27. justdata/acoustic/corruptions/reverb.py +49 -0
  28. justdata/acoustic/corruptions/timefreq.py +127 -0
  29. justdata/acoustic/datasets.py +52 -0
  30. justdata/acoustic/dcase2025.py +814 -0
  31. justdata/acoustic/decoding.py +73 -0
  32. justdata/acoustic/eval_views.py +54 -0
  33. justdata/acoustic/evals.py +71 -0
  34. justdata/acoustic/filters.py +18 -0
  35. justdata/acoustic/frontends/__init__.py +37 -0
  36. justdata/acoustic/frontends/ast.py +116 -0
  37. justdata/acoustic/frontends/kaldi.py +34 -0
  38. justdata/acoustic/frontends/log.py +94 -0
  39. justdata/acoustic/frontends/mel.py +173 -0
  40. justdata/acoustic/frontends/mfcc.py +26 -0
  41. justdata/acoustic/frontends/pcen.py +26 -0
  42. justdata/acoustic/frontends/stft.py +174 -0
  43. justdata/acoustic/jax.py +23 -0
  44. justdata/acoustic/labels.py +169 -0
  45. justdata/acoustic/layouts.py +86 -0
  46. justdata/acoustic/metadata.py +152 -0
  47. justdata/acoustic/normalization.py +196 -0
  48. justdata/acoustic/pipelines.py +112 -0
  49. justdata/acoustic/postprocessing.py +328 -0
  50. justdata/acoustic/preprocessing.py +175 -0
  51. justdata/acoustic/presets.py +257 -0
  52. justdata/acoustic/registry.py +381 -0
  53. justdata/acoustic/resampling.py +218 -0
  54. justdata/acoustic/schema.py +51 -0
  55. justdata/acoustic/segment.py +369 -0
  56. justdata/acoustic/sources.py +503 -0
  57. justdata/acoustic/stages.py +59 -0
  58. justdata/acoustic/stats.py +50 -0
  59. justdata/acoustic/tasks.py +375 -0
  60. justdata/audio/__init__.py +20 -0
  61. justdata/core/__init__.py +74 -0
  62. justdata/core/adapters.py +46 -0
  63. justdata/core/augmentations.py +34 -0
  64. justdata/core/filters.py +94 -0
  65. justdata/core/finalization.py +173 -0
  66. justdata/core/label_mixing.py +129 -0
  67. justdata/core/loader.py +384 -0
  68. justdata/core/metadata.py +250 -0
  69. justdata/core/presets.py +155 -0
  70. justdata/core/registry.py +229 -0
  71. justdata/core/sources.py +87 -0
  72. justdata/core/stats.py +227 -0
  73. justdata/corruptions.py +52 -0
  74. justdata/py.typed +0 -0
  75. justdata/vision/__init__.py +17 -0
  76. justdata/vision/augmentations/__init__.py +31 -0
  77. justdata/vision/augmentations/auto.py +480 -0
  78. justdata/vision/augmentations/color.py +163 -0
  79. justdata/vision/augmentations/composed.py +156 -0
  80. justdata/vision/augmentations/geometric.py +302 -0
  81. justdata/vision/augmentations/mixing.py +232 -0
  82. justdata/vision/augmentations/registry.py +142 -0
  83. justdata/vision/corruptions/__init__.py +16 -0
  84. justdata/vision/corruptions/blur.py +65 -0
  85. justdata/vision/corruptions/digital.py +26 -0
  86. justdata/vision/corruptions/noise.py +20 -0
  87. justdata/vision/corruptions/registry.py +113 -0
  88. justdata/vision/corruptions/weather.py +100 -0
  89. justdata/vision/datasets.py +34 -0
  90. justdata/vision/encodings/__init__.py +1 -0
  91. justdata/vision/encodings/heatmaps.py +196 -0
  92. justdata/vision/minic.py +69 -0
  93. justdata/vision/pipelines.py +62 -0
  94. justdata/vision/presets.py +509 -0
  95. justdata/vision/sources.py +1080 -0
  96. justdata/vision/stages.py +271 -0
  97. justdata/vision/tasks/__init__.py +1 -0
  98. justdata/vision/tasks/classification.py +321 -0
  99. justdata/vision/tasks/depth.py +25 -0
  100. justdata/vision/tasks/segmentation.py +194 -0
  101. justdata/vision/transforms.py +113 -0
  102. justdata/vision/utils.py +741 -0
  103. justdata-1.0.0.dist-info/METADATA +860 -0
  104. justdata-1.0.0.dist-info/RECORD +106 -0
  105. justdata-1.0.0.dist-info/WHEEL +4 -0
  106. justdata-1.0.0.dist-info/licenses/LICENSE +23 -0
@@ -0,0 +1,503 @@
1
+ from __future__ import annotations
2
+
3
+ import csv
4
+ import json
5
+ import os
6
+ from pathlib import Path
7
+ from typing import Literal, Union
8
+
9
+ import numpy as np
10
+ import tensorflow as tf
11
+ from loguru import logger
12
+
13
+ from justdata.acoustic.adapters import (
14
+ standardize_waveform_layout,
15
+ to_float32_waveform,
16
+ )
17
+ from justdata.acoustic.decoding import decode_audio_file, decode_wav_bytes
18
+ from justdata.acoustic.schema import (
19
+ CLIP_ID,
20
+ DATASET,
21
+ END_TIME,
22
+ EXAMPLE_ID,
23
+ FILENAME,
24
+ LABEL,
25
+ PATH,
26
+ SAMPLE_RATE,
27
+ SOURCE_ID,
28
+ SPLIT,
29
+ START_TIME,
30
+ WAVEFORM,
31
+ )
32
+ from justdata.core.sources import register_source_loader, source_cache_dir
33
+
34
+
35
+ _LOCAL_REQUIRED_COLUMNS = {
36
+ PATH,
37
+ LABEL,
38
+ SPLIT,
39
+ CLIP_ID,
40
+ SOURCE_ID,
41
+ START_TIME,
42
+ END_TIME,
43
+ }
44
+
45
+
46
+ def _strip_prefix(dataset_name: str, prefix: str) -> str:
47
+ if not dataset_name.startswith(prefix):
48
+ raise ValueError(
49
+ f"Expected dataset name to start with {prefix!r}; got {dataset_name!r}."
50
+ )
51
+ return dataset_name[len(prefix) :]
52
+
53
+
54
+ def _resolve_manifest_path(
55
+ dataset_name: str,
56
+ prefix: str,
57
+ data_dir: Union[None, str, os.PathLike],
58
+ ) -> Path:
59
+ spec = _strip_prefix(dataset_name, prefix)
60
+ if not spec:
61
+ if data_dir is None:
62
+ raise ValueError(f"{prefix} requires a manifest path or data_dir.")
63
+ path = Path(data_dir)
64
+ else:
65
+ path = Path(spec).expanduser()
66
+ if not path.is_absolute() and data_dir is not None:
67
+ candidate = Path(data_dir).expanduser() / path
68
+ if candidate.exists() or not path.exists():
69
+ path = candidate
70
+
71
+ if path.is_dir():
72
+ for name in (
73
+ "manifest.csv",
74
+ "manifest.jsonl",
75
+ "metadata.csv",
76
+ "metadata.jsonl",
77
+ ):
78
+ candidate = path / name
79
+ if candidate.exists():
80
+ return candidate
81
+ raise ValueError(
82
+ f"No manifest.csv/jsonl or metadata.csv/jsonl found in {path}."
83
+ )
84
+
85
+ return path
86
+
87
+
88
+ def _read_manifest(path: Path) -> tuple[list[dict[str, str]], set[str]]:
89
+ suffix = path.suffix.lower()
90
+ if suffix == ".jsonl":
91
+ rows = []
92
+ columns = set()
93
+ with path.open("r", encoding="utf-8") as f:
94
+ for line in f:
95
+ line = line.strip()
96
+ if not line:
97
+ continue
98
+ row = json.loads(line)
99
+ row = {str(k): "" if v is None else str(v) for k, v in row.items()}
100
+ rows.append(row)
101
+ columns.update(row)
102
+ return rows, columns
103
+
104
+ with path.open("r", encoding="utf-8", newline="") as f:
105
+ dialect = csv.excel_tab if suffix in {".tsv", ".tab"} else csv.excel
106
+ reader = csv.DictReader(f, dialect=dialect)
107
+ rows = [
108
+ {str(k): "" if v is None else str(v) for k, v in row.items()}
109
+ for row in reader
110
+ ]
111
+ return rows, set(reader.fieldnames or [])
112
+
113
+
114
+ def _coalesce(row: dict[str, str], *keys: str, default: str = "") -> str:
115
+ for key in keys:
116
+ value = row.get(key)
117
+ if value not in (None, ""):
118
+ return value
119
+ return default
120
+
121
+
122
+ def _float_value(value: str, default: float = 0.0) -> float:
123
+ if value == "":
124
+ return default
125
+ return float(value)
126
+
127
+
128
+ def _resolve_audio_path(
129
+ value: str, manifest_path: Path, data_dir: Union[None, str, os.PathLike]
130
+ ) -> str:
131
+ path = Path(value).expanduser()
132
+ if path.is_absolute():
133
+ return os.fspath(path)
134
+
135
+ manifest_relative = manifest_path.parent / path
136
+ if manifest_relative.exists() or data_dir is None:
137
+ return os.fspath(manifest_relative)
138
+
139
+ data_relative = Path(data_dir).expanduser() / path
140
+ return os.fspath(data_relative)
141
+
142
+
143
+ def _label_dtype(records: list[dict]) -> tf.DType:
144
+ labels = [record[LABEL] for record in records if LABEL in record]
145
+ if labels and all(isinstance(label, (int, np.integer)) for label in labels):
146
+ return tf.int64
147
+ return tf.string
148
+
149
+
150
+ def _looks_int(value: str) -> bool:
151
+ try:
152
+ int(value)
153
+ except (TypeError, ValueError):
154
+ return False
155
+ return True
156
+
157
+
158
+ def _manifest_records(
159
+ rows: list[dict[str, str]],
160
+ manifest_path: Path,
161
+ requested_split: str,
162
+ *,
163
+ data_dir: Union[None, str, os.PathLike],
164
+ dataset: str,
165
+ require_local_columns: bool,
166
+ ) -> list[dict]:
167
+ records = []
168
+ for row in rows:
169
+ split = _coalesce(row, SPLIT, "subset", "partition", default=requested_split)
170
+ if split != requested_split:
171
+ continue
172
+
173
+ audio_path_value = _coalesce(row, PATH, "audio_path", "filename", "file")
174
+ if not audio_path_value:
175
+ raise ValueError("Acoustic manifest rows must contain a path or filename.")
176
+
177
+ label_value = _coalesce(row, LABEL, "scene_label", "event_label", "target")
178
+ if require_local_columns and label_value == "":
179
+ raise ValueError("Local acoustic manifest rows must contain a label.")
180
+
181
+ audio_path = _resolve_audio_path(audio_path_value, manifest_path, data_dir)
182
+ filename = _coalesce(row, FILENAME, "filename", default=Path(audio_path).name)
183
+ clip_id = _coalesce(row, CLIP_ID, "segment_id", default=Path(filename).stem)
184
+
185
+ label = int(label_value) if _looks_int(label_value) else label_value
186
+ records.append(
187
+ {
188
+ PATH: audio_path,
189
+ LABEL: label,
190
+ SPLIT: split,
191
+ CLIP_ID: clip_id,
192
+ SOURCE_ID: _coalesce(row, SOURCE_ID, "recording_id", "device"),
193
+ START_TIME: _float_value(_coalesce(row, START_TIME)),
194
+ END_TIME: _float_value(_coalesce(row, END_TIME)),
195
+ DATASET: dataset,
196
+ EXAMPLE_ID: _coalesce(row, EXAMPLE_ID, default=clip_id),
197
+ FILENAME: filename,
198
+ }
199
+ )
200
+ return records
201
+
202
+
203
+ def _records_to_dataset(records: list[dict]) -> tf.data.Dataset:
204
+ label_dtype = _label_dtype(records)
205
+
206
+ def gen():
207
+ for record in records:
208
+ item = dict(record)
209
+ if label_dtype == tf.string:
210
+ item[LABEL] = str(item[LABEL])
211
+ else:
212
+ item[LABEL] = np.int64(item[LABEL])
213
+ yield item
214
+
215
+ ds = tf.data.Dataset.from_generator(
216
+ gen,
217
+ output_signature={
218
+ PATH: tf.TensorSpec(shape=(), dtype=tf.string),
219
+ LABEL: tf.TensorSpec(shape=(), dtype=label_dtype),
220
+ SPLIT: tf.TensorSpec(shape=(), dtype=tf.string),
221
+ CLIP_ID: tf.TensorSpec(shape=(), dtype=tf.string),
222
+ SOURCE_ID: tf.TensorSpec(shape=(), dtype=tf.string),
223
+ START_TIME: tf.TensorSpec(shape=(), dtype=tf.float32),
224
+ END_TIME: tf.TensorSpec(shape=(), dtype=tf.float32),
225
+ DATASET: tf.TensorSpec(shape=(), dtype=tf.string),
226
+ EXAMPLE_ID: tf.TensorSpec(shape=(), dtype=tf.string),
227
+ FILENAME: tf.TensorSpec(shape=(), dtype=tf.string),
228
+ },
229
+ )
230
+
231
+ try:
232
+ ds = ds.apply(tf.data.experimental.assert_cardinality(len(records)))
233
+ except Exception as e:
234
+ logger.warning(f"Failed to assert cardinality for acoustic manifest: {e}")
235
+
236
+ return ds
237
+
238
+
239
+ def _load_manifest_splits(
240
+ dataset_name: str,
241
+ splits: list[str],
242
+ data_dir: Union[None, str, os.PathLike],
243
+ *,
244
+ prefix: str,
245
+ require_local_columns: bool,
246
+ ) -> list[tf.data.Dataset]:
247
+ manifest_path = _resolve_manifest_path(dataset_name, prefix, data_dir)
248
+ rows, columns = _read_manifest(manifest_path)
249
+
250
+ if require_local_columns:
251
+ missing = _LOCAL_REQUIRED_COLUMNS - columns
252
+ if missing:
253
+ missing_cols = ", ".join(sorted(missing))
254
+ raise ValueError(
255
+ f"Local acoustic manifest is missing required columns: {missing_cols}."
256
+ )
257
+
258
+ dataset = manifest_path.stem
259
+ return [
260
+ _records_to_dataset(
261
+ _manifest_records(
262
+ rows,
263
+ manifest_path,
264
+ split,
265
+ data_dir=data_dir,
266
+ dataset=dataset,
267
+ require_local_columns=require_local_columns,
268
+ )
269
+ )
270
+ for split in splits
271
+ ]
272
+
273
+
274
+ @register_source_loader("local_audio:")
275
+ def load_local_audio_manifest_splits(
276
+ dataset_name: str,
277
+ splits: list[str],
278
+ data_dir: Union[None, str, os.PathLike] = None,
279
+ ) -> list[tf.data.Dataset]:
280
+ return _load_manifest_splits(
281
+ dataset_name,
282
+ splits,
283
+ data_dir,
284
+ prefix="local_audio:",
285
+ require_local_columns=True,
286
+ )
287
+
288
+
289
+ @register_source_loader("dcase2025:")
290
+ def load_dcase2025_splits(
291
+ dataset_name: str,
292
+ splits: list[str],
293
+ data_dir: Union[None, str, os.PathLike] = None,
294
+ ) -> list[tf.data.Dataset]:
295
+ from justdata.acoustic.dcase2025 import load_dcase2025_task1_splits
296
+
297
+ return load_dcase2025_task1_splits(dataset_name, splits, data_dir)
298
+
299
+
300
+ def _import_datasets():
301
+ try:
302
+ import datasets
303
+ except ImportError as e:
304
+ raise ImportError(
305
+ "The 'datasets' package is required for hf_audio: source loading. "
306
+ "Install justdata with the acoustic extra."
307
+ ) from e
308
+ return datasets
309
+
310
+
311
+ def _hf_label_signature(value) -> tuple[tf.DType, tuple | None]:
312
+ if isinstance(value, (list, tuple, np.ndarray)):
313
+ arr = np.asarray(value)
314
+ if np.issubdtype(arr.dtype, np.integer):
315
+ return tf.int64, (None,)
316
+ if np.issubdtype(arr.dtype, np.floating):
317
+ return tf.float32, (None,)
318
+ return tf.string, (None,)
319
+ if isinstance(value, (int, np.integer, bool)):
320
+ return tf.int64, ()
321
+ if isinstance(value, (float, np.floating)):
322
+ return tf.float32, ()
323
+ return tf.string, ()
324
+
325
+
326
+ def _normalize_label(value, dtype: tf.DType):
327
+ if dtype == tf.int64:
328
+ return np.asarray(value, dtype=np.int64)
329
+ if dtype == tf.float32:
330
+ return np.asarray(value, dtype=np.float32)
331
+ if isinstance(value, (list, tuple, np.ndarray)):
332
+ return np.asarray(value, dtype=str)
333
+ return str(value)
334
+
335
+
336
+ def _extract_hf_audio_value(audio_value, fallback_sample_rate: int | None):
337
+ sample_rate = fallback_sample_rate
338
+ path = None
339
+ contents = None
340
+ layout_hint = "auto"
341
+
342
+ if isinstance(audio_value, dict):
343
+ path = audio_value.get("path")
344
+ contents = audio_value.get("bytes")
345
+ sample_rate = audio_value.get(
346
+ "sampling_rate", audio_value.get(SAMPLE_RATE, sample_rate)
347
+ )
348
+ array = audio_value.get("array")
349
+ elif isinstance(audio_value, (str, bytes, os.PathLike)):
350
+ array = None
351
+ path = os.fspath(audio_value)
352
+ else:
353
+ try:
354
+ array = audio_value["array"]
355
+ except (KeyError, TypeError, IndexError):
356
+ array = audio_value
357
+ else:
358
+ layout_hint = "ct"
359
+ try:
360
+ sample_rate = audio_value["sampling_rate"]
361
+ except (KeyError, TypeError, IndexError):
362
+ pass
363
+
364
+ return array, sample_rate, path, contents, layout_hint
365
+
366
+
367
+ def _as_waveform_np(audio_value, *, decode_mode: str, fallback_sample_rate: int | None):
368
+ array, sample_rate, path, contents, layout_hint = _extract_hf_audio_value(
369
+ audio_value, fallback_sample_rate
370
+ )
371
+
372
+ if decode_mode == "justdata":
373
+ if contents is not None:
374
+ waveform, decoded_sample_rate = decode_wav_bytes(contents)
375
+ sample_rate = int(decoded_sample_rate.numpy())
376
+ array = waveform.numpy()
377
+ layout_hint = "tc"
378
+ elif path:
379
+ waveform, decoded_sample_rate = decode_audio_file(path)
380
+ sample_rate = int(decoded_sample_rate.numpy())
381
+ array = waveform.numpy()
382
+ layout_hint = "tc"
383
+
384
+ if array is None and path:
385
+ waveform, decoded_sample_rate = decode_audio_file(path)
386
+ sample_rate = int(decoded_sample_rate.numpy())
387
+ array = waveform.numpy()
388
+ layout_hint = "tc"
389
+
390
+ if sample_rate is None:
391
+ raise ValueError("Hugging Face audio records must provide a sampling_rate.")
392
+
393
+ tensor = tf.convert_to_tensor(array)
394
+ tensor = to_float32_waveform(tensor, input_dtype=tensor.dtype)
395
+ tensor = standardize_waveform_layout(tensor, layout_hint=layout_hint)
396
+ return tensor.numpy().astype(np.float32), np.int32(sample_rate), path
397
+
398
+
399
+ @register_source_loader("hf_audio:")
400
+ def load_huggingface_audio_splits(
401
+ dataset_name: str,
402
+ splits: list[str],
403
+ data_dir: Union[None, str, os.PathLike] = None,
404
+ *,
405
+ decode_mode: Literal["hf_native", "justdata"] = "hf_native",
406
+ hf_audio_sampling_rate: int | None = None,
407
+ audio_column: str = "audio",
408
+ label_column: str | None = "label",
409
+ ) -> list[tf.data.Dataset]:
410
+ if decode_mode not in {"hf_native", "justdata"}:
411
+ raise ValueError("decode_mode must be 'hf_native' or 'justdata'.")
412
+
413
+ datasets = _import_datasets()
414
+ hf_name = _strip_prefix(dataset_name, "hf_audio:")
415
+ loaded_splits = []
416
+
417
+ for split in splits:
418
+ ds_hf = datasets.load_dataset(
419
+ hf_name,
420
+ split=split,
421
+ cache_dir=os.fspath(source_cache_dir(data_dir, "hf", "acoustic")),
422
+ )
423
+
424
+ if audio_column not in ds_hf.column_names:
425
+ raise ValueError(
426
+ f"Hugging Face audio dataset '{hf_name}' must contain audio column "
427
+ f"{audio_column!r}. Found columns: {ds_hf.column_names}."
428
+ )
429
+ if label_column is not None and label_column not in ds_hf.column_names:
430
+ raise ValueError(
431
+ f"Hugging Face audio dataset '{hf_name}' must contain label column "
432
+ f"{label_column!r}. Found columns: {ds_hf.column_names}."
433
+ )
434
+
435
+ if decode_mode == "justdata":
436
+ ds_hf = ds_hf.cast_column(audio_column, datasets.Audio(decode=False))
437
+ elif hf_audio_sampling_rate is not None:
438
+ ds_hf = ds_hf.cast_column(
439
+ audio_column,
440
+ datasets.Audio(sampling_rate=hf_audio_sampling_rate),
441
+ )
442
+
443
+ first = next(iter(ds_hf), None)
444
+ label_dtype = None
445
+ label_shape = None
446
+ if label_column is not None:
447
+ if first is None:
448
+ label_dtype, label_shape = tf.int64, ()
449
+ else:
450
+ label_dtype, label_shape = _hf_label_signature(first[label_column])
451
+
452
+ def gen(_ds=ds_hf, _split=split, _label_dtype=label_dtype):
453
+ for idx, sample in enumerate(_ds):
454
+ waveform, sample_rate, path = _as_waveform_np(
455
+ sample[audio_column],
456
+ decode_mode=decode_mode,
457
+ fallback_sample_rate=hf_audio_sampling_rate,
458
+ )
459
+ filename = Path(path).name if path else ""
460
+ item = {
461
+ WAVEFORM: waveform,
462
+ SAMPLE_RATE: sample_rate,
463
+ DATASET: hf_name,
464
+ SPLIT: _split,
465
+ EXAMPLE_ID: str(sample.get("id", idx)),
466
+ FILENAME: filename,
467
+ }
468
+ if label_column is not None:
469
+ item[LABEL] = _normalize_label(sample[label_column], _label_dtype)
470
+ yield item
471
+
472
+ output_signature = {
473
+ WAVEFORM: tf.TensorSpec(shape=(None, None), dtype=tf.float32),
474
+ SAMPLE_RATE: tf.TensorSpec(shape=(), dtype=tf.int32),
475
+ DATASET: tf.TensorSpec(shape=(), dtype=tf.string),
476
+ SPLIT: tf.TensorSpec(shape=(), dtype=tf.string),
477
+ EXAMPLE_ID: tf.TensorSpec(shape=(), dtype=tf.string),
478
+ FILENAME: tf.TensorSpec(shape=(), dtype=tf.string),
479
+ }
480
+ if label_column is not None:
481
+ output_signature[LABEL] = tf.TensorSpec(
482
+ shape=label_shape, dtype=label_dtype
483
+ )
484
+
485
+ ds_tf = tf.data.Dataset.from_generator(gen, output_signature=output_signature)
486
+
487
+ try:
488
+ ds_tf = ds_tf.apply(tf.data.experimental.assert_cardinality(len(ds_hf)))
489
+ except Exception as e:
490
+ logger.warning(
491
+ f"Failed to assert cardinality for HF audio dataset {hf_name}: {e}"
492
+ )
493
+
494
+ loaded_splits.append(ds_tf)
495
+
496
+ return loaded_splits
497
+
498
+
499
+ __all__ = [
500
+ "load_dcase2025_splits",
501
+ "load_huggingface_audio_splits",
502
+ "load_local_audio_manifest_splits",
503
+ ]
@@ -0,0 +1,59 @@
1
+ from __future__ import annotations
2
+
3
+ from collections.abc import Callable
4
+
5
+ import tensorflow as tf
6
+
7
+ from justdata.acoustic.configs import AudioPreprocessConfig, SegmentStrategyConfig
8
+ from justdata.acoustic.preprocessing import make_preprocessing
9
+ from justdata.acoustic.segment import segment_waveform
10
+ from justdata.acoustic.schema import DURATION, METADATA, SAMPLE_RATE, WAVEFORM
11
+
12
+
13
+ def make_segment_stage(
14
+ config: SegmentStrategyConfig,
15
+ *,
16
+ is_training: bool,
17
+ audio_key: str = WAVEFORM,
18
+ ) -> Callable[[dict, tf.Tensor | int | None], dict]:
19
+ config = SegmentStrategyConfig.from_dict(config)
20
+
21
+ def stage(sample: dict, seed: tf.Tensor | int | None = None) -> dict:
22
+ segmented = segment_waveform(
23
+ sample[audio_key],
24
+ sample[SAMPLE_RATE],
25
+ config,
26
+ is_training=is_training,
27
+ seed=seed,
28
+ )
29
+ metadata = dict(sample.get(METADATA, {}))
30
+ if "original_duration" not in metadata and DURATION in sample:
31
+ metadata["original_duration"] = sample[DURATION]
32
+ metadata.update(segmented[METADATA])
33
+
34
+ audio = segmented["audio"]
35
+ time_axis = 1 if audio.shape.rank == 3 else 0
36
+ duration = tf.cast(tf.shape(audio)[time_axis], tf.float32) / tf.cast(
37
+ sample[SAMPLE_RATE], tf.float32
38
+ )
39
+ metadata[DURATION] = duration
40
+
41
+ result = dict(sample)
42
+ result[audio_key] = audio
43
+ result[DURATION] = duration
44
+ result[METADATA] = metadata
45
+ return result
46
+
47
+ return stage
48
+
49
+
50
+ def make_preprocessing_stage(
51
+ config: AudioPreprocessConfig,
52
+ ) -> Callable[[dict], dict]:
53
+ return make_preprocessing(config)
54
+
55
+
56
+ __all__ = [
57
+ "make_preprocessing_stage",
58
+ "make_segment_stage",
59
+ ]
@@ -0,0 +1,50 @@
1
+ from __future__ import annotations
2
+
3
+ from typing import Literal
4
+
5
+ import tensorflow as tf
6
+
7
+ from justdata.acoustic.dcase2025 import assert_stats_allowed
8
+ from justdata.core.stats import compute_feature_stats as _compute_feature_stats
9
+ from justdata.core.stats import make_stats_iterator as _make_stats_iterator
10
+
11
+
12
+ def make_stats_iterator(
13
+ dataset: tf.data.Dataset,
14
+ split: str,
15
+ groupby: str | list[str] | None,
16
+ batch_size: int,
17
+ deterministic: bool = True,
18
+ augment: bool = False,
19
+ allow_override: bool = False,
20
+ metadata_mode: Literal["full", "numeric_only", "none"] = "numeric_only",
21
+ ):
22
+ assert_stats_allowed(split, allow_override=allow_override)
23
+ return _make_stats_iterator(
24
+ dataset,
25
+ groupby=groupby,
26
+ batch_size=batch_size,
27
+ deterministic=deterministic,
28
+ augment=augment,
29
+ metadata_mode=metadata_mode,
30
+ )
31
+
32
+
33
+ def compute_feature_stats(
34
+ dataset,
35
+ axes=("time",),
36
+ groupby="device",
37
+ feature_key="inputs",
38
+ ) -> dict:
39
+ return _compute_feature_stats(
40
+ dataset,
41
+ axes=tuple(axes),
42
+ groupby=groupby,
43
+ feature_key=feature_key,
44
+ )
45
+
46
+
47
+ __all__ = [
48
+ "compute_feature_stats",
49
+ "make_stats_iterator",
50
+ ]