justdata 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. justdata/__init__.py +3 -0
  2. justdata/acoustic/__init__.py +381 -0
  3. justdata/acoustic/adapters.py +329 -0
  4. justdata/acoustic/augment/__init__.py +119 -0
  5. justdata/acoustic/augment/batch.py +813 -0
  6. justdata/acoustic/augment/configs.py +204 -0
  7. justdata/acoustic/augment/patchout.py +331 -0
  8. justdata/acoustic/augment/spectrogram.py +806 -0
  9. justdata/acoustic/augment/waveform.py +879 -0
  10. justdata/acoustic/batching.py +27 -0
  11. justdata/acoustic/channel.py +53 -0
  12. justdata/acoustic/compat/__init__.py +95 -0
  13. justdata/acoustic/compat/ast.py +626 -0
  14. justdata/acoustic/compat/ced.py +152 -0
  15. justdata/acoustic/compat/efficientat.py +243 -0
  16. justdata/acoustic/compat/panns.py +115 -0
  17. justdata/acoustic/compat/passt.py +191 -0
  18. justdata/acoustic/configs.py +519 -0
  19. justdata/acoustic/corruptions/__init__.py +33 -0
  20. justdata/acoustic/corruptions/codec.py +92 -0
  21. justdata/acoustic/corruptions/datasets.py +155 -0
  22. justdata/acoustic/corruptions/device.py +39 -0
  23. justdata/acoustic/corruptions/dynamics.py +55 -0
  24. justdata/acoustic/corruptions/filters.py +105 -0
  25. justdata/acoustic/corruptions/noise.py +173 -0
  26. justdata/acoustic/corruptions/registry.py +230 -0
  27. justdata/acoustic/corruptions/reverb.py +49 -0
  28. justdata/acoustic/corruptions/timefreq.py +127 -0
  29. justdata/acoustic/datasets.py +52 -0
  30. justdata/acoustic/dcase2025.py +814 -0
  31. justdata/acoustic/decoding.py +73 -0
  32. justdata/acoustic/eval_views.py +54 -0
  33. justdata/acoustic/evals.py +71 -0
  34. justdata/acoustic/filters.py +18 -0
  35. justdata/acoustic/frontends/__init__.py +37 -0
  36. justdata/acoustic/frontends/ast.py +116 -0
  37. justdata/acoustic/frontends/kaldi.py +34 -0
  38. justdata/acoustic/frontends/log.py +94 -0
  39. justdata/acoustic/frontends/mel.py +173 -0
  40. justdata/acoustic/frontends/mfcc.py +26 -0
  41. justdata/acoustic/frontends/pcen.py +26 -0
  42. justdata/acoustic/frontends/stft.py +174 -0
  43. justdata/acoustic/jax.py +23 -0
  44. justdata/acoustic/labels.py +169 -0
  45. justdata/acoustic/layouts.py +86 -0
  46. justdata/acoustic/metadata.py +152 -0
  47. justdata/acoustic/normalization.py +196 -0
  48. justdata/acoustic/pipelines.py +112 -0
  49. justdata/acoustic/postprocessing.py +328 -0
  50. justdata/acoustic/preprocessing.py +175 -0
  51. justdata/acoustic/presets.py +257 -0
  52. justdata/acoustic/registry.py +381 -0
  53. justdata/acoustic/resampling.py +218 -0
  54. justdata/acoustic/schema.py +51 -0
  55. justdata/acoustic/segment.py +369 -0
  56. justdata/acoustic/sources.py +503 -0
  57. justdata/acoustic/stages.py +59 -0
  58. justdata/acoustic/stats.py +50 -0
  59. justdata/acoustic/tasks.py +375 -0
  60. justdata/audio/__init__.py +20 -0
  61. justdata/core/__init__.py +74 -0
  62. justdata/core/adapters.py +46 -0
  63. justdata/core/augmentations.py +34 -0
  64. justdata/core/filters.py +94 -0
  65. justdata/core/finalization.py +173 -0
  66. justdata/core/label_mixing.py +129 -0
  67. justdata/core/loader.py +384 -0
  68. justdata/core/metadata.py +250 -0
  69. justdata/core/presets.py +155 -0
  70. justdata/core/registry.py +229 -0
  71. justdata/core/sources.py +87 -0
  72. justdata/core/stats.py +227 -0
  73. justdata/corruptions.py +52 -0
  74. justdata/py.typed +0 -0
  75. justdata/vision/__init__.py +17 -0
  76. justdata/vision/augmentations/__init__.py +31 -0
  77. justdata/vision/augmentations/auto.py +480 -0
  78. justdata/vision/augmentations/color.py +163 -0
  79. justdata/vision/augmentations/composed.py +156 -0
  80. justdata/vision/augmentations/geometric.py +302 -0
  81. justdata/vision/augmentations/mixing.py +232 -0
  82. justdata/vision/augmentations/registry.py +142 -0
  83. justdata/vision/corruptions/__init__.py +16 -0
  84. justdata/vision/corruptions/blur.py +65 -0
  85. justdata/vision/corruptions/digital.py +26 -0
  86. justdata/vision/corruptions/noise.py +20 -0
  87. justdata/vision/corruptions/registry.py +113 -0
  88. justdata/vision/corruptions/weather.py +100 -0
  89. justdata/vision/datasets.py +34 -0
  90. justdata/vision/encodings/__init__.py +1 -0
  91. justdata/vision/encodings/heatmaps.py +196 -0
  92. justdata/vision/minic.py +69 -0
  93. justdata/vision/pipelines.py +62 -0
  94. justdata/vision/presets.py +509 -0
  95. justdata/vision/sources.py +1080 -0
  96. justdata/vision/stages.py +271 -0
  97. justdata/vision/tasks/__init__.py +1 -0
  98. justdata/vision/tasks/classification.py +321 -0
  99. justdata/vision/tasks/depth.py +25 -0
  100. justdata/vision/tasks/segmentation.py +194 -0
  101. justdata/vision/transforms.py +113 -0
  102. justdata/vision/utils.py +741 -0
  103. justdata-1.0.0.dist-info/METADATA +860 -0
  104. justdata-1.0.0.dist-info/RECORD +106 -0
  105. justdata-1.0.0.dist-info/WHEEL +4 -0
  106. justdata-1.0.0.dist-info/licenses/LICENSE +23 -0
@@ -0,0 +1,814 @@
1
+ from __future__ import annotations
2
+
3
+ import csv
4
+ import json
5
+ import os
6
+ import re
7
+ import warnings
8
+ from pathlib import Path
9
+ from typing import Any, Literal, Union
10
+
11
+ import numpy as np
12
+ import tensorflow as tf
13
+ from loguru import logger
14
+
15
+ from justdata.acoustic.adapters import adapt_acoustic_sample
16
+ from justdata.acoustic.configs import AudioPreset
17
+ from justdata.acoustic.presets import register_preset
18
+ from justdata.acoustic.schema import (
19
+ CLIP_ID,
20
+ DATASET,
21
+ END_TIME,
22
+ EXAMPLE_ID,
23
+ FILENAME,
24
+ LABEL,
25
+ METADATA,
26
+ PATH,
27
+ SOURCE_ID,
28
+ SPLIT,
29
+ START_TIME,
30
+ )
31
+ from justdata.core.adapters import register_adapter
32
+ from justdata.core.filters import metadata_filter_predicate
33
+ from justdata.core.loader import load_ds
34
+ from justdata.core.registry import get_pipeline
35
+ from justdata.core.sources import register_source_loader
36
+
37
+
38
+ class SplitLeakageError(ValueError):
39
+ pass
40
+
41
+
42
+ known_train_devices = {"A", "B", "C", "S1", "S2", "S3"}
43
+ real_devices = {"A", "B", "C", "D"}
44
+ simulated_devices = {"S1", "S2", "S3", "S4", "S5", "S6", "S7", "S8", "S9", "S10"}
45
+
46
+ allow_stats_on_split = {
47
+ "dev_train_25": True,
48
+ "dev_test": False,
49
+ "eval": False,
50
+ }
51
+
52
+ _DEVICE_RE = re.compile(
53
+ r"(?:^|[._\-/])(?P<device>unknown|s10|s[1-9]|[abcd])(?:$|[._\-/])",
54
+ re.IGNORECASE,
55
+ )
56
+ _MISSING_VALUES = {"", "none", "null", "nan", "n/a", "na"}
57
+
58
+
59
+ def _text(value: Any) -> str | None:
60
+ if value is None:
61
+ return None
62
+ if hasattr(value, "numpy"):
63
+ try:
64
+ value = value.numpy()
65
+ except Exception:
66
+ return None
67
+ if isinstance(value, bytes):
68
+ value = value.decode("utf-8")
69
+ value = str(value).strip()
70
+ if value.lower() in _MISSING_VALUES:
71
+ return None
72
+ return value
73
+
74
+
75
+ def _row_value(row: dict, *keys: str) -> str | None:
76
+ lowered = {str(key).lower(): value for key, value in row.items()}
77
+ for key in keys:
78
+ value = row.get(key)
79
+ if value is None:
80
+ value = lowered.get(key.lower())
81
+ value = _text(value)
82
+ if value is not None:
83
+ return value
84
+ return None
85
+
86
+
87
+ def _filename_from_row(row: dict) -> str | None:
88
+ value = _row_value(row, FILENAME, "file", "filename", "audio_filename")
89
+ if value is not None:
90
+ return Path(value).name
91
+ path = _row_value(row, PATH, "audio_path", "filepath", "file_path")
92
+ if path is None:
93
+ return None
94
+ return Path(path).name
95
+
96
+
97
+ def _stem_tokens(row: dict) -> list[str]:
98
+ filename = _filename_from_row(row)
99
+ if filename is None:
100
+ return []
101
+ return [token for token in re.split(r"[._\-/]+", Path(filename).stem) if token]
102
+
103
+
104
+ def _normalize_device(value: str | None) -> str | None:
105
+ if value is None:
106
+ return None
107
+ value = Path(value).stem.strip()
108
+ if value.lower() == "unknown":
109
+ return "unknown"
110
+ value = value.upper()
111
+ if value in real_devices or value in simulated_devices:
112
+ return value
113
+ return None
114
+
115
+
116
+ def _scene_from_filename(row: dict) -> str | None:
117
+ filename = _filename_from_row(row)
118
+ if filename is None:
119
+ return None
120
+ stem = Path(filename).stem.lower()
121
+ for name in DCASE2025Task1Adapter.class_names:
122
+ name_pattern = re.escape(name).replace("_", r"[._\-/]")
123
+ pattern = re.compile(rf"(?:^|[._\-/]){name_pattern}(?:$|[._\-/])")
124
+ if pattern.search(stem):
125
+ return name
126
+ return None
127
+
128
+
129
+ def parse_scene_from_row(row) -> str | None:
130
+ value = _row_value(row, "scene_label", "scene", "scene_name", "class_name")
131
+ if value is None:
132
+ value = _row_value(row, LABEL, "target")
133
+ if value is not None:
134
+ normalized = value.lower().replace("-", "_").replace(" ", "_")
135
+ if normalized in DCASE2025Task1Adapter.class_names:
136
+ return normalized
137
+ if normalized.isdigit():
138
+ index = int(normalized)
139
+ if 0 <= index < len(DCASE2025Task1Adapter.class_names):
140
+ return DCASE2025Task1Adapter.class_names[index]
141
+ return _scene_from_filename(row)
142
+
143
+
144
+ def parse_device_from_row(row) -> str:
145
+ value = _normalize_device(
146
+ _row_value(
147
+ row,
148
+ "device",
149
+ "device_id",
150
+ "recording_device",
151
+ "source_device",
152
+ "domain",
153
+ )
154
+ )
155
+ if value is not None:
156
+ return value
157
+
158
+ filename = _filename_from_row(row)
159
+ if filename is not None:
160
+ matches = list(_DEVICE_RE.finditer(filename))
161
+ if matches:
162
+ device = _normalize_device(matches[-1].group("device"))
163
+ if device is not None:
164
+ return device
165
+
166
+ split = _row_value(row, SPLIT, "subset", "partition")
167
+ if split == "eval":
168
+ return "unknown"
169
+ raise ValueError("Could not parse DCASE device from row.")
170
+
171
+
172
+ def parse_city_from_row(row) -> str | None:
173
+ value = _row_value(row, "city", "recording_city", "location_city")
174
+ if value is not None:
175
+ return value
176
+
177
+ scene = parse_scene_from_row(row)
178
+ if scene is None:
179
+ return None
180
+ tokens = _stem_tokens(row)
181
+ scene_tokens = scene.split("_") if scene else []
182
+ if (
183
+ scene_tokens
184
+ and [token.lower() for token in tokens[: len(scene_tokens)]] == scene_tokens
185
+ ):
186
+ tokens = tokens[len(scene_tokens) :]
187
+ if tokens:
188
+ candidate = tokens[0]
189
+ if _normalize_device(candidate) is None and not candidate.isdigit():
190
+ return candidate
191
+ return None
192
+
193
+
194
+ def _parse_location_id(row) -> str | None:
195
+ value = _row_value(row, "location_id", "location", "recording_location")
196
+ if value is not None:
197
+ return value
198
+ tokens = _stem_tokens(row)
199
+ scene = parse_scene_from_row(row)
200
+ if scene is None:
201
+ return None
202
+ offset = len(scene.split("_")) if scene else 0
203
+ if len(tokens) > offset + 1:
204
+ candidate = tokens[offset + 1]
205
+ if _normalize_device(candidate) is None:
206
+ return candidate
207
+ return None
208
+
209
+
210
+ def _parse_segment_id(row) -> str | None:
211
+ value = _row_value(row, "segment_id", "segment", CLIP_ID)
212
+ if value is not None:
213
+ return value
214
+ tokens = _stem_tokens(row)
215
+ if tokens and _normalize_device(tokens[-1]) is not None:
216
+ tokens = tokens[:-1]
217
+ if tokens and tokens[-1].isdigit():
218
+ return tokens[-1]
219
+ return None
220
+
221
+
222
+ def parse_source_recording_id(row) -> str | None:
223
+ value = _row_value(
224
+ row,
225
+ "source_recording_id",
226
+ "original_recording_id",
227
+ "recording_id",
228
+ SOURCE_ID,
229
+ )
230
+ if value is not None:
231
+ return value
232
+
233
+ tokens = _stem_tokens(row)
234
+ if not tokens:
235
+ return None
236
+ if _normalize_device(tokens[-1]) is not None:
237
+ tokens = tokens[:-1]
238
+ segment_id = _parse_segment_id(row)
239
+ if segment_id is not None and tokens and tokens[-1] == segment_id:
240
+ tokens = tokens[:-1]
241
+ elif tokens and tokens[-1].isdigit():
242
+ tokens = tokens[:-1]
243
+ return "-".join(tokens) if tokens else None
244
+
245
+
246
+ def _device_type(device: str) -> Literal["real", "simulated", "unknown"]:
247
+ if device in real_devices:
248
+ return "real"
249
+ if device in simulated_devices:
250
+ return "simulated"
251
+ return "unknown"
252
+
253
+
254
+ def _scene_id(scene_label: str | None) -> int | None:
255
+ if scene_label is None:
256
+ return None
257
+ try:
258
+ return DCASE2025Task1Adapter.class_names.index(scene_label)
259
+ except ValueError:
260
+ return None
261
+
262
+
263
+ def _tensor_scene_id(scene_label: tf.Tensor) -> tf.Tensor:
264
+ labels = tf.constant(DCASE2025Task1Adapter.class_names, dtype=tf.string)
265
+ matches = tf.equal(labels, scene_label)
266
+ return tf.cond(
267
+ tf.reduce_any(matches),
268
+ lambda: tf.cast(tf.argmax(tf.cast(matches, tf.int32)), tf.int64),
269
+ lambda: tf.constant(-1, dtype=tf.int64),
270
+ )
271
+
272
+
273
+ def _tensor_device_type(device: tf.Tensor) -> tf.Tensor:
274
+ real = tf.constant(sorted(real_devices), dtype=tf.string)
275
+ simulated = tf.constant(sorted(simulated_devices), dtype=tf.string)
276
+ return tf.case(
277
+ (
278
+ (tf.reduce_any(tf.equal(real, device)), lambda: tf.constant("real")),
279
+ (
280
+ tf.reduce_any(tf.equal(simulated, device)),
281
+ lambda: tf.constant("simulated"),
282
+ ),
283
+ ),
284
+ default=lambda: tf.constant("unknown"),
285
+ exclusive=False,
286
+ )
287
+
288
+
289
+ def _tensor_known_device(device: tf.Tensor) -> tf.Tensor:
290
+ known = tf.constant(sorted(known_train_devices), dtype=tf.string)
291
+ return tf.reduce_any(tf.equal(known, device))
292
+
293
+
294
+ def _optional_text(value: Any) -> Any:
295
+ if hasattr(value, "dtype"):
296
+ return value
297
+ return _text(value)
298
+
299
+
300
+ def _optional_int(value: Any) -> Any:
301
+ if hasattr(value, "dtype"):
302
+ return value
303
+ if value is None:
304
+ return None
305
+ value = int(value)
306
+ return value if value >= 0 else None
307
+
308
+
309
+ def _field_or_parse(sample: dict, key: str, parser) -> Any:
310
+ if key in sample:
311
+ return sample[key]
312
+ return parser(sample)
313
+
314
+
315
+ class DCASE2025Task1Adapter:
316
+ dataset_name = "dcase2025_task1"
317
+ class_names = (
318
+ "airport",
319
+ "shopping_mall",
320
+ "metro_station",
321
+ "street_pedestrian",
322
+ "public_square",
323
+ "street_traffic",
324
+ "tram",
325
+ "bus",
326
+ "metro",
327
+ "park",
328
+ )
329
+
330
+ allow_stats_on_split = allow_stats_on_split
331
+
332
+ def __call__(self, sample: dict) -> dict:
333
+ has_audio = any(key in sample for key in ("waveform", "audio", PATH))
334
+ result = (
335
+ adapt_acoustic_sample(sample, dataset=self.dataset_name, label_key=None)
336
+ if has_audio
337
+ else dict(sample)
338
+ )
339
+
340
+ metadata = dict(result.get(METADATA, {}))
341
+ split = sample.get(SPLIT)
342
+ if split is None:
343
+ split = _row_value(sample, SPLIT, "subset", "partition")
344
+ filename = sample.get(FILENAME)
345
+ if filename is None:
346
+ filename = _filename_from_row(sample)
347
+
348
+ scene_label = sample.get("scene_label")
349
+ if scene_label is None:
350
+ scene_label = parse_scene_from_row(sample)
351
+ scene_id = sample.get("scene_id")
352
+ if scene_id is None:
353
+ if hasattr(scene_label, "dtype"):
354
+ scene_id = _tensor_scene_id(scene_label)
355
+ else:
356
+ scene_id = _scene_id(scene_label)
357
+
358
+ device = sample.get("device")
359
+ if device is None:
360
+ device = parse_device_from_row(sample)
361
+ if not hasattr(device, "dtype"):
362
+ device = parse_device_from_row({**sample, "device": device})
363
+
364
+ if hasattr(device, "dtype"):
365
+ device_type = _tensor_device_type(device)
366
+ is_known_device = _tensor_known_device(device)
367
+ else:
368
+ device_type = _device_type(device)
369
+ is_known_device = device in known_train_devices
370
+
371
+ if LABEL in sample:
372
+ result[LABEL] = tf.cast(sample[LABEL], tf.int64)
373
+ elif not hasattr(scene_id, "dtype") and scene_id is not None:
374
+ result[LABEL] = scene_id
375
+
376
+ metadata.update(
377
+ {
378
+ "dataset": tf.constant(self.dataset_name)
379
+ if hasattr(device, "dtype")
380
+ else self.dataset_name,
381
+ "split": _optional_text(split),
382
+ "filename": _optional_text(filename),
383
+ "scene_label": _optional_text(scene_label),
384
+ "scene_id": _optional_int(scene_id),
385
+ "device": device,
386
+ "device_type": device_type,
387
+ "is_known_device": is_known_device,
388
+ "city": _optional_text(
389
+ _field_or_parse(sample, "city", parse_city_from_row)
390
+ ),
391
+ "location_id": _optional_text(
392
+ _field_or_parse(sample, "location_id", _parse_location_id)
393
+ ),
394
+ "segment_id": _optional_text(
395
+ _field_or_parse(sample, "segment_id", _parse_segment_id)
396
+ ),
397
+ "source_recording_id": _optional_text(
398
+ _field_or_parse(
399
+ sample, "source_recording_id", parse_source_recording_id
400
+ )
401
+ ),
402
+ "fold": _optional_text(sample.get("fold")),
403
+ }
404
+ )
405
+ result[METADATA] = metadata
406
+ return result
407
+
408
+
409
+ _ADAPTER = DCASE2025Task1Adapter()
410
+
411
+
412
+ @register_adapter("dcase2025_task1")
413
+ @register_adapter("dcase2025:")
414
+ def dcase2025_task1_adapter(sample: dict) -> dict:
415
+ return _ADAPTER(sample)
416
+
417
+
418
+ def assert_stats_allowed(split: str, *, allow_override: bool = False) -> None:
419
+ if split not in allow_stats_on_split:
420
+ raise ValueError(f"Unknown DCASE 2025 split: {split!r}.")
421
+ if allow_stats_on_split[split]:
422
+ return
423
+ message = (
424
+ f"Statistics on split {split!r} are not DCASE 2025 Task 1 compliant. "
425
+ "Use dev_train_25 for training, normalization statistics, and adaptation."
426
+ )
427
+ if allow_override:
428
+ logger.warning(f"{message} Proceeding because allow_override=True.")
429
+ warnings.warn(message, RuntimeWarning, stacklevel=2)
430
+ return
431
+ raise SplitLeakageError(message)
432
+
433
+
434
+ def _register_presets() -> None:
435
+ native = {
436
+ "name": "dcase2025_task1_native_44k_1s",
437
+ "input_duration": 1.0,
438
+ "target_sample_rate": 44100,
439
+ "preprocess": {
440
+ "target_sample_rate": 44100,
441
+ "resampler": "tensorflow",
442
+ "channel_strategy": "mono_mean",
443
+ },
444
+ "segment": {
445
+ "clip_duration": 1.0,
446
+ "train_mode": "random_crop",
447
+ "eval_mode": "center_crop",
448
+ "pad_mode": "zero",
449
+ "pad_position": "right",
450
+ },
451
+ "frontend": {
452
+ "name": "logmel",
453
+ "stft": {
454
+ "sample_rate": 44100,
455
+ "n_fft": 1024,
456
+ "win_length": 1024,
457
+ "hop_length": 320,
458
+ "center": True,
459
+ },
460
+ "mel": {"n_mels": 128, "f_min": 0.0, "f_max": 14000.0},
461
+ "log": {"kind": "log"},
462
+ },
463
+ "label_transform": {
464
+ "mode": "index",
465
+ "num_classes": len(DCASE2025Task1Adapter.class_names),
466
+ "class_names": DCASE2025Task1Adapter.class_names,
467
+ },
468
+ "layout": "btf",
469
+ "output_key": "inputs",
470
+ "dtype": "float32",
471
+ "metadata_mode": "numeric_only",
472
+ "metadata": {
473
+ "preset_version": 1,
474
+ "model_family": "generic",
475
+ "frontend_contract": "dcase2025-native-logmel-44k-v1",
476
+ "compatibility_status": "declared",
477
+ },
478
+ }
479
+ register_preset(
480
+ "dcase2025_task1_native_44k_1s",
481
+ AudioPreset.from_dict(native),
482
+ )
483
+
484
+
485
+ _register_presets()
486
+
487
+
488
+ def _read_manifest(path: Path) -> list[dict[str, str]]:
489
+ suffix = path.suffix.lower()
490
+ if suffix == ".jsonl":
491
+ rows = []
492
+ with path.open("r", encoding="utf-8") as handle:
493
+ for line in handle:
494
+ line = line.strip()
495
+ if line:
496
+ rows.append(
497
+ {
498
+ str(key): "" if value is None else str(value)
499
+ for key, value in json.loads(line).items()
500
+ }
501
+ )
502
+ return rows
503
+
504
+ dialect = csv.excel_tab if suffix in {".tsv", ".tab"} else csv.excel
505
+ with path.open("r", encoding="utf-8", newline="") as handle:
506
+ reader = csv.DictReader(handle, dialect=dialect)
507
+ return [
508
+ {
509
+ str(key): "" if value is None else str(value)
510
+ for key, value in row.items()
511
+ }
512
+ for row in reader
513
+ ]
514
+
515
+
516
+ def _resolve_manifest_path(
517
+ dataset_name: str,
518
+ data_dir: Union[None, str, os.PathLike],
519
+ ) -> Path:
520
+ if dataset_name.startswith("dcase2025:"):
521
+ spec = dataset_name[len("dcase2025:") :]
522
+ path = Path(spec).expanduser() if spec else Path(data_dir or "")
523
+ if not path.is_absolute() and data_dir is not None:
524
+ candidate = Path(data_dir).expanduser() / path
525
+ if candidate.exists() or not path.exists():
526
+ path = candidate
527
+ elif data_dir is not None:
528
+ path = Path(data_dir).expanduser()
529
+ else:
530
+ raise ValueError(
531
+ "dcase2025_task1 loading requires data_dir or a dcase2025: manifest path."
532
+ )
533
+
534
+ if path.is_dir():
535
+ for name in (
536
+ "manifest.csv",
537
+ "metadata.csv",
538
+ "dcase2025_task1.csv",
539
+ "manifest.tsv",
540
+ "metadata.tsv",
541
+ "manifest.jsonl",
542
+ "metadata.jsonl",
543
+ ):
544
+ candidate = path / name
545
+ if candidate.exists():
546
+ return candidate
547
+ raise ValueError(f"No DCASE manifest found in {path}.")
548
+ return path
549
+
550
+
551
+ def _resolve_audio_path(
552
+ row: dict, manifest_path: Path, data_dir: Union[None, str, os.PathLike]
553
+ ) -> str:
554
+ value = _row_value(row, PATH, "audio_path", "filepath", "file_path", FILENAME)
555
+ if value is None:
556
+ raise ValueError(
557
+ "DCASE manifest rows must contain a path, audio_path, or filename."
558
+ )
559
+ path = Path(value).expanduser()
560
+ if path.is_absolute():
561
+ return os.fspath(path)
562
+ manifest_relative = manifest_path.parent / path
563
+ if manifest_relative.exists() or data_dir is None:
564
+ return os.fspath(manifest_relative)
565
+ return os.fspath(Path(data_dir).expanduser() / path)
566
+
567
+
568
+ def _float_value(value: str | None) -> float:
569
+ return 0.0 if value is None else float(value)
570
+
571
+
572
+ def _record(row: dict[str, str], manifest_path: Path, split: str, data_dir) -> dict:
573
+ audio_path = _resolve_audio_path(row, manifest_path, data_dir)
574
+ filename = _row_value(row, FILENAME, "filename") or Path(audio_path).name
575
+ scene_label = parse_scene_from_row(row)
576
+ scene_id = _scene_id(scene_label)
577
+ device = parse_device_from_row({**row, SPLIT: split})
578
+ segment_id = _parse_segment_id(row) or Path(filename).stem
579
+ source_recording_id = parse_source_recording_id(row) or segment_id
580
+
581
+ record = {
582
+ PATH: audio_path,
583
+ SPLIT: split,
584
+ DATASET: "dcase2025_task1",
585
+ EXAMPLE_ID: _row_value(row, EXAMPLE_ID, "example_id")
586
+ or f"{split}:{Path(filename).stem}",
587
+ CLIP_ID: _row_value(row, CLIP_ID, "clip_id") or segment_id,
588
+ SOURCE_ID: _row_value(row, SOURCE_ID, "source_id") or source_recording_id,
589
+ START_TIME: _float_value(_row_value(row, START_TIME, "start")),
590
+ END_TIME: _float_value(_row_value(row, END_TIME, "end")),
591
+ FILENAME: filename,
592
+ "scene_label": scene_label or "",
593
+ "scene_id": -1 if scene_id is None else scene_id,
594
+ "device": device,
595
+ "device_type": _device_type(device),
596
+ "is_known_device": device in known_train_devices,
597
+ "city": parse_city_from_row(row) or "",
598
+ "location_id": _parse_location_id(row) or "",
599
+ "segment_id": segment_id,
600
+ "source_recording_id": source_recording_id,
601
+ "fold": _row_value(row, "fold") or "",
602
+ }
603
+ if scene_id is not None:
604
+ record[LABEL] = np.int64(scene_id)
605
+ return record
606
+
607
+
608
+ def _records_to_dataset(records: list[dict]) -> tf.data.Dataset:
609
+ has_label = any(LABEL in record for record in records)
610
+ keys = [
611
+ PATH,
612
+ SPLIT,
613
+ DATASET,
614
+ EXAMPLE_ID,
615
+ CLIP_ID,
616
+ SOURCE_ID,
617
+ START_TIME,
618
+ END_TIME,
619
+ FILENAME,
620
+ "scene_label",
621
+ "scene_id",
622
+ "device",
623
+ "device_type",
624
+ "is_known_device",
625
+ "city",
626
+ "location_id",
627
+ "segment_id",
628
+ "source_recording_id",
629
+ "fold",
630
+ ]
631
+ if has_label:
632
+ keys.append(LABEL)
633
+
634
+ def gen():
635
+ for record in records:
636
+ item = {}
637
+ for key in keys:
638
+ if key == LABEL:
639
+ item[key] = np.int64(record.get(key, -1))
640
+ else:
641
+ item[key] = record[key]
642
+ yield item
643
+
644
+ signature = {
645
+ PATH: tf.TensorSpec(shape=(), dtype=tf.string),
646
+ SPLIT: tf.TensorSpec(shape=(), dtype=tf.string),
647
+ DATASET: tf.TensorSpec(shape=(), dtype=tf.string),
648
+ EXAMPLE_ID: tf.TensorSpec(shape=(), dtype=tf.string),
649
+ CLIP_ID: tf.TensorSpec(shape=(), dtype=tf.string),
650
+ SOURCE_ID: tf.TensorSpec(shape=(), dtype=tf.string),
651
+ START_TIME: tf.TensorSpec(shape=(), dtype=tf.float32),
652
+ END_TIME: tf.TensorSpec(shape=(), dtype=tf.float32),
653
+ FILENAME: tf.TensorSpec(shape=(), dtype=tf.string),
654
+ "scene_label": tf.TensorSpec(shape=(), dtype=tf.string),
655
+ "scene_id": tf.TensorSpec(shape=(), dtype=tf.int64),
656
+ "device": tf.TensorSpec(shape=(), dtype=tf.string),
657
+ "device_type": tf.TensorSpec(shape=(), dtype=tf.string),
658
+ "is_known_device": tf.TensorSpec(shape=(), dtype=tf.bool),
659
+ "city": tf.TensorSpec(shape=(), dtype=tf.string),
660
+ "location_id": tf.TensorSpec(shape=(), dtype=tf.string),
661
+ "segment_id": tf.TensorSpec(shape=(), dtype=tf.string),
662
+ "source_recording_id": tf.TensorSpec(shape=(), dtype=tf.string),
663
+ "fold": tf.TensorSpec(shape=(), dtype=tf.string),
664
+ }
665
+ if has_label:
666
+ signature[LABEL] = tf.TensorSpec(shape=(), dtype=tf.int64)
667
+
668
+ ds = tf.data.Dataset.from_generator(gen, output_signature=signature)
669
+ return ds.apply(tf.data.experimental.assert_cardinality(len(records)))
670
+
671
+
672
+ def load_dcase2025_task1_splits(
673
+ dataset_name: str,
674
+ splits: list[str],
675
+ data_dir: Union[None, str, os.PathLike] = None,
676
+ ) -> list[tf.data.Dataset]:
677
+ manifest_path = _resolve_manifest_path(dataset_name, data_dir)
678
+ rows = _read_manifest(manifest_path)
679
+ datasets = []
680
+ for split in splits:
681
+ split_rows = [
682
+ row
683
+ for row in rows
684
+ if (_row_value(row, SPLIT, "subset", "partition") or split) == split
685
+ ]
686
+ records = [_record(row, manifest_path, split, data_dir) for row in split_rows]
687
+ datasets.append(_records_to_dataset(records))
688
+ return datasets
689
+
690
+
691
+ @register_source_loader("dcase2025_task1")
692
+ def _load_dcase2025_task1_splits(
693
+ dataset_name: str,
694
+ splits: list[str],
695
+ data_dir: Union[None, str, os.PathLike] = None,
696
+ ) -> list[tf.data.Dataset]:
697
+ return load_dcase2025_task1_splits(dataset_name, splits, data_dir)
698
+
699
+
700
+ def _dataset_type_for_split(split: str, source: bool) -> str:
701
+ if source and split == "dev_train_25":
702
+ return "train"
703
+ return "validation"
704
+
705
+
706
+ def _make_dataset(
707
+ *,
708
+ dataset: str,
709
+ split: str,
710
+ domain: dict[str, Any] | None,
711
+ preset: str,
712
+ allow_stats: bool,
713
+ allow_override: bool,
714
+ source: bool,
715
+ load_kwargs: dict[str, Any],
716
+ ):
717
+ if allow_stats:
718
+ assert_stats_allowed(split, allow_override=allow_override)
719
+
720
+ filter_fn = load_kwargs.pop("filter_fn", None)
721
+ source_filter_fn = load_kwargs.pop("source_filter_fn", None)
722
+ if domain:
723
+ domain_filter = metadata_filter_predicate(**domain)
724
+ if source_filter_fn is None:
725
+ source_filter_fn = domain_filter
726
+ else:
727
+ previous_source_filter = source_filter_fn
728
+
729
+ def source_filter_fn(sample):
730
+ return tf.logical_and(
731
+ previous_source_filter(sample),
732
+ domain_filter(sample),
733
+ )
734
+
735
+ pipeline = load_kwargs.pop("pipeline", None)
736
+ if pipeline is None:
737
+ pipeline = get_pipeline(dataset=dataset, preset=preset)
738
+
739
+ return load_ds(
740
+ dataset,
741
+ split,
742
+ load_kwargs.pop("dataset_type", _dataset_type_for_split(split, source=source)),
743
+ load_kwargs.pop("batch_size", 32),
744
+ load_kwargs.pop("seed", 0),
745
+ pipeline=pipeline,
746
+ deterministic=load_kwargs.pop("deterministic", True),
747
+ cache_dataset=load_kwargs.pop("cache_dataset", False),
748
+ source_filter_fn=source_filter_fn,
749
+ filter_fn=filter_fn,
750
+ **load_kwargs,
751
+ )
752
+
753
+
754
+ def make_source_dataset(
755
+ dataset: str = "dcase2025_task1",
756
+ split: str = "dev_train_25",
757
+ source_domain: dict[str, Any] | None = None,
758
+ preset: str = "dcase2025_task1_efficientat_32k_1s",
759
+ *,
760
+ allow_stats: bool = True,
761
+ allow_override: bool = False,
762
+ **load_kwargs,
763
+ ):
764
+ return _make_dataset(
765
+ dataset=dataset,
766
+ split=split,
767
+ domain=source_domain,
768
+ preset=preset,
769
+ allow_stats=allow_stats,
770
+ allow_override=allow_override,
771
+ source=True,
772
+ load_kwargs=dict(load_kwargs),
773
+ )
774
+
775
+
776
+ def make_target_dataset(
777
+ dataset: str = "dcase2025_task1",
778
+ split: str = "dev_test",
779
+ target_domain: dict[str, Any] | None = None,
780
+ preset: str = "dcase2025_task1_efficientat_32k_1s",
781
+ *,
782
+ allow_stats: bool = False,
783
+ allow_override: bool = False,
784
+ **load_kwargs,
785
+ ):
786
+ return _make_dataset(
787
+ dataset=dataset,
788
+ split=split,
789
+ domain=target_domain,
790
+ preset=preset,
791
+ allow_stats=allow_stats,
792
+ allow_override=allow_override,
793
+ source=False,
794
+ load_kwargs=dict(load_kwargs),
795
+ )
796
+
797
+
798
+ __all__ = [
799
+ "DCASE2025Task1Adapter",
800
+ "SplitLeakageError",
801
+ "allow_stats_on_split",
802
+ "assert_stats_allowed",
803
+ "dcase2025_task1_adapter",
804
+ "known_train_devices",
805
+ "load_dcase2025_task1_splits",
806
+ "make_source_dataset",
807
+ "make_target_dataset",
808
+ "parse_city_from_row",
809
+ "parse_device_from_row",
810
+ "parse_scene_from_row",
811
+ "parse_source_recording_id",
812
+ "real_devices",
813
+ "simulated_devices",
814
+ ]