justdata 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. justdata/__init__.py +3 -0
  2. justdata/acoustic/__init__.py +381 -0
  3. justdata/acoustic/adapters.py +329 -0
  4. justdata/acoustic/augment/__init__.py +119 -0
  5. justdata/acoustic/augment/batch.py +813 -0
  6. justdata/acoustic/augment/configs.py +204 -0
  7. justdata/acoustic/augment/patchout.py +331 -0
  8. justdata/acoustic/augment/spectrogram.py +806 -0
  9. justdata/acoustic/augment/waveform.py +879 -0
  10. justdata/acoustic/batching.py +27 -0
  11. justdata/acoustic/channel.py +53 -0
  12. justdata/acoustic/compat/__init__.py +95 -0
  13. justdata/acoustic/compat/ast.py +626 -0
  14. justdata/acoustic/compat/ced.py +152 -0
  15. justdata/acoustic/compat/efficientat.py +243 -0
  16. justdata/acoustic/compat/panns.py +115 -0
  17. justdata/acoustic/compat/passt.py +191 -0
  18. justdata/acoustic/configs.py +519 -0
  19. justdata/acoustic/corruptions/__init__.py +33 -0
  20. justdata/acoustic/corruptions/codec.py +92 -0
  21. justdata/acoustic/corruptions/datasets.py +155 -0
  22. justdata/acoustic/corruptions/device.py +39 -0
  23. justdata/acoustic/corruptions/dynamics.py +55 -0
  24. justdata/acoustic/corruptions/filters.py +105 -0
  25. justdata/acoustic/corruptions/noise.py +173 -0
  26. justdata/acoustic/corruptions/registry.py +230 -0
  27. justdata/acoustic/corruptions/reverb.py +49 -0
  28. justdata/acoustic/corruptions/timefreq.py +127 -0
  29. justdata/acoustic/datasets.py +52 -0
  30. justdata/acoustic/dcase2025.py +814 -0
  31. justdata/acoustic/decoding.py +73 -0
  32. justdata/acoustic/eval_views.py +54 -0
  33. justdata/acoustic/evals.py +71 -0
  34. justdata/acoustic/filters.py +18 -0
  35. justdata/acoustic/frontends/__init__.py +37 -0
  36. justdata/acoustic/frontends/ast.py +116 -0
  37. justdata/acoustic/frontends/kaldi.py +34 -0
  38. justdata/acoustic/frontends/log.py +94 -0
  39. justdata/acoustic/frontends/mel.py +173 -0
  40. justdata/acoustic/frontends/mfcc.py +26 -0
  41. justdata/acoustic/frontends/pcen.py +26 -0
  42. justdata/acoustic/frontends/stft.py +174 -0
  43. justdata/acoustic/jax.py +23 -0
  44. justdata/acoustic/labels.py +169 -0
  45. justdata/acoustic/layouts.py +86 -0
  46. justdata/acoustic/metadata.py +152 -0
  47. justdata/acoustic/normalization.py +196 -0
  48. justdata/acoustic/pipelines.py +112 -0
  49. justdata/acoustic/postprocessing.py +328 -0
  50. justdata/acoustic/preprocessing.py +175 -0
  51. justdata/acoustic/presets.py +257 -0
  52. justdata/acoustic/registry.py +381 -0
  53. justdata/acoustic/resampling.py +218 -0
  54. justdata/acoustic/schema.py +51 -0
  55. justdata/acoustic/segment.py +369 -0
  56. justdata/acoustic/sources.py +503 -0
  57. justdata/acoustic/stages.py +59 -0
  58. justdata/acoustic/stats.py +50 -0
  59. justdata/acoustic/tasks.py +375 -0
  60. justdata/audio/__init__.py +20 -0
  61. justdata/core/__init__.py +74 -0
  62. justdata/core/adapters.py +46 -0
  63. justdata/core/augmentations.py +34 -0
  64. justdata/core/filters.py +94 -0
  65. justdata/core/finalization.py +173 -0
  66. justdata/core/label_mixing.py +129 -0
  67. justdata/core/loader.py +384 -0
  68. justdata/core/metadata.py +250 -0
  69. justdata/core/presets.py +155 -0
  70. justdata/core/registry.py +229 -0
  71. justdata/core/sources.py +87 -0
  72. justdata/core/stats.py +227 -0
  73. justdata/corruptions.py +52 -0
  74. justdata/py.typed +0 -0
  75. justdata/vision/__init__.py +17 -0
  76. justdata/vision/augmentations/__init__.py +31 -0
  77. justdata/vision/augmentations/auto.py +480 -0
  78. justdata/vision/augmentations/color.py +163 -0
  79. justdata/vision/augmentations/composed.py +156 -0
  80. justdata/vision/augmentations/geometric.py +302 -0
  81. justdata/vision/augmentations/mixing.py +232 -0
  82. justdata/vision/augmentations/registry.py +142 -0
  83. justdata/vision/corruptions/__init__.py +16 -0
  84. justdata/vision/corruptions/blur.py +65 -0
  85. justdata/vision/corruptions/digital.py +26 -0
  86. justdata/vision/corruptions/noise.py +20 -0
  87. justdata/vision/corruptions/registry.py +113 -0
  88. justdata/vision/corruptions/weather.py +100 -0
  89. justdata/vision/datasets.py +34 -0
  90. justdata/vision/encodings/__init__.py +1 -0
  91. justdata/vision/encodings/heatmaps.py +196 -0
  92. justdata/vision/minic.py +69 -0
  93. justdata/vision/pipelines.py +62 -0
  94. justdata/vision/presets.py +509 -0
  95. justdata/vision/sources.py +1080 -0
  96. justdata/vision/stages.py +271 -0
  97. justdata/vision/tasks/__init__.py +1 -0
  98. justdata/vision/tasks/classification.py +321 -0
  99. justdata/vision/tasks/depth.py +25 -0
  100. justdata/vision/tasks/segmentation.py +194 -0
  101. justdata/vision/transforms.py +113 -0
  102. justdata/vision/utils.py +741 -0
  103. justdata-1.0.0.dist-info/METADATA +860 -0
  104. justdata-1.0.0.dist-info/RECORD +106 -0
  105. justdata-1.0.0.dist-info/WHEEL +4 -0
  106. justdata-1.0.0.dist-info/licenses/LICENSE +23 -0
@@ -0,0 +1,250 @@
1
+ from __future__ import annotations
2
+
3
+ import hashlib
4
+ import json
5
+ import threading
6
+ from collections.abc import Mapping
7
+ from pathlib import Path
8
+ from typing import Any, Literal
9
+
10
+ import tensorflow as tf
11
+
12
+
13
+ def python_value(value: Any) -> Any:
14
+ if hasattr(value, "numpy"):
15
+ value = value.numpy()
16
+ if isinstance(value, bytes):
17
+ return value.decode("utf-8")
18
+ if hasattr(value, "tolist"):
19
+ return python_value(value.tolist())
20
+ if isinstance(value, (list, tuple)):
21
+ return [python_value(child) for child in value]
22
+ if hasattr(value, "item"):
23
+ return value.item()
24
+ return value
25
+
26
+
27
+ def get_metadata_value(sample: dict, key: str) -> Any | None:
28
+ metadata = sample.get("metadata")
29
+ if isinstance(metadata, dict) and key in metadata:
30
+ return metadata[key]
31
+ return sample.get(key)
32
+
33
+
34
+ def _is_numeric_metadata_value(value: Any) -> bool:
35
+ return hasattr(value, "dtype") and value.dtype != tf.string
36
+
37
+
38
+ def numeric_metadata(metadata: dict) -> dict:
39
+ numeric = {}
40
+ for key, value in metadata.items():
41
+ if isinstance(value, dict):
42
+ child = numeric_metadata(value)
43
+ if child:
44
+ numeric[key] = child
45
+ elif _is_numeric_metadata_value(value):
46
+ numeric[key] = value
47
+ return numeric
48
+
49
+
50
+ def apply_metadata_mode(
51
+ sample: dict,
52
+ metadata_mode: Literal["full", "numeric_only", "none"],
53
+ ) -> dict:
54
+ if metadata_mode == "full" or "metadata" not in sample:
55
+ return sample
56
+
57
+ result = dict(sample)
58
+ if metadata_mode == "none":
59
+ result.pop("metadata", None)
60
+ return result
61
+
62
+ metadata = result.get("metadata")
63
+ if isinstance(metadata, dict):
64
+ metadata = numeric_metadata(metadata)
65
+ if metadata:
66
+ result["metadata"] = metadata
67
+ else:
68
+ result.pop("metadata", None)
69
+ return result
70
+
71
+
72
+ def _flatten_metadata(metadata: dict, prefix: tuple[str, ...] = ()):
73
+ leaves = []
74
+ for key, value in metadata.items():
75
+ path = prefix + (key,)
76
+ if isinstance(value, dict):
77
+ leaves.extend(_flatten_metadata(value, path))
78
+ else:
79
+ leaves.append((path, value))
80
+ return leaves
81
+
82
+
83
+ def _identity_structure(value):
84
+ if isinstance(value, dict):
85
+ return {key: _identity_structure(child) for key, child in value.items()}
86
+ return tf.identity(value)
87
+
88
+
89
+ def json_value(value: Any) -> Any:
90
+ value = python_value(value)
91
+ if isinstance(value, Mapping):
92
+ return {str(key): json_value(child) for key, child in value.items()}
93
+ if isinstance(value, (list, tuple)):
94
+ return [json_value(child) for child in value]
95
+ return value
96
+
97
+
98
+ def stable_int64_hash(value: Any) -> int:
99
+ """Return a process-stable non-negative int64 hash for metadata ids."""
100
+ value = python_value(value)
101
+ if value is None:
102
+ value = ""
103
+ digest = hashlib.sha256(str(value).encode("utf-8")).digest()
104
+ return int.from_bytes(digest[:8], "big", signed=False) & ((1 << 63) - 1)
105
+
106
+
107
+ def _assign_nested(target: dict, path: tuple[str, ...], value: Any) -> None:
108
+ current = target
109
+ for key in path[:-1]:
110
+ current = current.setdefault(key, {})
111
+ current[path[-1]] = value
112
+
113
+
114
+ def _sidecar_example_id(values_by_path: dict[tuple[str, ...], Any]) -> int:
115
+ by_name = {path[-1]: value for path, value in values_by_path.items()}
116
+ composite_values = [
117
+ python_value(by_name.get(key)) for key in ("dataset", "split", "clip_id")
118
+ ]
119
+ if all(value is not None for value in composite_values):
120
+ return stable_int64_hash(
121
+ f"{composite_values[0]}::{composite_values[1]}::{composite_values[2]}"
122
+ )
123
+
124
+ value = python_value(by_name.get("example_id"))
125
+ if isinstance(value, int) and not isinstance(value, bool):
126
+ return value
127
+ if isinstance(value, str):
128
+ return stable_int64_hash(value)
129
+ raise ValueError(
130
+ "Sidecar metadata requires either non-null 'dataset', 'split', and "
131
+ "'clip_id' fields or an integer/string 'example_id'."
132
+ )
133
+
134
+
135
+ class MetadataSidecar:
136
+ def __init__(self) -> None:
137
+ self.records: dict[int, dict] = {}
138
+
139
+ def add(self, example_id: int, metadata: dict) -> None:
140
+ example_id = int(example_id)
141
+ metadata = json_value(metadata)
142
+ existing = self.records.get(example_id)
143
+ if existing is not None and existing != metadata:
144
+ raise ValueError(f"Conflicting metadata for sidecar ID {example_id}.")
145
+ self.records[example_id] = metadata
146
+
147
+ def write_jsonl(self, path: str) -> None:
148
+ output = Path(path)
149
+ if output.parent != Path("."):
150
+ output.parent.mkdir(parents=True, exist_ok=True)
151
+ with output.open("w", encoding="utf-8") as handle:
152
+ for example_id in sorted(self.records):
153
+ handle.write(
154
+ json.dumps(
155
+ {
156
+ "example_id": example_id,
157
+ "metadata": self.records[example_id],
158
+ },
159
+ sort_keys=True,
160
+ )
161
+ + "\n"
162
+ )
163
+
164
+ @classmethod
165
+ def read_jsonl(cls, path: str) -> "MetadataSidecar":
166
+ sidecar = cls()
167
+ with Path(path).open("r", encoding="utf-8") as handle:
168
+ for line in handle:
169
+ if not line.strip():
170
+ continue
171
+ record = json.loads(line)
172
+ sidecar.add(record["example_id"], record.get("metadata", {}))
173
+ return sidecar
174
+
175
+
176
+ def attach_sidecar_writer(ds: tf.data.Dataset, path: str) -> tf.data.Dataset:
177
+ output = Path(path)
178
+ if output.parent != Path("."):
179
+ output.parent.mkdir(parents=True, exist_ok=True)
180
+ output.write_text("", encoding="utf-8")
181
+ lock = threading.Lock()
182
+ serialized_by_id: dict[int, str] = {}
183
+
184
+ def add_writer(sample):
185
+ metadata = sample.get("metadata")
186
+ if not isinstance(metadata, dict):
187
+ return sample
188
+
189
+ leaves = _flatten_metadata(metadata)
190
+ string_leaves = [
191
+ (leaf_path, value)
192
+ for leaf_path, value in leaves
193
+ if value.dtype == tf.string
194
+ ]
195
+ if not string_leaves:
196
+ return sample
197
+
198
+ all_paths = [leaf_path for leaf_path, _ in leaves]
199
+ all_values = [value for _, value in leaves]
200
+ string_paths = {leaf_path for leaf_path, _ in string_leaves}
201
+
202
+ def write_record(*values):
203
+ values_by_path = {
204
+ leaf_path: python_value(value)
205
+ for leaf_path, value in zip(all_paths, values)
206
+ }
207
+ string_metadata = {}
208
+ for leaf_path in string_paths:
209
+ _assign_nested(
210
+ string_metadata,
211
+ leaf_path,
212
+ values_by_path[leaf_path],
213
+ )
214
+
215
+ example_id = _sidecar_example_id(values_by_path)
216
+ record = {
217
+ "example_id": example_id,
218
+ "metadata": string_metadata,
219
+ }
220
+ serialized = json.dumps(record, sort_keys=True)
221
+ with lock:
222
+ existing = serialized_by_id.get(example_id)
223
+ if existing is not None:
224
+ if existing == serialized:
225
+ return 0
226
+ raise ValueError(
227
+ f"Conflicting metadata for sidecar ID {example_id}."
228
+ )
229
+ with output.open("a", encoding="utf-8") as handle:
230
+ handle.write(serialized + "\n")
231
+ serialized_by_id[example_id] = serialized
232
+ return 0
233
+
234
+ marker = tf.py_function(write_record, all_values, Tout=tf.int64)
235
+ with tf.control_dependencies([marker]):
236
+ return _identity_structure(sample)
237
+
238
+ return ds.map(add_writer, num_parallel_calls=tf.data.AUTOTUNE)
239
+
240
+
241
+ __all__ = [
242
+ "apply_metadata_mode",
243
+ "attach_sidecar_writer",
244
+ "get_metadata_value",
245
+ "json_value",
246
+ "MetadataSidecar",
247
+ "numeric_metadata",
248
+ "python_value",
249
+ "stable_int64_hash",
250
+ ]
@@ -0,0 +1,155 @@
1
+ import copy
2
+ import dataclasses
3
+ import hashlib
4
+ import json
5
+ import threading
6
+ from collections.abc import Mapping
7
+ from dataclasses import dataclass
8
+ from typing import Any, Dict
9
+
10
+ _PRESETS: Dict[str, Dict[str, Any]] = {}
11
+ _PRESET_LOCK = threading.Lock()
12
+
13
+
14
+ def _plain_value(value: Any) -> Any:
15
+ if dataclasses.is_dataclass(value):
16
+ value = dataclasses.asdict(value)
17
+
18
+ if isinstance(value, Mapping):
19
+ return {str(k): _plain_value(v) for k, v in value.items()}
20
+
21
+ if isinstance(value, (tuple, list)):
22
+ return [_plain_value(v) for v in value]
23
+
24
+ return value
25
+
26
+
27
+ def canonical_preset_json(config: Any) -> str:
28
+ return json.dumps(_plain_value(config), sort_keys=True, separators=(",", ":"))
29
+
30
+
31
+ def preset_hash(config: Any) -> str:
32
+ return hashlib.sha256(canonical_preset_json(config).encode("utf-8")).hexdigest()[
33
+ :16
34
+ ]
35
+
36
+
37
+ @dataclass(frozen=True)
38
+ class ResolvedPreset(Mapping[str, Any]):
39
+ name: str
40
+ modality: str
41
+ config: Any
42
+
43
+ def to_dict(self) -> Dict[str, Any]:
44
+ plain = _plain_value(self.config)
45
+ return copy.deepcopy(plain)
46
+
47
+ def to_json(self) -> str:
48
+ return canonical_preset_json(self.config)
49
+
50
+ def hash(self) -> str:
51
+ return preset_hash(self.config)
52
+
53
+ def __getitem__(self, key: str) -> Any:
54
+ return self.to_dict()[key]
55
+
56
+ def __iter__(self):
57
+ return iter(self.to_dict())
58
+
59
+ def __len__(self) -> int:
60
+ return len(self.to_dict())
61
+
62
+
63
+ def register_preset(dataset: str, config: Any, *, modality: str = "core"):
64
+ key = dataset.lower()
65
+ with _PRESET_LOCK:
66
+ presets = _PRESETS.setdefault(modality, {})
67
+ if key in presets:
68
+ raise ValueError(
69
+ f"Preset '{key}' already registered for modality '{modality}'"
70
+ )
71
+ presets[key] = copy.deepcopy(config)
72
+
73
+
74
+ def _find_preset(dataset: str, *, modality: str = "core") -> tuple[str | None, Any]:
75
+ presets = _PRESETS.get(modality, {})
76
+ dataset = dataset.lower()
77
+
78
+ if dataset in presets:
79
+ return dataset, presets[dataset]
80
+
81
+ for key, val in sorted(presets.items(), key=lambda kv: len(kv[0]), reverse=True):
82
+ if key != "_default" and dataset.startswith(key):
83
+ return key, val
84
+
85
+ if "_default" in presets:
86
+ return "_default", presets["_default"]
87
+
88
+ return None, {}
89
+
90
+
91
+ def get_dataset_presets(dataset: str, *, modality: str = "core") -> Dict[str, Any]:
92
+ _name, config = _find_preset(dataset, modality=modality)
93
+ return copy.deepcopy(config)
94
+
95
+
96
+ def get_resolved_preset(dataset: str, *, modality: str = "core") -> ResolvedPreset:
97
+ name, config = _find_preset(dataset, modality=modality)
98
+ return ResolvedPreset(
99
+ name=name or dataset.lower(), modality=modality, config=copy.deepcopy(config)
100
+ )
101
+
102
+
103
+ def merge_with_presets(
104
+ dataset: str, user_kwargs: Dict[str, Any], *, modality: str = "core"
105
+ ) -> Dict[str, Any]:
106
+ """
107
+ Smartly merge user-provided kwargs with modality-specific presets.
108
+
109
+ If a user kwarg matches the modality default, it is treated as an
110
+ unmodified default and the dataset-specific preset takes precedence.
111
+ """
112
+
113
+ dataset_presets = copy.deepcopy(get_dataset_presets(dataset, modality=modality))
114
+ default_presets = get_dataset_presets("_default", modality=modality)
115
+
116
+ def smart_merge(base, user, default):
117
+ result = base.copy()
118
+ for k, v in user.items():
119
+ if isinstance(v, dict) and k in base and isinstance(base[k], dict):
120
+ result[k] = smart_merge(
121
+ base[k], v, default.get(k, {}) if isinstance(default, dict) else {}
122
+ )
123
+ else:
124
+ default_val = default.get(k) if isinstance(default, dict) else None
125
+ val_to_compare = tuple(v) if isinstance(v, list) else v
126
+ def_to_compare = (
127
+ tuple(default_val) if isinstance(default_val, list) else default_val
128
+ )
129
+
130
+ if (
131
+ isinstance(val_to_compare, tuple)
132
+ and len(val_to_compare) > 0
133
+ and isinstance(val_to_compare[0], list)
134
+ ):
135
+ val_to_compare = tuple(
136
+ tuple(x) if isinstance(x, list) else x for x in val_to_compare
137
+ )
138
+ if (
139
+ isinstance(def_to_compare, tuple)
140
+ and len(def_to_compare) > 0
141
+ and isinstance(def_to_compare[0], list)
142
+ ):
143
+ def_to_compare = tuple(
144
+ tuple(x) if isinstance(x, list) else x for x in def_to_compare
145
+ )
146
+
147
+ if (
148
+ k not in default
149
+ or val_to_compare != def_to_compare
150
+ or k not in result
151
+ ):
152
+ result[k] = v
153
+ return result
154
+
155
+ return smart_merge(dataset_presets, user_kwargs, default_presets)
@@ -0,0 +1,229 @@
1
+ from dataclasses import dataclass
2
+ import threading
3
+ from typing import Callable, Dict, Tuple
4
+
5
+ PipelineFuncs = Tuple[Callable, Callable, Callable, Callable]
6
+
7
+
8
+ @dataclass(frozen=True)
9
+ class DatasetInfo:
10
+ name: str
11
+ modality: str
12
+ task: str | None = None
13
+ pipeline_name: str | None = None
14
+ preset: str | None = None
15
+
16
+
17
+ _REGISTRY_LOCK = threading.Lock()
18
+ _DATASETS: Dict[str, DatasetInfo] = {}
19
+ _PIPELINES: Dict[str, Callable[..., PipelineFuncs]] = {}
20
+
21
+
22
+ def register_dataset(
23
+ name: str,
24
+ task_type: str | None = None,
25
+ *,
26
+ modality: str = "core",
27
+ pipeline_name: str | None = None,
28
+ preset: str | None = None,
29
+ ):
30
+ """Register dataset metadata at runtime."""
31
+ with _REGISTRY_LOCK:
32
+ if name in _DATASETS:
33
+ existing = _DATASETS[name]
34
+ raise ValueError(
35
+ f"Dataset '{name}' is already registered with modality "
36
+ f"'{existing.modality}' and task '{existing.task}'."
37
+ )
38
+ _DATASETS[name] = DatasetInfo(
39
+ name=name,
40
+ modality=modality,
41
+ task=task_type,
42
+ pipeline_name=pipeline_name,
43
+ preset=preset,
44
+ )
45
+
46
+
47
+ def get_dataset_info(name: str) -> DatasetInfo | None:
48
+ with _REGISTRY_LOCK:
49
+ exact = _DATASETS.get(name)
50
+ entries = tuple(_DATASETS.items())
51
+
52
+ if exact is not None:
53
+ return exact
54
+
55
+ for key, val in sorted(entries, key=lambda kv: len(kv[0]), reverse=True):
56
+ if name.startswith(key):
57
+ return val
58
+
59
+ return None
60
+
61
+
62
+ def get_task_for_dataset(name: str) -> str | None:
63
+ info = get_dataset_info(name)
64
+ return info.task if info else None
65
+
66
+
67
+ def _pipeline_modality(pipeline_name: str | None) -> str | None:
68
+ if pipeline_name and "/" in pipeline_name:
69
+ return pipeline_name.split("/", 1)[0]
70
+ return None
71
+
72
+
73
+ class DataPipeline:
74
+ """Encapsulates modality-specific pipeline logic and configuration."""
75
+
76
+ def __init__(
77
+ self,
78
+ pipeline_name: str,
79
+ *,
80
+ modality: str | None = None,
81
+ _is_training_legacy: bool = False,
82
+ **kwargs,
83
+ ):
84
+ self.pipeline_name = pipeline_name
85
+ self.modality = modality or _pipeline_modality(pipeline_name)
86
+ self.kwargs = kwargs
87
+ self._is_training_legacy = _is_training_legacy
88
+
89
+ def build(self, is_training: bool) -> PipelineFuncs:
90
+ """Return the 4 callables: preprocess, augment, batch augment, postprocess."""
91
+ import copy
92
+
93
+ config = copy.deepcopy(self.kwargs)
94
+ postproc = config.setdefault("postproc_kwargs", {})
95
+ postproc["is_training"] = is_training
96
+
97
+ with _REGISTRY_LOCK:
98
+ factory = _PIPELINES.get(self.pipeline_name)
99
+ if factory is None:
100
+ raise ValueError(f"Pipeline '{self.pipeline_name}' not found.")
101
+
102
+ return factory(**config)
103
+
104
+ def __iter__(self):
105
+ return iter(self.build(is_training=self._is_training_legacy))
106
+
107
+ def __repr__(self):
108
+ return (
109
+ f"DataPipeline(pipeline_name={self.pipeline_name!r}, "
110
+ f"modality={self.modality!r}, config={self.kwargs})"
111
+ )
112
+
113
+
114
+ def register_pipeline(name: str):
115
+ """
116
+ Decorator to register a pipeline.
117
+
118
+ A pipeline returns four callables:
119
+ (preprocess, augment, batch_augment, postprocess).
120
+ """
121
+
122
+ def decorator(fn: Callable[..., PipelineFuncs]):
123
+ with _REGISTRY_LOCK:
124
+ if name in _PIPELINES:
125
+ raise ValueError(
126
+ f"Pipeline '{name}' already registered by "
127
+ f"{_PIPELINES[name].__module__}.{_PIPELINES[name].__qualname__}"
128
+ )
129
+ _PIPELINES[name] = fn
130
+ return fn
131
+
132
+ return decorator
133
+
134
+
135
+ def list_pipelines() -> tuple[str, ...]:
136
+ with _REGISTRY_LOCK:
137
+ return tuple(sorted(_PIPELINES))
138
+
139
+
140
+ def has_pipeline(name: str) -> bool:
141
+ with _REGISTRY_LOCK:
142
+ return name in _PIPELINES
143
+
144
+
145
+ def get_pipeline(
146
+ dataset: str | None = None,
147
+ preset: str | None = None,
148
+ task: str | None = None,
149
+ pipeline_name: str | None = None,
150
+ modality: str | None = None,
151
+ apply_presets: bool = True,
152
+ **kwargs,
153
+ ) -> DataPipeline:
154
+ """
155
+ Resolve and configure a registered pipeline.
156
+
157
+ Dataset registration supplies the default modality, task, pipeline, and
158
+ preset. Explicit ``pipeline_name`` wins over explicit ``task``; both win
159
+ over dataset-driven resolution.
160
+ """
161
+ from justdata.core.presets import merge_with_presets
162
+
163
+ dataset_info = get_dataset_info(dataset) if dataset else None
164
+ effective_modality = modality or (dataset_info.modality if dataset_info else None)
165
+ effective_pipeline = pipeline_name
166
+
167
+ if effective_pipeline is None and task is not None:
168
+ effective_pipeline = (
169
+ task
170
+ if "/" in task or effective_modality is None
171
+ else f"{effective_modality}/{task}"
172
+ )
173
+
174
+ dataset_is_pipeline_name = False
175
+ if effective_pipeline is None and dataset:
176
+ if dataset_info is not None:
177
+ effective_pipeline = dataset_info.pipeline_name
178
+ if effective_pipeline is None and dataset_info.task is not None:
179
+ effective_pipeline = f"{dataset_info.modality}/{dataset_info.task}"
180
+ elif has_pipeline(dataset):
181
+ effective_pipeline = dataset
182
+ effective_modality = effective_modality or _pipeline_modality(dataset)
183
+ dataset_is_pipeline_name = True
184
+
185
+ if effective_pipeline is None:
186
+ raise ValueError(
187
+ f"Could not resolve pipeline for dataset='{dataset}' and preset='{preset}'. "
188
+ "Please specify `task` or `pipeline_name`."
189
+ )
190
+
191
+ effective_modality = effective_modality or _pipeline_modality(effective_pipeline)
192
+
193
+ preset_name = preset
194
+ if preset_name is None and dataset_info is not None:
195
+ preset_name = dataset_info.preset or dataset
196
+ elif preset_name is None and dataset and not dataset_is_pipeline_name:
197
+ preset_name = dataset
198
+
199
+ if apply_presets and preset_name and effective_modality:
200
+ kwargs = merge_with_presets(
201
+ preset_name,
202
+ kwargs,
203
+ modality=effective_modality,
204
+ )
205
+
206
+ return DataPipeline(
207
+ effective_pipeline,
208
+ modality=effective_modality,
209
+ **kwargs,
210
+ )
211
+
212
+
213
+ def get_pipeline_for_dataset(
214
+ dataset: str,
215
+ task_type: str | None = None,
216
+ pipeline_name: str | None = None,
217
+ apply_presets: bool = True,
218
+ is_training: bool = False,
219
+ **kwargs,
220
+ ) -> DataPipeline:
221
+ """Compatibility alias for call sites that still resolve by dataset."""
222
+ return get_pipeline(
223
+ dataset=dataset,
224
+ task=task_type,
225
+ pipeline_name=pipeline_name,
226
+ apply_presets=apply_presets,
227
+ _is_training_legacy=is_training,
228
+ **kwargs,
229
+ )