justdata 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- justdata/__init__.py +3 -0
- justdata/acoustic/__init__.py +381 -0
- justdata/acoustic/adapters.py +329 -0
- justdata/acoustic/augment/__init__.py +119 -0
- justdata/acoustic/augment/batch.py +813 -0
- justdata/acoustic/augment/configs.py +204 -0
- justdata/acoustic/augment/patchout.py +331 -0
- justdata/acoustic/augment/spectrogram.py +806 -0
- justdata/acoustic/augment/waveform.py +879 -0
- justdata/acoustic/batching.py +27 -0
- justdata/acoustic/channel.py +53 -0
- justdata/acoustic/compat/__init__.py +95 -0
- justdata/acoustic/compat/ast.py +626 -0
- justdata/acoustic/compat/ced.py +152 -0
- justdata/acoustic/compat/efficientat.py +243 -0
- justdata/acoustic/compat/panns.py +115 -0
- justdata/acoustic/compat/passt.py +191 -0
- justdata/acoustic/configs.py +519 -0
- justdata/acoustic/corruptions/__init__.py +33 -0
- justdata/acoustic/corruptions/codec.py +92 -0
- justdata/acoustic/corruptions/datasets.py +155 -0
- justdata/acoustic/corruptions/device.py +39 -0
- justdata/acoustic/corruptions/dynamics.py +55 -0
- justdata/acoustic/corruptions/filters.py +105 -0
- justdata/acoustic/corruptions/noise.py +173 -0
- justdata/acoustic/corruptions/registry.py +230 -0
- justdata/acoustic/corruptions/reverb.py +49 -0
- justdata/acoustic/corruptions/timefreq.py +127 -0
- justdata/acoustic/datasets.py +52 -0
- justdata/acoustic/dcase2025.py +814 -0
- justdata/acoustic/decoding.py +73 -0
- justdata/acoustic/eval_views.py +54 -0
- justdata/acoustic/evals.py +71 -0
- justdata/acoustic/filters.py +18 -0
- justdata/acoustic/frontends/__init__.py +37 -0
- justdata/acoustic/frontends/ast.py +116 -0
- justdata/acoustic/frontends/kaldi.py +34 -0
- justdata/acoustic/frontends/log.py +94 -0
- justdata/acoustic/frontends/mel.py +173 -0
- justdata/acoustic/frontends/mfcc.py +26 -0
- justdata/acoustic/frontends/pcen.py +26 -0
- justdata/acoustic/frontends/stft.py +174 -0
- justdata/acoustic/jax.py +23 -0
- justdata/acoustic/labels.py +169 -0
- justdata/acoustic/layouts.py +86 -0
- justdata/acoustic/metadata.py +152 -0
- justdata/acoustic/normalization.py +196 -0
- justdata/acoustic/pipelines.py +112 -0
- justdata/acoustic/postprocessing.py +328 -0
- justdata/acoustic/preprocessing.py +175 -0
- justdata/acoustic/presets.py +257 -0
- justdata/acoustic/registry.py +381 -0
- justdata/acoustic/resampling.py +218 -0
- justdata/acoustic/schema.py +51 -0
- justdata/acoustic/segment.py +369 -0
- justdata/acoustic/sources.py +503 -0
- justdata/acoustic/stages.py +59 -0
- justdata/acoustic/stats.py +50 -0
- justdata/acoustic/tasks.py +375 -0
- justdata/audio/__init__.py +20 -0
- justdata/core/__init__.py +74 -0
- justdata/core/adapters.py +46 -0
- justdata/core/augmentations.py +34 -0
- justdata/core/filters.py +94 -0
- justdata/core/finalization.py +173 -0
- justdata/core/label_mixing.py +129 -0
- justdata/core/loader.py +384 -0
- justdata/core/metadata.py +250 -0
- justdata/core/presets.py +155 -0
- justdata/core/registry.py +229 -0
- justdata/core/sources.py +87 -0
- justdata/core/stats.py +227 -0
- justdata/corruptions.py +52 -0
- justdata/py.typed +0 -0
- justdata/vision/__init__.py +17 -0
- justdata/vision/augmentations/__init__.py +31 -0
- justdata/vision/augmentations/auto.py +480 -0
- justdata/vision/augmentations/color.py +163 -0
- justdata/vision/augmentations/composed.py +156 -0
- justdata/vision/augmentations/geometric.py +302 -0
- justdata/vision/augmentations/mixing.py +232 -0
- justdata/vision/augmentations/registry.py +142 -0
- justdata/vision/corruptions/__init__.py +16 -0
- justdata/vision/corruptions/blur.py +65 -0
- justdata/vision/corruptions/digital.py +26 -0
- justdata/vision/corruptions/noise.py +20 -0
- justdata/vision/corruptions/registry.py +113 -0
- justdata/vision/corruptions/weather.py +100 -0
- justdata/vision/datasets.py +34 -0
- justdata/vision/encodings/__init__.py +1 -0
- justdata/vision/encodings/heatmaps.py +196 -0
- justdata/vision/minic.py +69 -0
- justdata/vision/pipelines.py +62 -0
- justdata/vision/presets.py +509 -0
- justdata/vision/sources.py +1080 -0
- justdata/vision/stages.py +271 -0
- justdata/vision/tasks/__init__.py +1 -0
- justdata/vision/tasks/classification.py +321 -0
- justdata/vision/tasks/depth.py +25 -0
- justdata/vision/tasks/segmentation.py +194 -0
- justdata/vision/transforms.py +113 -0
- justdata/vision/utils.py +741 -0
- justdata-1.0.0.dist-info/METADATA +860 -0
- justdata-1.0.0.dist-info/RECORD +106 -0
- justdata-1.0.0.dist-info/WHEEL +4 -0
- justdata-1.0.0.dist-info/licenses/LICENSE +23 -0
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
import json
|
|
5
|
+
import threading
|
|
6
|
+
from collections.abc import Mapping
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any, Literal
|
|
9
|
+
|
|
10
|
+
import tensorflow as tf
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def python_value(value: Any) -> Any:
|
|
14
|
+
if hasattr(value, "numpy"):
|
|
15
|
+
value = value.numpy()
|
|
16
|
+
if isinstance(value, bytes):
|
|
17
|
+
return value.decode("utf-8")
|
|
18
|
+
if hasattr(value, "tolist"):
|
|
19
|
+
return python_value(value.tolist())
|
|
20
|
+
if isinstance(value, (list, tuple)):
|
|
21
|
+
return [python_value(child) for child in value]
|
|
22
|
+
if hasattr(value, "item"):
|
|
23
|
+
return value.item()
|
|
24
|
+
return value
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def get_metadata_value(sample: dict, key: str) -> Any | None:
|
|
28
|
+
metadata = sample.get("metadata")
|
|
29
|
+
if isinstance(metadata, dict) and key in metadata:
|
|
30
|
+
return metadata[key]
|
|
31
|
+
return sample.get(key)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _is_numeric_metadata_value(value: Any) -> bool:
|
|
35
|
+
return hasattr(value, "dtype") and value.dtype != tf.string
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def numeric_metadata(metadata: dict) -> dict:
|
|
39
|
+
numeric = {}
|
|
40
|
+
for key, value in metadata.items():
|
|
41
|
+
if isinstance(value, dict):
|
|
42
|
+
child = numeric_metadata(value)
|
|
43
|
+
if child:
|
|
44
|
+
numeric[key] = child
|
|
45
|
+
elif _is_numeric_metadata_value(value):
|
|
46
|
+
numeric[key] = value
|
|
47
|
+
return numeric
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def apply_metadata_mode(
|
|
51
|
+
sample: dict,
|
|
52
|
+
metadata_mode: Literal["full", "numeric_only", "none"],
|
|
53
|
+
) -> dict:
|
|
54
|
+
if metadata_mode == "full" or "metadata" not in sample:
|
|
55
|
+
return sample
|
|
56
|
+
|
|
57
|
+
result = dict(sample)
|
|
58
|
+
if metadata_mode == "none":
|
|
59
|
+
result.pop("metadata", None)
|
|
60
|
+
return result
|
|
61
|
+
|
|
62
|
+
metadata = result.get("metadata")
|
|
63
|
+
if isinstance(metadata, dict):
|
|
64
|
+
metadata = numeric_metadata(metadata)
|
|
65
|
+
if metadata:
|
|
66
|
+
result["metadata"] = metadata
|
|
67
|
+
else:
|
|
68
|
+
result.pop("metadata", None)
|
|
69
|
+
return result
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _flatten_metadata(metadata: dict, prefix: tuple[str, ...] = ()):
|
|
73
|
+
leaves = []
|
|
74
|
+
for key, value in metadata.items():
|
|
75
|
+
path = prefix + (key,)
|
|
76
|
+
if isinstance(value, dict):
|
|
77
|
+
leaves.extend(_flatten_metadata(value, path))
|
|
78
|
+
else:
|
|
79
|
+
leaves.append((path, value))
|
|
80
|
+
return leaves
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _identity_structure(value):
|
|
84
|
+
if isinstance(value, dict):
|
|
85
|
+
return {key: _identity_structure(child) for key, child in value.items()}
|
|
86
|
+
return tf.identity(value)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def json_value(value: Any) -> Any:
|
|
90
|
+
value = python_value(value)
|
|
91
|
+
if isinstance(value, Mapping):
|
|
92
|
+
return {str(key): json_value(child) for key, child in value.items()}
|
|
93
|
+
if isinstance(value, (list, tuple)):
|
|
94
|
+
return [json_value(child) for child in value]
|
|
95
|
+
return value
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def stable_int64_hash(value: Any) -> int:
|
|
99
|
+
"""Return a process-stable non-negative int64 hash for metadata ids."""
|
|
100
|
+
value = python_value(value)
|
|
101
|
+
if value is None:
|
|
102
|
+
value = ""
|
|
103
|
+
digest = hashlib.sha256(str(value).encode("utf-8")).digest()
|
|
104
|
+
return int.from_bytes(digest[:8], "big", signed=False) & ((1 << 63) - 1)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _assign_nested(target: dict, path: tuple[str, ...], value: Any) -> None:
|
|
108
|
+
current = target
|
|
109
|
+
for key in path[:-1]:
|
|
110
|
+
current = current.setdefault(key, {})
|
|
111
|
+
current[path[-1]] = value
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _sidecar_example_id(values_by_path: dict[tuple[str, ...], Any]) -> int:
|
|
115
|
+
by_name = {path[-1]: value for path, value in values_by_path.items()}
|
|
116
|
+
composite_values = [
|
|
117
|
+
python_value(by_name.get(key)) for key in ("dataset", "split", "clip_id")
|
|
118
|
+
]
|
|
119
|
+
if all(value is not None for value in composite_values):
|
|
120
|
+
return stable_int64_hash(
|
|
121
|
+
f"{composite_values[0]}::{composite_values[1]}::{composite_values[2]}"
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
value = python_value(by_name.get("example_id"))
|
|
125
|
+
if isinstance(value, int) and not isinstance(value, bool):
|
|
126
|
+
return value
|
|
127
|
+
if isinstance(value, str):
|
|
128
|
+
return stable_int64_hash(value)
|
|
129
|
+
raise ValueError(
|
|
130
|
+
"Sidecar metadata requires either non-null 'dataset', 'split', and "
|
|
131
|
+
"'clip_id' fields or an integer/string 'example_id'."
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
class MetadataSidecar:
|
|
136
|
+
def __init__(self) -> None:
|
|
137
|
+
self.records: dict[int, dict] = {}
|
|
138
|
+
|
|
139
|
+
def add(self, example_id: int, metadata: dict) -> None:
|
|
140
|
+
example_id = int(example_id)
|
|
141
|
+
metadata = json_value(metadata)
|
|
142
|
+
existing = self.records.get(example_id)
|
|
143
|
+
if existing is not None and existing != metadata:
|
|
144
|
+
raise ValueError(f"Conflicting metadata for sidecar ID {example_id}.")
|
|
145
|
+
self.records[example_id] = metadata
|
|
146
|
+
|
|
147
|
+
def write_jsonl(self, path: str) -> None:
|
|
148
|
+
output = Path(path)
|
|
149
|
+
if output.parent != Path("."):
|
|
150
|
+
output.parent.mkdir(parents=True, exist_ok=True)
|
|
151
|
+
with output.open("w", encoding="utf-8") as handle:
|
|
152
|
+
for example_id in sorted(self.records):
|
|
153
|
+
handle.write(
|
|
154
|
+
json.dumps(
|
|
155
|
+
{
|
|
156
|
+
"example_id": example_id,
|
|
157
|
+
"metadata": self.records[example_id],
|
|
158
|
+
},
|
|
159
|
+
sort_keys=True,
|
|
160
|
+
)
|
|
161
|
+
+ "\n"
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
@classmethod
|
|
165
|
+
def read_jsonl(cls, path: str) -> "MetadataSidecar":
|
|
166
|
+
sidecar = cls()
|
|
167
|
+
with Path(path).open("r", encoding="utf-8") as handle:
|
|
168
|
+
for line in handle:
|
|
169
|
+
if not line.strip():
|
|
170
|
+
continue
|
|
171
|
+
record = json.loads(line)
|
|
172
|
+
sidecar.add(record["example_id"], record.get("metadata", {}))
|
|
173
|
+
return sidecar
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def attach_sidecar_writer(ds: tf.data.Dataset, path: str) -> tf.data.Dataset:
|
|
177
|
+
output = Path(path)
|
|
178
|
+
if output.parent != Path("."):
|
|
179
|
+
output.parent.mkdir(parents=True, exist_ok=True)
|
|
180
|
+
output.write_text("", encoding="utf-8")
|
|
181
|
+
lock = threading.Lock()
|
|
182
|
+
serialized_by_id: dict[int, str] = {}
|
|
183
|
+
|
|
184
|
+
def add_writer(sample):
|
|
185
|
+
metadata = sample.get("metadata")
|
|
186
|
+
if not isinstance(metadata, dict):
|
|
187
|
+
return sample
|
|
188
|
+
|
|
189
|
+
leaves = _flatten_metadata(metadata)
|
|
190
|
+
string_leaves = [
|
|
191
|
+
(leaf_path, value)
|
|
192
|
+
for leaf_path, value in leaves
|
|
193
|
+
if value.dtype == tf.string
|
|
194
|
+
]
|
|
195
|
+
if not string_leaves:
|
|
196
|
+
return sample
|
|
197
|
+
|
|
198
|
+
all_paths = [leaf_path for leaf_path, _ in leaves]
|
|
199
|
+
all_values = [value for _, value in leaves]
|
|
200
|
+
string_paths = {leaf_path for leaf_path, _ in string_leaves}
|
|
201
|
+
|
|
202
|
+
def write_record(*values):
|
|
203
|
+
values_by_path = {
|
|
204
|
+
leaf_path: python_value(value)
|
|
205
|
+
for leaf_path, value in zip(all_paths, values)
|
|
206
|
+
}
|
|
207
|
+
string_metadata = {}
|
|
208
|
+
for leaf_path in string_paths:
|
|
209
|
+
_assign_nested(
|
|
210
|
+
string_metadata,
|
|
211
|
+
leaf_path,
|
|
212
|
+
values_by_path[leaf_path],
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
example_id = _sidecar_example_id(values_by_path)
|
|
216
|
+
record = {
|
|
217
|
+
"example_id": example_id,
|
|
218
|
+
"metadata": string_metadata,
|
|
219
|
+
}
|
|
220
|
+
serialized = json.dumps(record, sort_keys=True)
|
|
221
|
+
with lock:
|
|
222
|
+
existing = serialized_by_id.get(example_id)
|
|
223
|
+
if existing is not None:
|
|
224
|
+
if existing == serialized:
|
|
225
|
+
return 0
|
|
226
|
+
raise ValueError(
|
|
227
|
+
f"Conflicting metadata for sidecar ID {example_id}."
|
|
228
|
+
)
|
|
229
|
+
with output.open("a", encoding="utf-8") as handle:
|
|
230
|
+
handle.write(serialized + "\n")
|
|
231
|
+
serialized_by_id[example_id] = serialized
|
|
232
|
+
return 0
|
|
233
|
+
|
|
234
|
+
marker = tf.py_function(write_record, all_values, Tout=tf.int64)
|
|
235
|
+
with tf.control_dependencies([marker]):
|
|
236
|
+
return _identity_structure(sample)
|
|
237
|
+
|
|
238
|
+
return ds.map(add_writer, num_parallel_calls=tf.data.AUTOTUNE)
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
__all__ = [
|
|
242
|
+
"apply_metadata_mode",
|
|
243
|
+
"attach_sidecar_writer",
|
|
244
|
+
"get_metadata_value",
|
|
245
|
+
"json_value",
|
|
246
|
+
"MetadataSidecar",
|
|
247
|
+
"numeric_metadata",
|
|
248
|
+
"python_value",
|
|
249
|
+
"stable_int64_hash",
|
|
250
|
+
]
|
justdata/core/presets.py
ADDED
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
import copy
|
|
2
|
+
import dataclasses
|
|
3
|
+
import hashlib
|
|
4
|
+
import json
|
|
5
|
+
import threading
|
|
6
|
+
from collections.abc import Mapping
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from typing import Any, Dict
|
|
9
|
+
|
|
10
|
+
_PRESETS: Dict[str, Dict[str, Any]] = {}
|
|
11
|
+
_PRESET_LOCK = threading.Lock()
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _plain_value(value: Any) -> Any:
|
|
15
|
+
if dataclasses.is_dataclass(value):
|
|
16
|
+
value = dataclasses.asdict(value)
|
|
17
|
+
|
|
18
|
+
if isinstance(value, Mapping):
|
|
19
|
+
return {str(k): _plain_value(v) for k, v in value.items()}
|
|
20
|
+
|
|
21
|
+
if isinstance(value, (tuple, list)):
|
|
22
|
+
return [_plain_value(v) for v in value]
|
|
23
|
+
|
|
24
|
+
return value
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def canonical_preset_json(config: Any) -> str:
|
|
28
|
+
return json.dumps(_plain_value(config), sort_keys=True, separators=(",", ":"))
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def preset_hash(config: Any) -> str:
|
|
32
|
+
return hashlib.sha256(canonical_preset_json(config).encode("utf-8")).hexdigest()[
|
|
33
|
+
:16
|
|
34
|
+
]
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass(frozen=True)
|
|
38
|
+
class ResolvedPreset(Mapping[str, Any]):
|
|
39
|
+
name: str
|
|
40
|
+
modality: str
|
|
41
|
+
config: Any
|
|
42
|
+
|
|
43
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
44
|
+
plain = _plain_value(self.config)
|
|
45
|
+
return copy.deepcopy(plain)
|
|
46
|
+
|
|
47
|
+
def to_json(self) -> str:
|
|
48
|
+
return canonical_preset_json(self.config)
|
|
49
|
+
|
|
50
|
+
def hash(self) -> str:
|
|
51
|
+
return preset_hash(self.config)
|
|
52
|
+
|
|
53
|
+
def __getitem__(self, key: str) -> Any:
|
|
54
|
+
return self.to_dict()[key]
|
|
55
|
+
|
|
56
|
+
def __iter__(self):
|
|
57
|
+
return iter(self.to_dict())
|
|
58
|
+
|
|
59
|
+
def __len__(self) -> int:
|
|
60
|
+
return len(self.to_dict())
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def register_preset(dataset: str, config: Any, *, modality: str = "core"):
|
|
64
|
+
key = dataset.lower()
|
|
65
|
+
with _PRESET_LOCK:
|
|
66
|
+
presets = _PRESETS.setdefault(modality, {})
|
|
67
|
+
if key in presets:
|
|
68
|
+
raise ValueError(
|
|
69
|
+
f"Preset '{key}' already registered for modality '{modality}'"
|
|
70
|
+
)
|
|
71
|
+
presets[key] = copy.deepcopy(config)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _find_preset(dataset: str, *, modality: str = "core") -> tuple[str | None, Any]:
|
|
75
|
+
presets = _PRESETS.get(modality, {})
|
|
76
|
+
dataset = dataset.lower()
|
|
77
|
+
|
|
78
|
+
if dataset in presets:
|
|
79
|
+
return dataset, presets[dataset]
|
|
80
|
+
|
|
81
|
+
for key, val in sorted(presets.items(), key=lambda kv: len(kv[0]), reverse=True):
|
|
82
|
+
if key != "_default" and dataset.startswith(key):
|
|
83
|
+
return key, val
|
|
84
|
+
|
|
85
|
+
if "_default" in presets:
|
|
86
|
+
return "_default", presets["_default"]
|
|
87
|
+
|
|
88
|
+
return None, {}
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def get_dataset_presets(dataset: str, *, modality: str = "core") -> Dict[str, Any]:
|
|
92
|
+
_name, config = _find_preset(dataset, modality=modality)
|
|
93
|
+
return copy.deepcopy(config)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def get_resolved_preset(dataset: str, *, modality: str = "core") -> ResolvedPreset:
|
|
97
|
+
name, config = _find_preset(dataset, modality=modality)
|
|
98
|
+
return ResolvedPreset(
|
|
99
|
+
name=name or dataset.lower(), modality=modality, config=copy.deepcopy(config)
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def merge_with_presets(
|
|
104
|
+
dataset: str, user_kwargs: Dict[str, Any], *, modality: str = "core"
|
|
105
|
+
) -> Dict[str, Any]:
|
|
106
|
+
"""
|
|
107
|
+
Smartly merge user-provided kwargs with modality-specific presets.
|
|
108
|
+
|
|
109
|
+
If a user kwarg matches the modality default, it is treated as an
|
|
110
|
+
unmodified default and the dataset-specific preset takes precedence.
|
|
111
|
+
"""
|
|
112
|
+
|
|
113
|
+
dataset_presets = copy.deepcopy(get_dataset_presets(dataset, modality=modality))
|
|
114
|
+
default_presets = get_dataset_presets("_default", modality=modality)
|
|
115
|
+
|
|
116
|
+
def smart_merge(base, user, default):
|
|
117
|
+
result = base.copy()
|
|
118
|
+
for k, v in user.items():
|
|
119
|
+
if isinstance(v, dict) and k in base and isinstance(base[k], dict):
|
|
120
|
+
result[k] = smart_merge(
|
|
121
|
+
base[k], v, default.get(k, {}) if isinstance(default, dict) else {}
|
|
122
|
+
)
|
|
123
|
+
else:
|
|
124
|
+
default_val = default.get(k) if isinstance(default, dict) else None
|
|
125
|
+
val_to_compare = tuple(v) if isinstance(v, list) else v
|
|
126
|
+
def_to_compare = (
|
|
127
|
+
tuple(default_val) if isinstance(default_val, list) else default_val
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
if (
|
|
131
|
+
isinstance(val_to_compare, tuple)
|
|
132
|
+
and len(val_to_compare) > 0
|
|
133
|
+
and isinstance(val_to_compare[0], list)
|
|
134
|
+
):
|
|
135
|
+
val_to_compare = tuple(
|
|
136
|
+
tuple(x) if isinstance(x, list) else x for x in val_to_compare
|
|
137
|
+
)
|
|
138
|
+
if (
|
|
139
|
+
isinstance(def_to_compare, tuple)
|
|
140
|
+
and len(def_to_compare) > 0
|
|
141
|
+
and isinstance(def_to_compare[0], list)
|
|
142
|
+
):
|
|
143
|
+
def_to_compare = tuple(
|
|
144
|
+
tuple(x) if isinstance(x, list) else x for x in def_to_compare
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
if (
|
|
148
|
+
k not in default
|
|
149
|
+
or val_to_compare != def_to_compare
|
|
150
|
+
or k not in result
|
|
151
|
+
):
|
|
152
|
+
result[k] = v
|
|
153
|
+
return result
|
|
154
|
+
|
|
155
|
+
return smart_merge(dataset_presets, user_kwargs, default_presets)
|
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
import threading
|
|
3
|
+
from typing import Callable, Dict, Tuple
|
|
4
|
+
|
|
5
|
+
PipelineFuncs = Tuple[Callable, Callable, Callable, Callable]
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@dataclass(frozen=True)
|
|
9
|
+
class DatasetInfo:
|
|
10
|
+
name: str
|
|
11
|
+
modality: str
|
|
12
|
+
task: str | None = None
|
|
13
|
+
pipeline_name: str | None = None
|
|
14
|
+
preset: str | None = None
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
_REGISTRY_LOCK = threading.Lock()
|
|
18
|
+
_DATASETS: Dict[str, DatasetInfo] = {}
|
|
19
|
+
_PIPELINES: Dict[str, Callable[..., PipelineFuncs]] = {}
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def register_dataset(
|
|
23
|
+
name: str,
|
|
24
|
+
task_type: str | None = None,
|
|
25
|
+
*,
|
|
26
|
+
modality: str = "core",
|
|
27
|
+
pipeline_name: str | None = None,
|
|
28
|
+
preset: str | None = None,
|
|
29
|
+
):
|
|
30
|
+
"""Register dataset metadata at runtime."""
|
|
31
|
+
with _REGISTRY_LOCK:
|
|
32
|
+
if name in _DATASETS:
|
|
33
|
+
existing = _DATASETS[name]
|
|
34
|
+
raise ValueError(
|
|
35
|
+
f"Dataset '{name}' is already registered with modality "
|
|
36
|
+
f"'{existing.modality}' and task '{existing.task}'."
|
|
37
|
+
)
|
|
38
|
+
_DATASETS[name] = DatasetInfo(
|
|
39
|
+
name=name,
|
|
40
|
+
modality=modality,
|
|
41
|
+
task=task_type,
|
|
42
|
+
pipeline_name=pipeline_name,
|
|
43
|
+
preset=preset,
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def get_dataset_info(name: str) -> DatasetInfo | None:
|
|
48
|
+
with _REGISTRY_LOCK:
|
|
49
|
+
exact = _DATASETS.get(name)
|
|
50
|
+
entries = tuple(_DATASETS.items())
|
|
51
|
+
|
|
52
|
+
if exact is not None:
|
|
53
|
+
return exact
|
|
54
|
+
|
|
55
|
+
for key, val in sorted(entries, key=lambda kv: len(kv[0]), reverse=True):
|
|
56
|
+
if name.startswith(key):
|
|
57
|
+
return val
|
|
58
|
+
|
|
59
|
+
return None
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def get_task_for_dataset(name: str) -> str | None:
|
|
63
|
+
info = get_dataset_info(name)
|
|
64
|
+
return info.task if info else None
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _pipeline_modality(pipeline_name: str | None) -> str | None:
|
|
68
|
+
if pipeline_name and "/" in pipeline_name:
|
|
69
|
+
return pipeline_name.split("/", 1)[0]
|
|
70
|
+
return None
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class DataPipeline:
|
|
74
|
+
"""Encapsulates modality-specific pipeline logic and configuration."""
|
|
75
|
+
|
|
76
|
+
def __init__(
|
|
77
|
+
self,
|
|
78
|
+
pipeline_name: str,
|
|
79
|
+
*,
|
|
80
|
+
modality: str | None = None,
|
|
81
|
+
_is_training_legacy: bool = False,
|
|
82
|
+
**kwargs,
|
|
83
|
+
):
|
|
84
|
+
self.pipeline_name = pipeline_name
|
|
85
|
+
self.modality = modality or _pipeline_modality(pipeline_name)
|
|
86
|
+
self.kwargs = kwargs
|
|
87
|
+
self._is_training_legacy = _is_training_legacy
|
|
88
|
+
|
|
89
|
+
def build(self, is_training: bool) -> PipelineFuncs:
|
|
90
|
+
"""Return the 4 callables: preprocess, augment, batch augment, postprocess."""
|
|
91
|
+
import copy
|
|
92
|
+
|
|
93
|
+
config = copy.deepcopy(self.kwargs)
|
|
94
|
+
postproc = config.setdefault("postproc_kwargs", {})
|
|
95
|
+
postproc["is_training"] = is_training
|
|
96
|
+
|
|
97
|
+
with _REGISTRY_LOCK:
|
|
98
|
+
factory = _PIPELINES.get(self.pipeline_name)
|
|
99
|
+
if factory is None:
|
|
100
|
+
raise ValueError(f"Pipeline '{self.pipeline_name}' not found.")
|
|
101
|
+
|
|
102
|
+
return factory(**config)
|
|
103
|
+
|
|
104
|
+
def __iter__(self):
|
|
105
|
+
return iter(self.build(is_training=self._is_training_legacy))
|
|
106
|
+
|
|
107
|
+
def __repr__(self):
|
|
108
|
+
return (
|
|
109
|
+
f"DataPipeline(pipeline_name={self.pipeline_name!r}, "
|
|
110
|
+
f"modality={self.modality!r}, config={self.kwargs})"
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def register_pipeline(name: str):
|
|
115
|
+
"""
|
|
116
|
+
Decorator to register a pipeline.
|
|
117
|
+
|
|
118
|
+
A pipeline returns four callables:
|
|
119
|
+
(preprocess, augment, batch_augment, postprocess).
|
|
120
|
+
"""
|
|
121
|
+
|
|
122
|
+
def decorator(fn: Callable[..., PipelineFuncs]):
|
|
123
|
+
with _REGISTRY_LOCK:
|
|
124
|
+
if name in _PIPELINES:
|
|
125
|
+
raise ValueError(
|
|
126
|
+
f"Pipeline '{name}' already registered by "
|
|
127
|
+
f"{_PIPELINES[name].__module__}.{_PIPELINES[name].__qualname__}"
|
|
128
|
+
)
|
|
129
|
+
_PIPELINES[name] = fn
|
|
130
|
+
return fn
|
|
131
|
+
|
|
132
|
+
return decorator
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def list_pipelines() -> tuple[str, ...]:
|
|
136
|
+
with _REGISTRY_LOCK:
|
|
137
|
+
return tuple(sorted(_PIPELINES))
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def has_pipeline(name: str) -> bool:
|
|
141
|
+
with _REGISTRY_LOCK:
|
|
142
|
+
return name in _PIPELINES
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def get_pipeline(
|
|
146
|
+
dataset: str | None = None,
|
|
147
|
+
preset: str | None = None,
|
|
148
|
+
task: str | None = None,
|
|
149
|
+
pipeline_name: str | None = None,
|
|
150
|
+
modality: str | None = None,
|
|
151
|
+
apply_presets: bool = True,
|
|
152
|
+
**kwargs,
|
|
153
|
+
) -> DataPipeline:
|
|
154
|
+
"""
|
|
155
|
+
Resolve and configure a registered pipeline.
|
|
156
|
+
|
|
157
|
+
Dataset registration supplies the default modality, task, pipeline, and
|
|
158
|
+
preset. Explicit ``pipeline_name`` wins over explicit ``task``; both win
|
|
159
|
+
over dataset-driven resolution.
|
|
160
|
+
"""
|
|
161
|
+
from justdata.core.presets import merge_with_presets
|
|
162
|
+
|
|
163
|
+
dataset_info = get_dataset_info(dataset) if dataset else None
|
|
164
|
+
effective_modality = modality or (dataset_info.modality if dataset_info else None)
|
|
165
|
+
effective_pipeline = pipeline_name
|
|
166
|
+
|
|
167
|
+
if effective_pipeline is None and task is not None:
|
|
168
|
+
effective_pipeline = (
|
|
169
|
+
task
|
|
170
|
+
if "/" in task or effective_modality is None
|
|
171
|
+
else f"{effective_modality}/{task}"
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
dataset_is_pipeline_name = False
|
|
175
|
+
if effective_pipeline is None and dataset:
|
|
176
|
+
if dataset_info is not None:
|
|
177
|
+
effective_pipeline = dataset_info.pipeline_name
|
|
178
|
+
if effective_pipeline is None and dataset_info.task is not None:
|
|
179
|
+
effective_pipeline = f"{dataset_info.modality}/{dataset_info.task}"
|
|
180
|
+
elif has_pipeline(dataset):
|
|
181
|
+
effective_pipeline = dataset
|
|
182
|
+
effective_modality = effective_modality or _pipeline_modality(dataset)
|
|
183
|
+
dataset_is_pipeline_name = True
|
|
184
|
+
|
|
185
|
+
if effective_pipeline is None:
|
|
186
|
+
raise ValueError(
|
|
187
|
+
f"Could not resolve pipeline for dataset='{dataset}' and preset='{preset}'. "
|
|
188
|
+
"Please specify `task` or `pipeline_name`."
|
|
189
|
+
)
|
|
190
|
+
|
|
191
|
+
effective_modality = effective_modality or _pipeline_modality(effective_pipeline)
|
|
192
|
+
|
|
193
|
+
preset_name = preset
|
|
194
|
+
if preset_name is None and dataset_info is not None:
|
|
195
|
+
preset_name = dataset_info.preset or dataset
|
|
196
|
+
elif preset_name is None and dataset and not dataset_is_pipeline_name:
|
|
197
|
+
preset_name = dataset
|
|
198
|
+
|
|
199
|
+
if apply_presets and preset_name and effective_modality:
|
|
200
|
+
kwargs = merge_with_presets(
|
|
201
|
+
preset_name,
|
|
202
|
+
kwargs,
|
|
203
|
+
modality=effective_modality,
|
|
204
|
+
)
|
|
205
|
+
|
|
206
|
+
return DataPipeline(
|
|
207
|
+
effective_pipeline,
|
|
208
|
+
modality=effective_modality,
|
|
209
|
+
**kwargs,
|
|
210
|
+
)
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def get_pipeline_for_dataset(
|
|
214
|
+
dataset: str,
|
|
215
|
+
task_type: str | None = None,
|
|
216
|
+
pipeline_name: str | None = None,
|
|
217
|
+
apply_presets: bool = True,
|
|
218
|
+
is_training: bool = False,
|
|
219
|
+
**kwargs,
|
|
220
|
+
) -> DataPipeline:
|
|
221
|
+
"""Compatibility alias for call sites that still resolve by dataset."""
|
|
222
|
+
return get_pipeline(
|
|
223
|
+
dataset=dataset,
|
|
224
|
+
task=task_type,
|
|
225
|
+
pipeline_name=pipeline_name,
|
|
226
|
+
apply_presets=apply_presets,
|
|
227
|
+
_is_training_legacy=is_training,
|
|
228
|
+
**kwargs,
|
|
229
|
+
)
|