justdata 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. justdata/__init__.py +3 -0
  2. justdata/acoustic/__init__.py +381 -0
  3. justdata/acoustic/adapters.py +329 -0
  4. justdata/acoustic/augment/__init__.py +119 -0
  5. justdata/acoustic/augment/batch.py +813 -0
  6. justdata/acoustic/augment/configs.py +204 -0
  7. justdata/acoustic/augment/patchout.py +331 -0
  8. justdata/acoustic/augment/spectrogram.py +806 -0
  9. justdata/acoustic/augment/waveform.py +879 -0
  10. justdata/acoustic/batching.py +27 -0
  11. justdata/acoustic/channel.py +53 -0
  12. justdata/acoustic/compat/__init__.py +95 -0
  13. justdata/acoustic/compat/ast.py +626 -0
  14. justdata/acoustic/compat/ced.py +152 -0
  15. justdata/acoustic/compat/efficientat.py +243 -0
  16. justdata/acoustic/compat/panns.py +115 -0
  17. justdata/acoustic/compat/passt.py +191 -0
  18. justdata/acoustic/configs.py +519 -0
  19. justdata/acoustic/corruptions/__init__.py +33 -0
  20. justdata/acoustic/corruptions/codec.py +92 -0
  21. justdata/acoustic/corruptions/datasets.py +155 -0
  22. justdata/acoustic/corruptions/device.py +39 -0
  23. justdata/acoustic/corruptions/dynamics.py +55 -0
  24. justdata/acoustic/corruptions/filters.py +105 -0
  25. justdata/acoustic/corruptions/noise.py +173 -0
  26. justdata/acoustic/corruptions/registry.py +230 -0
  27. justdata/acoustic/corruptions/reverb.py +49 -0
  28. justdata/acoustic/corruptions/timefreq.py +127 -0
  29. justdata/acoustic/datasets.py +52 -0
  30. justdata/acoustic/dcase2025.py +814 -0
  31. justdata/acoustic/decoding.py +73 -0
  32. justdata/acoustic/eval_views.py +54 -0
  33. justdata/acoustic/evals.py +71 -0
  34. justdata/acoustic/filters.py +18 -0
  35. justdata/acoustic/frontends/__init__.py +37 -0
  36. justdata/acoustic/frontends/ast.py +116 -0
  37. justdata/acoustic/frontends/kaldi.py +34 -0
  38. justdata/acoustic/frontends/log.py +94 -0
  39. justdata/acoustic/frontends/mel.py +173 -0
  40. justdata/acoustic/frontends/mfcc.py +26 -0
  41. justdata/acoustic/frontends/pcen.py +26 -0
  42. justdata/acoustic/frontends/stft.py +174 -0
  43. justdata/acoustic/jax.py +23 -0
  44. justdata/acoustic/labels.py +169 -0
  45. justdata/acoustic/layouts.py +86 -0
  46. justdata/acoustic/metadata.py +152 -0
  47. justdata/acoustic/normalization.py +196 -0
  48. justdata/acoustic/pipelines.py +112 -0
  49. justdata/acoustic/postprocessing.py +328 -0
  50. justdata/acoustic/preprocessing.py +175 -0
  51. justdata/acoustic/presets.py +257 -0
  52. justdata/acoustic/registry.py +381 -0
  53. justdata/acoustic/resampling.py +218 -0
  54. justdata/acoustic/schema.py +51 -0
  55. justdata/acoustic/segment.py +369 -0
  56. justdata/acoustic/sources.py +503 -0
  57. justdata/acoustic/stages.py +59 -0
  58. justdata/acoustic/stats.py +50 -0
  59. justdata/acoustic/tasks.py +375 -0
  60. justdata/audio/__init__.py +20 -0
  61. justdata/core/__init__.py +74 -0
  62. justdata/core/adapters.py +46 -0
  63. justdata/core/augmentations.py +34 -0
  64. justdata/core/filters.py +94 -0
  65. justdata/core/finalization.py +173 -0
  66. justdata/core/label_mixing.py +129 -0
  67. justdata/core/loader.py +384 -0
  68. justdata/core/metadata.py +250 -0
  69. justdata/core/presets.py +155 -0
  70. justdata/core/registry.py +229 -0
  71. justdata/core/sources.py +87 -0
  72. justdata/core/stats.py +227 -0
  73. justdata/corruptions.py +52 -0
  74. justdata/py.typed +0 -0
  75. justdata/vision/__init__.py +17 -0
  76. justdata/vision/augmentations/__init__.py +31 -0
  77. justdata/vision/augmentations/auto.py +480 -0
  78. justdata/vision/augmentations/color.py +163 -0
  79. justdata/vision/augmentations/composed.py +156 -0
  80. justdata/vision/augmentations/geometric.py +302 -0
  81. justdata/vision/augmentations/mixing.py +232 -0
  82. justdata/vision/augmentations/registry.py +142 -0
  83. justdata/vision/corruptions/__init__.py +16 -0
  84. justdata/vision/corruptions/blur.py +65 -0
  85. justdata/vision/corruptions/digital.py +26 -0
  86. justdata/vision/corruptions/noise.py +20 -0
  87. justdata/vision/corruptions/registry.py +113 -0
  88. justdata/vision/corruptions/weather.py +100 -0
  89. justdata/vision/datasets.py +34 -0
  90. justdata/vision/encodings/__init__.py +1 -0
  91. justdata/vision/encodings/heatmaps.py +196 -0
  92. justdata/vision/minic.py +69 -0
  93. justdata/vision/pipelines.py +62 -0
  94. justdata/vision/presets.py +509 -0
  95. justdata/vision/sources.py +1080 -0
  96. justdata/vision/stages.py +271 -0
  97. justdata/vision/tasks/__init__.py +1 -0
  98. justdata/vision/tasks/classification.py +321 -0
  99. justdata/vision/tasks/depth.py +25 -0
  100. justdata/vision/tasks/segmentation.py +194 -0
  101. justdata/vision/transforms.py +113 -0
  102. justdata/vision/utils.py +741 -0
  103. justdata-1.0.0.dist-info/METADATA +860 -0
  104. justdata-1.0.0.dist-info/RECORD +106 -0
  105. justdata-1.0.0.dist-info/WHEEL +4 -0
  106. justdata-1.0.0.dist-info/licenses/LICENSE +23 -0
@@ -0,0 +1,87 @@
1
+ import os
2
+ import threading
3
+ from pathlib import Path
4
+ from typing import Optional, Protocol, Union
5
+
6
+ import tensorflow as tf
7
+ import tensorflow_datasets as tfds
8
+
9
+
10
+ class SourceLoader(Protocol):
11
+ def __call__(
12
+ self,
13
+ dataset_name: str,
14
+ splits: list[str],
15
+ data_dir: Union[None, str, os.PathLike] = None,
16
+ ) -> list[tf.data.Dataset]: ...
17
+
18
+
19
+ _SOURCE_LOCK = threading.Lock()
20
+ _PREFIX_LOADERS: dict[str, SourceLoader] = {}
21
+ _DEFAULT_LOADER: Optional[SourceLoader] = None
22
+
23
+
24
+ def source_cache_dir(
25
+ data_dir: Union[None, str, os.PathLike],
26
+ *parts: str,
27
+ ) -> Path:
28
+ root = (
29
+ Path(data_dir).expanduser()
30
+ if data_dir is not None
31
+ else Path.home() / ".cache" / "justdata"
32
+ )
33
+ return root.joinpath(*parts)
34
+
35
+
36
+ def register_source_loader(prefix: str):
37
+ def decorator(fn: SourceLoader) -> SourceLoader:
38
+ with _SOURCE_LOCK:
39
+ if prefix in _PREFIX_LOADERS:
40
+ raise ValueError(f"Source loader for prefix '{prefix}' already exists.")
41
+ _PREFIX_LOADERS[prefix] = fn
42
+ return fn
43
+
44
+ return decorator
45
+
46
+
47
+ def register_default_source_loader(fn: SourceLoader) -> SourceLoader:
48
+ global _DEFAULT_LOADER
49
+ with _SOURCE_LOCK:
50
+ if _DEFAULT_LOADER is not None:
51
+ raise ValueError("Default source loader already exists.")
52
+ _DEFAULT_LOADER = fn
53
+ return fn
54
+
55
+
56
+ def get_source_loader(dataset_name: str) -> SourceLoader:
57
+ with _SOURCE_LOCK:
58
+ prefix_loaders = tuple(_PREFIX_LOADERS.items())
59
+ default_loader = _DEFAULT_LOADER
60
+
61
+ for prefix, loader in sorted(
62
+ prefix_loaders, key=lambda kv: len(kv[0]), reverse=True
63
+ ):
64
+ if dataset_name.startswith(prefix):
65
+ return loader
66
+
67
+ if default_loader is None:
68
+ raise ValueError("No default source loader registered.")
69
+ return default_loader
70
+
71
+
72
+ @register_default_source_loader
73
+ def load_tfds_splits(
74
+ dataset_name: str,
75
+ splits: list[str],
76
+ data_dir: Union[None, str, os.PathLike] = None,
77
+ ) -> list[tf.data.Dataset]:
78
+ return [
79
+ tfds.load(
80
+ dataset_name,
81
+ split=s,
82
+ as_supervised=False,
83
+ shuffle_files=False,
84
+ data_dir=os.fspath(source_cache_dir(data_dir, "tfds")),
85
+ )
86
+ for s in splits
87
+ ]
justdata/core/stats.py ADDED
@@ -0,0 +1,227 @@
1
+ from __future__ import annotations
2
+
3
+ from typing import Any, Literal
4
+
5
+ import numpy as np
6
+ import tensorflow as tf
7
+ from loguru import logger
8
+
9
+ from justdata.core.metadata import apply_metadata_mode, get_metadata_value, python_value
10
+
11
+
12
+ def make_stats_iterator(
13
+ dataset: tf.data.Dataset,
14
+ *,
15
+ groupby: str | list[str] | None = None,
16
+ batch_size: int,
17
+ deterministic: bool = True,
18
+ augment: bool = False,
19
+ metadata_mode: Literal["full", "numeric_only", "none"] = "numeric_only",
20
+ ):
21
+ if augment:
22
+ logger.warning(
23
+ "Statistics iterator requested augment=True; no augmentation is applied."
24
+ )
25
+ if metadata_mode not in {"full", "numeric_only", "none"}:
26
+ raise ValueError(
27
+ "metadata_mode must be one of 'full', 'numeric_only', or 'none'."
28
+ )
29
+
30
+ options = tf.data.Options()
31
+ options.deterministic = deterministic
32
+ ds = dataset.with_options(options)
33
+ if metadata_mode != "full":
34
+ ds = ds.map(
35
+ lambda sample: _apply_stats_metadata_mode(
36
+ sample, metadata_mode=metadata_mode, groupby=groupby
37
+ ),
38
+ num_parallel_calls=tf.data.AUTOTUNE,
39
+ deterministic=deterministic,
40
+ )
41
+ ds = ds.batch(batch_size)
42
+ return ds.as_numpy_iterator()
43
+
44
+
45
+ def _apply_stats_metadata_mode(
46
+ sample: dict,
47
+ *,
48
+ metadata_mode: Literal["full", "numeric_only", "none"],
49
+ groupby: str | list[str] | None,
50
+ ) -> dict:
51
+ result = apply_metadata_mode(sample, metadata_mode)
52
+ if metadata_mode != "numeric_only" or groupby is None:
53
+ return result
54
+
55
+ metadata = sample.get("metadata")
56
+ if not isinstance(metadata, dict):
57
+ return result
58
+
59
+ keys = (groupby,) if isinstance(groupby, str) else tuple(groupby)
60
+ grouping_metadata = {key: metadata[key] for key in keys if key in metadata}
61
+ if not grouping_metadata:
62
+ return result
63
+
64
+ result = dict(result)
65
+ result["metadata"] = {**result.get("metadata", {}), **grouping_metadata}
66
+ return result
67
+
68
+
69
+ def _to_numpy(value: Any) -> np.ndarray:
70
+ if hasattr(value, "numpy"):
71
+ value = value.numpy()
72
+ return np.asarray(value)
73
+
74
+
75
+ def _axis_index(axis: str | int, rank: int) -> int:
76
+ if isinstance(axis, int):
77
+ return axis if axis >= 0 else rank + axis
78
+ if axis == "time":
79
+ return 0
80
+ if axis in {"frequency", "freq"}:
81
+ return 1 if rank > 1 else 0
82
+ if axis == "channel":
83
+ return rank - 1
84
+ raise ValueError(f"Unknown feature axis: {axis!r}.")
85
+
86
+
87
+ def _observations(features: np.ndarray, axes: tuple[str | int, ...]) -> np.ndarray:
88
+ if not axes:
89
+ return features.reshape((1,) + features.shape)
90
+
91
+ rank = features.ndim
92
+ reduce_axes = sorted({_axis_index(axis, rank) for axis in axes})
93
+ remaining_axes = [axis for axis in range(rank) if axis not in reduce_axes]
94
+ moved = np.moveaxis(features, reduce_axes + remaining_axes, range(rank))
95
+ reduce_size = int(
96
+ np.prod([features.shape[axis] for axis in reduce_axes], dtype=np.int64)
97
+ )
98
+ remaining_shape = tuple(features.shape[axis] for axis in remaining_axes)
99
+ return moved.reshape((reduce_size,) + remaining_shape)
100
+
101
+
102
+ def _group_value(metadata: dict, groupby: str | list[str] | None) -> Any:
103
+ if groupby is None:
104
+ return "all"
105
+ if isinstance(groupby, str):
106
+ return python_value(metadata.get(groupby))
107
+ return tuple(python_value(metadata.get(key)) for key in groupby)
108
+
109
+
110
+ def _hashable_group_value(value: Any) -> Any:
111
+ value = python_value(value)
112
+ if isinstance(value, list):
113
+ return tuple(_hashable_group_value(child) for child in value)
114
+ return value
115
+
116
+
117
+ def _iter_group_feature_pairs(
118
+ dataset,
119
+ *,
120
+ groupby: str | list[str] | None,
121
+ feature_key: str,
122
+ ):
123
+ for sample in dataset:
124
+ features = _to_numpy(sample[feature_key])
125
+ metadata = sample.get("metadata", {})
126
+ if not isinstance(metadata, dict):
127
+ metadata = {}
128
+
129
+ group_value = _group_value(metadata, groupby)
130
+ if isinstance(group_value, list):
131
+ group_value = tuple(group_value)
132
+
133
+ batched_group = None
134
+ if groupby is not None:
135
+ keys = (groupby,) if isinstance(groupby, str) else tuple(groupby)
136
+ values = [python_value(get_metadata_value(sample, key)) for key in keys]
137
+ if values and all(hasattr(np.asarray(value), "shape") for value in values):
138
+ arrays = [np.asarray(value) for value in values]
139
+ if features.ndim > 0 and all(
140
+ array.shape[:1] == features.shape[:1] for array in arrays
141
+ ):
142
+ batched_group = (
143
+ arrays[0] if len(arrays) == 1 else list(zip(*arrays))
144
+ )
145
+
146
+ if batched_group is not None:
147
+ for idx, value in enumerate(batched_group):
148
+ yield _hashable_group_value(value), features[idx]
149
+ else:
150
+ yield group_value, features
151
+
152
+
153
+ def _partial_state(
154
+ observations: np.ndarray,
155
+ ) -> tuple[int, np.ndarray | None, np.ndarray | None]:
156
+ count = int(observations.shape[0])
157
+ if count == 0:
158
+ return 0, None, None
159
+
160
+ mean = np.mean(observations, axis=0, dtype=np.float64)
161
+ centered = observations - mean
162
+ m2 = np.sum(centered * centered, axis=0, dtype=np.float64)
163
+ return count, mean, m2
164
+
165
+
166
+ def _merge_partial_state(
167
+ state: dict[str, Any],
168
+ count_b: int,
169
+ mean_b: np.ndarray | None,
170
+ m2_b: np.ndarray | None,
171
+ ) -> None:
172
+ if count_b == 0:
173
+ return
174
+ if state["count"] == 0:
175
+ state["count"] = count_b
176
+ state["mean"] = mean_b
177
+ state["m2"] = m2_b
178
+ return
179
+
180
+ count_a = state["count"]
181
+ count = count_a + count_b
182
+ delta = mean_b - state["mean"]
183
+ state["mean"] = state["mean"] + delta * count_b / count
184
+ state["m2"] = state["m2"] + m2_b + delta * delta * count_a * count_b / count
185
+ state["count"] = count
186
+
187
+
188
+ def compute_feature_stats(
189
+ dataset,
190
+ axes: tuple[str | int, ...] = ("time",),
191
+ groupby: str | list[str] | None = "device",
192
+ feature_key: str = "inputs",
193
+ ) -> dict:
194
+ states: dict[Any, dict[str, Any]] = {}
195
+ axes = tuple(axes)
196
+
197
+ for group_value, features in _iter_group_feature_pairs(
198
+ dataset,
199
+ groupby=groupby,
200
+ feature_key=feature_key,
201
+ ):
202
+ observations = _observations(np.asarray(features, dtype=np.float64), axes)
203
+ state = states.setdefault(
204
+ group_value,
205
+ {"count": 0, "mean": None, "m2": None},
206
+ )
207
+ _merge_partial_state(state, *_partial_state(observations))
208
+
209
+ result = {}
210
+ for group_value, state in states.items():
211
+ count = int(state["count"])
212
+ mean = state["mean"] if state["mean"] is not None else np.asarray(0.0)
213
+ variance = state["m2"] / count if count else np.zeros_like(mean)
214
+ result[group_value] = {
215
+ "mean": tf.convert_to_tensor(mean, dtype=tf.float32),
216
+ "std": tf.convert_to_tensor(np.sqrt(variance), dtype=tf.float32),
217
+ "count": count,
218
+ "axes": axes,
219
+ "feature_key": feature_key,
220
+ }
221
+ return result
222
+
223
+
224
+ __all__ = [
225
+ "compute_feature_stats",
226
+ "make_stats_iterator",
227
+ ]
@@ -0,0 +1,52 @@
1
+ from __future__ import annotations
2
+
3
+ from typing import Literal
4
+
5
+
6
+ def create_corruption_datasets(
7
+ modality: Literal["vision", "acoustic"],
8
+ corruption_types: list[str],
9
+ severity: int,
10
+ base_dataset: str,
11
+ preset: str | None,
12
+ *,
13
+ split: str = "validation",
14
+ seed: int = 0,
15
+ **load_kwargs,
16
+ ):
17
+ if modality == "vision":
18
+ import justdata.vision # noqa: F401
19
+ from justdata.core.registry import get_pipeline
20
+ from justdata.vision.minic import create_minic_datasets
21
+
22
+ kwargs = dict(load_kwargs)
23
+ kwargs.setdefault("dataset_names_arg", base_dataset)
24
+ kwargs.setdefault("splits_arg", split)
25
+ kwargs.setdefault("dataset_type", "train" if split == "train" else "validation")
26
+ kwargs.setdefault("seed", seed)
27
+ if "pipeline" not in kwargs and "postprocess_fn" not in kwargs:
28
+ kwargs["pipeline"] = get_pipeline(dataset=base_dataset, preset=preset)
29
+ return create_minic_datasets(
30
+ corruption_types=corruption_types,
31
+ severity=severity,
32
+ **kwargs,
33
+ )
34
+
35
+ if modality == "acoustic":
36
+ import justdata.acoustic # noqa: F401
37
+ from justdata.acoustic.corruptions import create_audio_corruption_datasets
38
+
39
+ return create_audio_corruption_datasets(
40
+ corruption_types=corruption_types,
41
+ severity=severity,
42
+ base_dataset=base_dataset,
43
+ preset=preset,
44
+ split=split,
45
+ seed=seed,
46
+ **load_kwargs,
47
+ )
48
+
49
+ raise ValueError("modality must be 'vision' or 'acoustic'.")
50
+
51
+
52
+ __all__ = ["create_corruption_datasets"]
justdata/py.typed ADDED
File without changes
@@ -0,0 +1,17 @@
1
+ from justdata.vision import (
2
+ augmentations,
3
+ corruptions,
4
+ datasets,
5
+ pipelines,
6
+ presets,
7
+ sources,
8
+ )
9
+
10
+ __all__ = [
11
+ "augmentations",
12
+ "corruptions",
13
+ "datasets",
14
+ "pipelines",
15
+ "presets",
16
+ "sources",
17
+ ]
@@ -0,0 +1,31 @@
1
+ from justdata.vision.augmentations import auto, color, composed, geometric, mixing
2
+ from justdata.vision.augmentations.registry import (
3
+ get_augment_strategy,
4
+ get_augment_strategy_metadata,
5
+ get_crop_strategy,
6
+ get_crop_strategy_metadata,
7
+ list_augment_strategies,
8
+ list_augment_strategy_metadata,
9
+ list_crop_strategies,
10
+ list_crop_strategy_metadata,
11
+ register_augment_strategy,
12
+ register_crop_strategy,
13
+ )
14
+
15
+ __all__ = [
16
+ "auto",
17
+ "color",
18
+ "composed",
19
+ "geometric",
20
+ "get_augment_strategy",
21
+ "get_augment_strategy_metadata",
22
+ "get_crop_strategy",
23
+ "get_crop_strategy_metadata",
24
+ "list_augment_strategies",
25
+ "list_augment_strategy_metadata",
26
+ "list_crop_strategies",
27
+ "list_crop_strategy_metadata",
28
+ "mixing",
29
+ "register_augment_strategy",
30
+ "register_crop_strategy",
31
+ ]