justdata 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. justdata/__init__.py +3 -0
  2. justdata/acoustic/__init__.py +381 -0
  3. justdata/acoustic/adapters.py +329 -0
  4. justdata/acoustic/augment/__init__.py +119 -0
  5. justdata/acoustic/augment/batch.py +813 -0
  6. justdata/acoustic/augment/configs.py +204 -0
  7. justdata/acoustic/augment/patchout.py +331 -0
  8. justdata/acoustic/augment/spectrogram.py +806 -0
  9. justdata/acoustic/augment/waveform.py +879 -0
  10. justdata/acoustic/batching.py +27 -0
  11. justdata/acoustic/channel.py +53 -0
  12. justdata/acoustic/compat/__init__.py +95 -0
  13. justdata/acoustic/compat/ast.py +626 -0
  14. justdata/acoustic/compat/ced.py +152 -0
  15. justdata/acoustic/compat/efficientat.py +243 -0
  16. justdata/acoustic/compat/panns.py +115 -0
  17. justdata/acoustic/compat/passt.py +191 -0
  18. justdata/acoustic/configs.py +519 -0
  19. justdata/acoustic/corruptions/__init__.py +33 -0
  20. justdata/acoustic/corruptions/codec.py +92 -0
  21. justdata/acoustic/corruptions/datasets.py +155 -0
  22. justdata/acoustic/corruptions/device.py +39 -0
  23. justdata/acoustic/corruptions/dynamics.py +55 -0
  24. justdata/acoustic/corruptions/filters.py +105 -0
  25. justdata/acoustic/corruptions/noise.py +173 -0
  26. justdata/acoustic/corruptions/registry.py +230 -0
  27. justdata/acoustic/corruptions/reverb.py +49 -0
  28. justdata/acoustic/corruptions/timefreq.py +127 -0
  29. justdata/acoustic/datasets.py +52 -0
  30. justdata/acoustic/dcase2025.py +814 -0
  31. justdata/acoustic/decoding.py +73 -0
  32. justdata/acoustic/eval_views.py +54 -0
  33. justdata/acoustic/evals.py +71 -0
  34. justdata/acoustic/filters.py +18 -0
  35. justdata/acoustic/frontends/__init__.py +37 -0
  36. justdata/acoustic/frontends/ast.py +116 -0
  37. justdata/acoustic/frontends/kaldi.py +34 -0
  38. justdata/acoustic/frontends/log.py +94 -0
  39. justdata/acoustic/frontends/mel.py +173 -0
  40. justdata/acoustic/frontends/mfcc.py +26 -0
  41. justdata/acoustic/frontends/pcen.py +26 -0
  42. justdata/acoustic/frontends/stft.py +174 -0
  43. justdata/acoustic/jax.py +23 -0
  44. justdata/acoustic/labels.py +169 -0
  45. justdata/acoustic/layouts.py +86 -0
  46. justdata/acoustic/metadata.py +152 -0
  47. justdata/acoustic/normalization.py +196 -0
  48. justdata/acoustic/pipelines.py +112 -0
  49. justdata/acoustic/postprocessing.py +328 -0
  50. justdata/acoustic/preprocessing.py +175 -0
  51. justdata/acoustic/presets.py +257 -0
  52. justdata/acoustic/registry.py +381 -0
  53. justdata/acoustic/resampling.py +218 -0
  54. justdata/acoustic/schema.py +51 -0
  55. justdata/acoustic/segment.py +369 -0
  56. justdata/acoustic/sources.py +503 -0
  57. justdata/acoustic/stages.py +59 -0
  58. justdata/acoustic/stats.py +50 -0
  59. justdata/acoustic/tasks.py +375 -0
  60. justdata/audio/__init__.py +20 -0
  61. justdata/core/__init__.py +74 -0
  62. justdata/core/adapters.py +46 -0
  63. justdata/core/augmentations.py +34 -0
  64. justdata/core/filters.py +94 -0
  65. justdata/core/finalization.py +173 -0
  66. justdata/core/label_mixing.py +129 -0
  67. justdata/core/loader.py +384 -0
  68. justdata/core/metadata.py +250 -0
  69. justdata/core/presets.py +155 -0
  70. justdata/core/registry.py +229 -0
  71. justdata/core/sources.py +87 -0
  72. justdata/core/stats.py +227 -0
  73. justdata/corruptions.py +52 -0
  74. justdata/py.typed +0 -0
  75. justdata/vision/__init__.py +17 -0
  76. justdata/vision/augmentations/__init__.py +31 -0
  77. justdata/vision/augmentations/auto.py +480 -0
  78. justdata/vision/augmentations/color.py +163 -0
  79. justdata/vision/augmentations/composed.py +156 -0
  80. justdata/vision/augmentations/geometric.py +302 -0
  81. justdata/vision/augmentations/mixing.py +232 -0
  82. justdata/vision/augmentations/registry.py +142 -0
  83. justdata/vision/corruptions/__init__.py +16 -0
  84. justdata/vision/corruptions/blur.py +65 -0
  85. justdata/vision/corruptions/digital.py +26 -0
  86. justdata/vision/corruptions/noise.py +20 -0
  87. justdata/vision/corruptions/registry.py +113 -0
  88. justdata/vision/corruptions/weather.py +100 -0
  89. justdata/vision/datasets.py +34 -0
  90. justdata/vision/encodings/__init__.py +1 -0
  91. justdata/vision/encodings/heatmaps.py +196 -0
  92. justdata/vision/minic.py +69 -0
  93. justdata/vision/pipelines.py +62 -0
  94. justdata/vision/presets.py +509 -0
  95. justdata/vision/sources.py +1080 -0
  96. justdata/vision/stages.py +271 -0
  97. justdata/vision/tasks/__init__.py +1 -0
  98. justdata/vision/tasks/classification.py +321 -0
  99. justdata/vision/tasks/depth.py +25 -0
  100. justdata/vision/tasks/segmentation.py +194 -0
  101. justdata/vision/transforms.py +113 -0
  102. justdata/vision/utils.py +741 -0
  103. justdata-1.0.0.dist-info/METADATA +860 -0
  104. justdata-1.0.0.dist-info/RECORD +106 -0
  105. justdata-1.0.0.dist-info/WHEEL +4 -0
  106. justdata-1.0.0.dist-info/licenses/LICENSE +23 -0
@@ -0,0 +1,196 @@
1
+ import tensorflow as tf
2
+
3
+
4
+ @tf.function
5
+ def bboxes_to_heatmaps(image_tensor, bboxes_tensor, labels_tensor, num_classes):
6
+ """
7
+ Converts normalized bounding boxes [ymin, xmin, ymax, xmax] to heatmaps.
8
+
9
+ Args:
10
+ image_tensor: The image tensor (H, W, C).
11
+ bboxes_tensor: The bounding boxes tensor (N, 4), normalized coordinates.
12
+ labels_tensor: The class labels tensor (N,).
13
+ num_classes: The total number of classes (e.g., 20 for VOC).
14
+
15
+ Returns:
16
+ A tensor of heatmaps (H, W, num_classes).
17
+ """
18
+ height = tf.shape(image_tensor)[0]
19
+ width = tf.shape(image_tensor)[1]
20
+ num_objects = tf.shape(bboxes_tensor)[0]
21
+
22
+ initial_heatmaps = tf.zeros((height, width, num_classes), dtype=tf.float32)
23
+
24
+ # Loop condition and body
25
+ def condition(i, heatmaps):
26
+ return i < num_objects
27
+
28
+ def body(i, heatmaps):
29
+ bbox = bboxes_tensor[i]
30
+ label = tf.cast(labels_tensor[i], tf.int32)
31
+
32
+ # Denormalize coordinates
33
+ y1_f = bbox[0] * tf.cast(height, tf.float32)
34
+ x1_f = bbox[1] * tf.cast(width, tf.float32)
35
+ y2_f = bbox[2] * tf.cast(height, tf.float32)
36
+ x2_f = bbox[3] * tf.cast(width, tf.float32)
37
+
38
+ y1 = tf.cast(tf.math.floor(y1_f), tf.int32)
39
+ x1 = tf.cast(tf.math.floor(x1_f), tf.int32)
40
+ y2 = tf.cast(tf.math.ceil(y2_f), tf.int32)
41
+ x2 = tf.cast(tf.math.ceil(x2_f), tf.int32)
42
+
43
+ # Clip coordinates to be within image bounds
44
+ y1 = tf.maximum(0, y1)
45
+ x1 = tf.maximum(0, x1)
46
+ y2 = tf.minimum(height, y2)
47
+ x2 = tf.minimum(width, x2)
48
+
49
+ # Update heatmaps if the box is valid
50
+ def update_heatmap():
51
+ rows, cols = tf.meshgrid(tf.range(y1, y2), tf.range(x1, x2), indexing="ij")
52
+ label_indices = tf.fill(tf.shape(rows), label)
53
+
54
+ # Shape: (num_pixels_in_box, 3) where each row is [row_idx, col_idx, label]
55
+ indices = tf.stack(
56
+ [
57
+ tf.reshape(rows, [-1]),
58
+ tf.reshape(cols, [-1]),
59
+ tf.reshape(label_indices, [-1]),
60
+ ],
61
+ axis=1,
62
+ )
63
+
64
+ # Update the main class_heatmaps tensor at the specific class slice
65
+ updates = tf.ones(tf.shape(indices)[0], dtype=tf.float32)
66
+ updated_heatmaps = tf.tensor_scatter_nd_add(heatmaps, indices, updates)
67
+ return updated_heatmaps
68
+
69
+ # Unchanged heatmaps if the box is invalid
70
+ def no_update():
71
+ return heatmaps
72
+
73
+ # Conditionally update the heatmap based on box validity
74
+ is_valid_box = tf.logical_and(y1 < y2, x1 < x2)
75
+ updated_heatmaps = tf.cond(is_valid_box, update_heatmap, no_update)
76
+
77
+ return i + 1, updated_heatmaps
78
+
79
+ _, final_heatmaps = tf.while_loop(
80
+ condition,
81
+ body,
82
+ loop_vars=[tf.constant(0), initial_heatmaps],
83
+ shape_invariants=[
84
+ tf.TensorShape([]), # Shape of loop counter 'i'
85
+ tf.TensorShape([None, None, num_classes]), # Shape of heatmaps
86
+ ],
87
+ )
88
+
89
+ # Clip the final heatmap values to be between 0 and 1
90
+ final_heatmaps = tf.clip_by_value(final_heatmaps, 0.0, 1.0)
91
+
92
+ return final_heatmaps
93
+
94
+
95
+ @tf.function
96
+ def bboxes_to_gaussian_heatmaps(
97
+ image_tensor, bboxes_tensor, labels_tensor, num_classes
98
+ ):
99
+ """
100
+ Converts normalized bounding boxes [ymin, xmin, ymax, xmax] to Gaussian heatmaps
101
+ confined strictly within the bounding box.
102
+
103
+ Args:
104
+ image_tensor: The image tensor (H, W, C).
105
+ bboxes_tensor: The bounding boxes tensor (N, 4), normalized coordinates.
106
+ labels_tensor: The class labels tensor (N,).
107
+ num_classes: The total number of classes (e.g., 20 for VOC).
108
+
109
+ Returns:
110
+ A tensor of Gaussian heatmaps (H, W, num_classes).
111
+ """
112
+ height = tf.shape(image_tensor)[0]
113
+ width = tf.shape(image_tensor)[1]
114
+ num_objects = tf.shape(bboxes_tensor)[0]
115
+
116
+ initial_heatmaps = tf.zeros((height, width, num_classes), dtype=tf.float32)
117
+
118
+ y_grid, x_grid = tf.meshgrid(
119
+ tf.range(height, dtype=tf.float32),
120
+ tf.range(width, dtype=tf.float32),
121
+ indexing="ij",
122
+ )
123
+
124
+ # Loop condition and body
125
+ def condition(i, heatmaps):
126
+ return i < num_objects
127
+
128
+ def body(i, heatmaps):
129
+ bbox = bboxes_tensor[i]
130
+ label = tf.cast(labels_tensor[i], tf.int32)
131
+
132
+ # Denormalize coordinates for the original box
133
+ y1 = bbox[0] * tf.cast(height, tf.float32)
134
+ x1 = bbox[1] * tf.cast(width, tf.float32)
135
+ y2 = bbox[2] * tf.cast(height, tf.float32)
136
+ x2 = bbox[3] * tf.cast(width, tf.float32)
137
+
138
+ center_y = (y1 + y2) / 2.0
139
+ center_x = (x1 + x2) / 2.0
140
+ box_height = tf.maximum(y2 - y1, 1.0)
141
+ box_width = tf.maximum(x2 - x1, 1.0)
142
+
143
+ # Adjust sigma calculation if needed. Dividing by a larger number
144
+ # will make the Gaussian narrower within the box.
145
+ sigma_y = box_height / 4.0
146
+ sigma_x = box_width / 4.0
147
+
148
+ # Calculate gaussian values over the entire grid
149
+ squared_dist_y = tf.square(y_grid - center_y)
150
+ squared_dist_x = tf.square(x_grid - center_x)
151
+ # Add small epsilon to avoid division by zero in edge cases (e.g., 1-pixel box)
152
+ epsilon = 1e-6
153
+ gaussian = tf.exp(
154
+ -0.5
155
+ * (
156
+ squared_dist_y / (tf.square(sigma_y) + epsilon)
157
+ + squared_dist_x / (tf.square(sigma_x) + epsilon)
158
+ )
159
+ )
160
+
161
+ # Create a mask strictly within the original bounding box
162
+ mask = tf.logical_and(
163
+ tf.logical_and(y_grid >= y1, y_grid <= y2),
164
+ tf.logical_and(x_grid >= x1, x_grid <= x2),
165
+ )
166
+ masked_gaussian = tf.where(mask, gaussian, tf.zeros_like(gaussian))
167
+
168
+ class_heatmap = heatmaps[:, :, label]
169
+ updated_class_heatmap = tf.maximum(class_heatmap, masked_gaussian)
170
+
171
+ # Updated heatmaps tensor with the new class channel
172
+ indices = tf.stack(
173
+ [
174
+ tf.range(height)[:, tf.newaxis] * tf.ones_like(x_grid, dtype=tf.int32),
175
+ tf.ones_like(y_grid, dtype=tf.int32) * tf.range(width)[tf.newaxis, :],
176
+ tf.ones((height, width), dtype=tf.int32) * label,
177
+ ],
178
+ axis=-1,
179
+ )
180
+ updated_heatmaps = tf.tensor_scatter_nd_update(
181
+ heatmaps, indices, updated_class_heatmap
182
+ )
183
+
184
+ return i + 1, updated_heatmaps
185
+
186
+ _, final_heatmaps = tf.while_loop(
187
+ condition,
188
+ body,
189
+ loop_vars=[tf.constant(0), initial_heatmaps],
190
+ shape_invariants=[
191
+ tf.TensorShape([]), # Shape of loop counter 'i'
192
+ tf.TensorShape([None, None, num_classes]), # Shape of heatmaps
193
+ ],
194
+ )
195
+
196
+ return final_heatmaps
@@ -0,0 +1,69 @@
1
+ from typing import Union
2
+
3
+ import tensorflow as tf
4
+
5
+ from justdata.core.loader import load_ds
6
+ from justdata.vision.corruptions.registry import (
7
+ _metadata_with_corruption,
8
+ apply_minic_corruption,
9
+ )
10
+
11
+
12
+ _MAX_SEED = tf.constant(2**31 - 1, dtype=tf.int64)
13
+
14
+
15
+ def create_minic_datasets(
16
+ corruption_types: Union[str, list[str]], severity: int = 3, **load_ds_kwargs
17
+ ) -> Union[tuple[tf.data.Dataset, int], tuple[list[tf.data.Dataset], int]]:
18
+ """
19
+ Create Mini-C corruption benchmark datasets from a preprocessed base dataset.
20
+ """
21
+
22
+ load_ds_kwargs["return_raw_ds"] = True
23
+
24
+ ds, tools = load_ds(**load_ds_kwargs)
25
+
26
+ finalize_fn = tools["finalize_fn"]
27
+ seed = load_ds_kwargs.get("seed", 0)
28
+
29
+ return_list = isinstance(corruption_types, list)
30
+ c_list = corruption_types if return_list else [corruption_types]
31
+
32
+ datasets_out = []
33
+
34
+ for c_index, c_name in enumerate(c_list):
35
+
36
+ def corrupt_fn(index, sample, c_name=c_name, c_index=c_index):
37
+ s_seed = _seed_from_index(seed, index, salt=c_index * 1_000_003)
38
+ img = sample["image"]
39
+ img_corrupted = apply_minic_corruption(img, c_name, severity, s_seed)
40
+ return _metadata_with_corruption(
41
+ sample | {"image": img_corrupted},
42
+ name=c_name,
43
+ severity=severity,
44
+ domain="image",
45
+ )
46
+
47
+ ds_c = ds.enumerate().map(
48
+ corrupt_fn,
49
+ num_parallel_calls=tf.data.AUTOTUNE,
50
+ deterministic=True,
51
+ )
52
+ ds_c, n_batches = finalize_fn(ds_c)
53
+ datasets_out.append(ds_c)
54
+
55
+ if not datasets_out:
56
+ n_batches = 0
57
+
58
+ if return_list:
59
+ return datasets_out, n_batches
60
+ return datasets_out[0], n_batches
61
+
62
+
63
+ def _seed_from_index(
64
+ seed: int | tf.Tensor, index: tf.Tensor, salt: int = 0
65
+ ) -> tf.Tensor:
66
+ base = tf.cast(seed, tf.int64)
67
+ idx = tf.cast(index, tf.int64)
68
+ salted = tf.math.floormod(base * 1_103_515_245 + idx + salt, _MAX_SEED)
69
+ return tf.cast(tf.stack([tf.math.floormod(base, _MAX_SEED), salted]), tf.int32)
@@ -0,0 +1,62 @@
1
+ from justdata.core.registry import PipelineFuncs, register_pipeline
2
+
3
+
4
+ @register_pipeline("vision/classification")
5
+ def default_classification_pipeline(
6
+ preproc_kwargs: dict = None,
7
+ aug_kwargs: dict = None,
8
+ laug_kwargs: dict = None,
9
+ postproc_kwargs: dict = None,
10
+ **kwargs,
11
+ ) -> PipelineFuncs:
12
+ preproc_kwargs = preproc_kwargs or {}
13
+ aug_kwargs = aug_kwargs or {}
14
+ laug_kwargs = laug_kwargs or {}
15
+ postproc_kwargs = postproc_kwargs or {}
16
+
17
+ if laug_kwargs.get("mixup_alpha", 0) > 0 or laug_kwargs.get("cutmix_alpha", 0) > 0:
18
+ postproc_kwargs.setdefault("one_hot_labels", True)
19
+ if "label_mode" in laug_kwargs:
20
+ postproc_kwargs.setdefault("label_mode", laug_kwargs["label_mode"])
21
+
22
+ from justdata.vision.tasks.classification import (
23
+ make_augmentations,
24
+ make_late_augmentations,
25
+ make_postprocessing,
26
+ make_preprocessing,
27
+ )
28
+
29
+ return (
30
+ make_preprocessing(**preproc_kwargs),
31
+ make_augmentations(**aug_kwargs),
32
+ make_late_augmentations(**laug_kwargs),
33
+ make_postprocessing(**postproc_kwargs),
34
+ )
35
+
36
+
37
+ @register_pipeline("vision/segmentation")
38
+ def default_segmentation_pipeline(
39
+ preproc_kwargs: dict = None,
40
+ aug_kwargs: dict = None,
41
+ laug_kwargs: dict = None,
42
+ postproc_kwargs: dict = None,
43
+ **kwargs,
44
+ ) -> PipelineFuncs:
45
+ preproc_kwargs = preproc_kwargs or {}
46
+ aug_kwargs = aug_kwargs or {}
47
+ laug_kwargs = laug_kwargs or {}
48
+ postproc_kwargs = postproc_kwargs or {}
49
+
50
+ from justdata.vision.tasks.segmentation import (
51
+ make_augmentations,
52
+ make_late_augmentations,
53
+ make_postprocessing,
54
+ make_preprocessing,
55
+ )
56
+
57
+ return (
58
+ make_preprocessing(**preproc_kwargs),
59
+ make_augmentations(**aug_kwargs),
60
+ make_late_augmentations(**laug_kwargs),
61
+ make_postprocessing(**postproc_kwargs),
62
+ )