justdata 1.2.1__tar.gz → 1.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {justdata-1.2.1 → justdata-1.5.0}/PKG-INFO +69 -5
- {justdata-1.2.1 → justdata-1.5.0}/README.md +67 -4
- {justdata-1.2.1 → justdata-1.5.0}/pyproject.toml +2 -1
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/__init__.py +1 -1
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/core/execution.py +5 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/core/finalization.py +3 -1
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/core/loader.py +5 -1
- justdata-1.5.0/src/justdata/vision/colorspace.py +71 -0
- justdata-1.5.0/src/justdata/vision/compositing.py +717 -0
- justdata-1.5.0/src/justdata/vision/corruptions/__init__.py +65 -0
- justdata-1.5.0/src/justdata/vision/corruptions/_conditions.py +73 -0
- justdata-1.5.0/src/justdata/vision/corruptions/atmosphere.py +206 -0
- justdata-1.5.0/src/justdata/vision/corruptions/continuous.py +467 -0
- justdata-1.5.0/src/justdata/vision/corruptions/exposure.py +235 -0
- justdata-1.5.0/src/justdata/vision/corruptions/lens.py +183 -0
- justdata-1.5.0/src/justdata/vision/corruptions/motion.py +86 -0
- justdata-1.5.0/src/justdata/vision/corruptions/precipitation.py +153 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/corruptions/registry.py +328 -9
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/corruptions/standard.py +8 -38
- justdata-1.5.0/src/justdata/vision/corruptions/water.py +175 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/encodings/panoptic_targets.py +121 -34
- justdata-1.5.0/src/justdata/vision/filters.py +154 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/geometry.py +267 -35
- justdata-1.5.0/src/justdata/vision/horizon.py +81 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/pipelines.py +15 -4
- justdata-1.5.0/src/justdata/vision/poisson.py +177 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/stages.py +2 -2
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/tasks/panoptic.py +7 -1
- justdata-1.2.1/src/justdata/vision/corruptions/__init__.py +0 -27
- {justdata-1.2.1 → justdata-1.5.0}/LICENSE +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/__init__.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/_dcase.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/_manifests.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/_random.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/_signal.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/adapters.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/augment/__init__.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/augment/_common.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/augment/batch.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/augment/configs.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/augment/patchout.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/augment/spectrogram.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/augment/waveform.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/batching.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/channel.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/compat/__init__.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/compat/ast.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/compat/ced.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/compat/efficientat.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/compat/panns.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/compat/passt.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/configs.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/corruptions/__init__.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/corruptions/codec.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/corruptions/datasets.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/corruptions/device.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/corruptions/dynamics.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/corruptions/filters.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/corruptions/noise.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/corruptions/registry.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/corruptions/reverb.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/corruptions/timefreq.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/datasets.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/dcase2025.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/decoding.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/eval_views.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/evals.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/filters.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/frontends/__init__.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/frontends/ast.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/frontends/kaldi.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/frontends/log.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/frontends/mel.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/frontends/mfcc.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/frontends/pcen.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/frontends/stft.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/jax.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/labels.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/layouts.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/metadata.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/normalization.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/pipelines.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/postprocessing.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/preprocessing.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/presets.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/registry.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/resampling.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/schema.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/segment.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/sources.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/stages.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/stats.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/tasks.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/audio/__init__.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/core/__init__.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/core/_records.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/core/adapters.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/core/augmentations.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/core/cache.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/core/config_resolution.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/core/executed_config.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/core/filters.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/core/inventory.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/core/label_mixing.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/core/metadata.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/core/presets.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/core/registry.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/core/replay.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/core/sources.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/core/stats.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/corruptions.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/py.typed +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/__init__.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/augmentations/__init__.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/augmentations/auto.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/augmentations/color.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/augmentations/composed.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/augmentations/geometric.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/augmentations/mixing.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/augmentations/registry.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/corruptions/blur.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/corruptions/digital.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/corruptions/noise.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/corruptions/weather.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/datasets.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/encodings/__init__.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/encodings/heatmaps.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/encodings/semantic_targets.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/minic.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/presets.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/sources.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/tasks/__init__.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/tasks/classification.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/tasks/depth.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/tasks/segmentation.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/transforms.py +0 -0
- {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: justdata
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.5.0
|
|
4
4
|
Summary: A TensorFlow-native data pipeline library with a modality-neutral core and first-class computer vision and acoustic recipes
|
|
5
5
|
Author: clementpoiret
|
|
6
6
|
Author-email: clementpoiret <clement@linux.com>
|
|
@@ -27,6 +27,7 @@ Requires-Dist: soundfile>=0.12 ; extra == 'dev'
|
|
|
27
27
|
Requires-Dist: soxr>=0.5 ; extra == 'dev'
|
|
28
28
|
Requires-Dist: torch==2.10.0 ; extra == 'dev'
|
|
29
29
|
Requires-Dist: torchcodec==0.10.0 ; extra == 'dev'
|
|
30
|
+
Requires-Dist: opencv-python-headless==5.0.0.93 ; extra == 'golden'
|
|
30
31
|
Requires-Dist: torchaudio==2.10.0 ; extra == 'golden'
|
|
31
32
|
Requires-Dist: torchlibrosa==0.1.0 ; extra == 'golden'
|
|
32
33
|
Requires-Dist: datasets[vision]>=4.5.0 ; extra == 'vision'
|
|
@@ -124,6 +125,8 @@ enabled.
|
|
|
124
125
|
| `justdata.core.load_inventory(admitted, ...)` | Apply the shared pipeline to an admitted inventory, including raw epoch finalization. |
|
|
125
126
|
| `justdata.core.ExecutedConfig` | Immutable, canonical snapshot returned by loaders when `return_config=True`. |
|
|
126
127
|
| `justdata.vision.minic.create_minic_datasets(...)` | Constructs Mini-C corruption benchmark datasets from a shared preprocessed base dataset. |
|
|
128
|
+
| `justdata.vision.corruptions.apply_corruption_at(...)` | Renders a scene condition at a continuous strength and returns its record. |
|
|
129
|
+
| `justdata.vision.corruptions.apply_corruption_stack(...)` | Applies ordered conditions in float, rounds once, and returns the stack record. |
|
|
127
130
|
| `justdata.acoustic.corruptions.create_audio_corruption_datasets(...)` | Constructs acoustic corruption benchmark datasets from a shared preprocessed base dataset. |
|
|
128
131
|
| `justdata.acoustic.dcase2025.make_source_dataset(...)` | Builds DCASE Task 1 source-domain datasets with split-safety checks. |
|
|
129
132
|
|
|
@@ -546,7 +549,9 @@ For semantic segmentation with replayable geometry, select `vision/segmentation`
|
|
|
546
549
|
`geometry_kwargs`. It supports paired aspect-preserving training resize and crops, longer-side evaluation caps,
|
|
547
550
|
independent RGB/ignore padding, per-pixel validity, and original-coordinate score restoration. See
|
|
548
551
|
[the dense geometry contract](docs/vision.md#12-replayable-dense-segmentation) for configuration, record fields, replay,
|
|
549
|
-
and batching requirements.
|
|
552
|
+
and batching requirements. For ordered image/mask file lists without a registered source,
|
|
553
|
+
`justdata.vision.geometry.dense_views_from_files` builds the same views with explicit per-example seeds in a parallel,
|
|
554
|
+
order-preserving `tf.data` map.
|
|
550
555
|
|
|
551
556
|
For instance-aware labels, select `vision/panoptic_segmentation` with a segment-ID map and category table. It preserves
|
|
552
557
|
thing instances, merges stuff by category, and excludes crowd pixels from panoptic supervision. The
|
|
@@ -961,6 +966,43 @@ severity; `none` removes metadata according to the shared core contract. The dat
|
|
|
961
966
|
enumeration-position seed salting. Consumers with their own sample-level PRNG lineage should call `apply_corruption`
|
|
962
967
|
directly.
|
|
963
968
|
|
|
969
|
+
### Scene conditions at continuous strength
|
|
970
|
+
|
|
971
|
+
Ten photometric conditions declare a continuous-strength contract: `haze`, `low_light`, `sun_glitter`, `lens_droplets`,
|
|
972
|
+
`rain`, `motion_blur`, `overexposure`, `white_balance_warm`, `white_balance_cool`, and `shot_noise`.
|
|
973
|
+
`apply_corruption_at` renders one of them at any strength between 0 (identity) and 5 (the last severity row). It takes
|
|
974
|
+
an optional episode seed for structure that persists across frames, and optional `depth_m`, `horizon`, or `region`
|
|
975
|
+
inputs. It returns the image and a record of the resolved parameters and mode. `apply_corruption_stack` applies ordered
|
|
976
|
+
steps in float and rounds once.
|
|
977
|
+
|
|
978
|
+
```python
|
|
979
|
+
import tensorflow as tf
|
|
980
|
+
|
|
981
|
+
from justdata.vision.corruptions import apply_corruption_at, apply_corruption_stack
|
|
982
|
+
|
|
983
|
+
frame = apply_corruption_at(
|
|
984
|
+
decoded_rgb_uint8,
|
|
985
|
+
name="lens_droplets",
|
|
986
|
+
version="1.0.0",
|
|
987
|
+
strength=1.7,
|
|
988
|
+
seed=tf.constant([frame_index, 0], tf.int32),
|
|
989
|
+
episode_seed=tf.constant([episode_index, 0], tf.int32),
|
|
990
|
+
)
|
|
991
|
+
dusk = apply_corruption_stack(
|
|
992
|
+
decoded_rgb_uint8,
|
|
993
|
+
steps=[("low_light", "1.0.0", 2.0), ("white_balance_cool", "1.0.0", 1.5)],
|
|
994
|
+
seed=tf.constant([frame_index, 1], tf.int32),
|
|
995
|
+
)
|
|
996
|
+
```
|
|
997
|
+
|
|
998
|
+
See [continuous strength](docs/vision.md#continuous-strength-episodes-and-scene-inputs) and
|
|
999
|
+
[scene condition corruptions](docs/vision.md#scene-condition-corruptions) for the contract, the tables, and their
|
|
1000
|
+
sources, and `examples/vision/maritime_conditions.py` for a drift over an episode.
|
|
1001
|
+
|
|
1002
|
+
To add rare objects, `composite_instances` pastes annotated crops onto caller-chosen anchor pixels, updates the class
|
|
1003
|
+
and instance maps, and records every request as placed or rejected; see
|
|
1004
|
+
[compositing annotated instances](docs/vision.md#16-compositing-annotated-instances).
|
|
1005
|
+
|
|
964
1006
|
______________________________________________________________________
|
|
965
1007
|
|
|
966
1008
|
## Audio Corruption Benchmark
|
|
@@ -1094,14 +1136,36 @@ Replay prefetches after skipping committed batches and defaults to one batch.
|
|
|
1094
1136
|
When another framework uses the accelerator, configure TensorFlow before creating tensors or datasets:
|
|
1095
1137
|
|
|
1096
1138
|
```python
|
|
1139
|
+
import jax
|
|
1140
|
+
|
|
1141
|
+
jax.devices() # initialize JAX's GPU backend before TensorFlow is imported
|
|
1142
|
+
|
|
1097
1143
|
from justdata.core import configure_tensorflow_cpu
|
|
1098
1144
|
|
|
1099
1145
|
device_report = configure_tensorflow_cpu(intra_op_threads=1, inter_op_threads=1)
|
|
1100
1146
|
assert device_report["logical_gpus"] == []
|
|
1101
1147
|
```
|
|
1102
1148
|
|
|
1103
|
-
The report includes physical and logical device names and process thread settings. The helper raises if the
|
|
1104
|
-
initialized too early for the requested change. Core, vision, acoustic, and audio imports
|
|
1149
|
+
The report includes physical and logical device names and process thread settings. The helper raises if the TensorFlow
|
|
1150
|
+
runtime has initialized too early for the requested change. Core, vision, acoustic, and audio imports do not initialize
|
|
1151
|
+
it, so the helper can run after them.
|
|
1152
|
+
|
|
1153
|
+
Those imports do import TensorFlow. On Linux, the CUDA-enabled `tensorflow` wheel then exposes stubs for its CUDA
|
|
1154
|
+
libraries (CUDA 12 as of TensorFlow 2.21) to the whole process. A framework built for another CUDA version that
|
|
1155
|
+
initializes its GPU backend afterwards can bind to those stubs: JAX with its CUDA 13 plugin reports an outdated cuBLAS
|
|
1156
|
+
and falls back to the CPU. Hiding TensorFlow's GPUs does not prevent this. Either initialize the other framework's
|
|
1157
|
+
backend before the first TensorFlow import, as above (`import jax` alone is not enough), or install `tensorflow-cpu`,
|
|
1158
|
+
which exports no CUDA symbols, in place of `tensorflow`. With uv, on Linux x86_64:
|
|
1159
|
+
|
|
1160
|
+
```toml
|
|
1161
|
+
[project]
|
|
1162
|
+
dependencies = ["justdata", "tensorflow-cpu"]
|
|
1163
|
+
|
|
1164
|
+
[tool.uv]
|
|
1165
|
+
exclude-dependencies = ["tensorflow"]
|
|
1166
|
+
```
|
|
1167
|
+
|
|
1168
|
+
Set `JAX_PLATFORMS=cuda` to make a JAX fallback to the CPU an error rather than a warning.
|
|
1105
1169
|
|
|
1106
1170
|
`CachePolicy` enables identity-checked file caches on the existing preprocessing and model-input cache stages. The cache
|
|
1107
1171
|
path is a directory whose parent exists. The input identity must cover ordered content. `load_inventory` binds its
|
|
@@ -1306,7 +1370,7 @@ package-support jobs pass. Build artifacts are retained for one week. Tag pipeli
|
|
|
1306
1370
|
to complete a release.
|
|
1307
1371
|
|
|
1308
1372
|
Before tagging, update `pyproject.toml`, `src/justdata/__init__.py`, and `CHANGELOG.md`, then run `uv lock`. Push a tag
|
|
1309
|
-
matching the package version, with an optional `v` prefix (for example, `v1.
|
|
1373
|
+
matching the package version, with an optional `v` prefix (for example, `v1.5.0` or `1.5.0`). Publishing rejects tags
|
|
1310
1374
|
that disagree with either version declaration. Branch pipelines build artifacts without publishing.
|
|
1311
1375
|
|
|
1312
1376
|
Configure a [PyPI GitLab trusted publisher](https://docs.pypi.org/trusted-publishers/adding-a-publisher/) for the
|
|
@@ -81,6 +81,8 @@ enabled.
|
|
|
81
81
|
| `justdata.core.load_inventory(admitted, ...)` | Apply the shared pipeline to an admitted inventory, including raw epoch finalization. |
|
|
82
82
|
| `justdata.core.ExecutedConfig` | Immutable, canonical snapshot returned by loaders when `return_config=True`. |
|
|
83
83
|
| `justdata.vision.minic.create_minic_datasets(...)` | Constructs Mini-C corruption benchmark datasets from a shared preprocessed base dataset. |
|
|
84
|
+
| `justdata.vision.corruptions.apply_corruption_at(...)` | Renders a scene condition at a continuous strength and returns its record. |
|
|
85
|
+
| `justdata.vision.corruptions.apply_corruption_stack(...)` | Applies ordered conditions in float, rounds once, and returns the stack record. |
|
|
84
86
|
| `justdata.acoustic.corruptions.create_audio_corruption_datasets(...)` | Constructs acoustic corruption benchmark datasets from a shared preprocessed base dataset. |
|
|
85
87
|
| `justdata.acoustic.dcase2025.make_source_dataset(...)` | Builds DCASE Task 1 source-domain datasets with split-safety checks. |
|
|
86
88
|
|
|
@@ -503,7 +505,9 @@ For semantic segmentation with replayable geometry, select `vision/segmentation`
|
|
|
503
505
|
`geometry_kwargs`. It supports paired aspect-preserving training resize and crops, longer-side evaluation caps,
|
|
504
506
|
independent RGB/ignore padding, per-pixel validity, and original-coordinate score restoration. See
|
|
505
507
|
[the dense geometry contract](docs/vision.md#12-replayable-dense-segmentation) for configuration, record fields, replay,
|
|
506
|
-
and batching requirements.
|
|
508
|
+
and batching requirements. For ordered image/mask file lists without a registered source,
|
|
509
|
+
`justdata.vision.geometry.dense_views_from_files` builds the same views with explicit per-example seeds in a parallel,
|
|
510
|
+
order-preserving `tf.data` map.
|
|
507
511
|
|
|
508
512
|
For instance-aware labels, select `vision/panoptic_segmentation` with a segment-ID map and category table. It preserves
|
|
509
513
|
thing instances, merges stuff by category, and excludes crowd pixels from panoptic supervision. The
|
|
@@ -918,6 +922,43 @@ severity; `none` removes metadata according to the shared core contract. The dat
|
|
|
918
922
|
enumeration-position seed salting. Consumers with their own sample-level PRNG lineage should call `apply_corruption`
|
|
919
923
|
directly.
|
|
920
924
|
|
|
925
|
+
### Scene conditions at continuous strength
|
|
926
|
+
|
|
927
|
+
Ten photometric conditions declare a continuous-strength contract: `haze`, `low_light`, `sun_glitter`, `lens_droplets`,
|
|
928
|
+
`rain`, `motion_blur`, `overexposure`, `white_balance_warm`, `white_balance_cool`, and `shot_noise`.
|
|
929
|
+
`apply_corruption_at` renders one of them at any strength between 0 (identity) and 5 (the last severity row). It takes
|
|
930
|
+
an optional episode seed for structure that persists across frames, and optional `depth_m`, `horizon`, or `region`
|
|
931
|
+
inputs. It returns the image and a record of the resolved parameters and mode. `apply_corruption_stack` applies ordered
|
|
932
|
+
steps in float and rounds once.
|
|
933
|
+
|
|
934
|
+
```python
|
|
935
|
+
import tensorflow as tf
|
|
936
|
+
|
|
937
|
+
from justdata.vision.corruptions import apply_corruption_at, apply_corruption_stack
|
|
938
|
+
|
|
939
|
+
frame = apply_corruption_at(
|
|
940
|
+
decoded_rgb_uint8,
|
|
941
|
+
name="lens_droplets",
|
|
942
|
+
version="1.0.0",
|
|
943
|
+
strength=1.7,
|
|
944
|
+
seed=tf.constant([frame_index, 0], tf.int32),
|
|
945
|
+
episode_seed=tf.constant([episode_index, 0], tf.int32),
|
|
946
|
+
)
|
|
947
|
+
dusk = apply_corruption_stack(
|
|
948
|
+
decoded_rgb_uint8,
|
|
949
|
+
steps=[("low_light", "1.0.0", 2.0), ("white_balance_cool", "1.0.0", 1.5)],
|
|
950
|
+
seed=tf.constant([frame_index, 1], tf.int32),
|
|
951
|
+
)
|
|
952
|
+
```
|
|
953
|
+
|
|
954
|
+
See [continuous strength](docs/vision.md#continuous-strength-episodes-and-scene-inputs) and
|
|
955
|
+
[scene condition corruptions](docs/vision.md#scene-condition-corruptions) for the contract, the tables, and their
|
|
956
|
+
sources, and `examples/vision/maritime_conditions.py` for a drift over an episode.
|
|
957
|
+
|
|
958
|
+
To add rare objects, `composite_instances` pastes annotated crops onto caller-chosen anchor pixels, updates the class
|
|
959
|
+
and instance maps, and records every request as placed or rejected; see
|
|
960
|
+
[compositing annotated instances](docs/vision.md#16-compositing-annotated-instances).
|
|
961
|
+
|
|
921
962
|
______________________________________________________________________
|
|
922
963
|
|
|
923
964
|
## Audio Corruption Benchmark
|
|
@@ -1051,14 +1092,36 @@ Replay prefetches after skipping committed batches and defaults to one batch.
|
|
|
1051
1092
|
When another framework uses the accelerator, configure TensorFlow before creating tensors or datasets:
|
|
1052
1093
|
|
|
1053
1094
|
```python
|
|
1095
|
+
import jax
|
|
1096
|
+
|
|
1097
|
+
jax.devices() # initialize JAX's GPU backend before TensorFlow is imported
|
|
1098
|
+
|
|
1054
1099
|
from justdata.core import configure_tensorflow_cpu
|
|
1055
1100
|
|
|
1056
1101
|
device_report = configure_tensorflow_cpu(intra_op_threads=1, inter_op_threads=1)
|
|
1057
1102
|
assert device_report["logical_gpus"] == []
|
|
1058
1103
|
```
|
|
1059
1104
|
|
|
1060
|
-
The report includes physical and logical device names and process thread settings. The helper raises if the
|
|
1061
|
-
initialized too early for the requested change. Core, vision, acoustic, and audio imports
|
|
1105
|
+
The report includes physical and logical device names and process thread settings. The helper raises if the TensorFlow
|
|
1106
|
+
runtime has initialized too early for the requested change. Core, vision, acoustic, and audio imports do not initialize
|
|
1107
|
+
it, so the helper can run after them.
|
|
1108
|
+
|
|
1109
|
+
Those imports do import TensorFlow. On Linux, the CUDA-enabled `tensorflow` wheel then exposes stubs for its CUDA
|
|
1110
|
+
libraries (CUDA 12 as of TensorFlow 2.21) to the whole process. A framework built for another CUDA version that
|
|
1111
|
+
initializes its GPU backend afterwards can bind to those stubs: JAX with its CUDA 13 plugin reports an outdated cuBLAS
|
|
1112
|
+
and falls back to the CPU. Hiding TensorFlow's GPUs does not prevent this. Either initialize the other framework's
|
|
1113
|
+
backend before the first TensorFlow import, as above (`import jax` alone is not enough), or install `tensorflow-cpu`,
|
|
1114
|
+
which exports no CUDA symbols, in place of `tensorflow`. With uv, on Linux x86_64:
|
|
1115
|
+
|
|
1116
|
+
```toml
|
|
1117
|
+
[project]
|
|
1118
|
+
dependencies = ["justdata", "tensorflow-cpu"]
|
|
1119
|
+
|
|
1120
|
+
[tool.uv]
|
|
1121
|
+
exclude-dependencies = ["tensorflow"]
|
|
1122
|
+
```
|
|
1123
|
+
|
|
1124
|
+
Set `JAX_PLATFORMS=cuda` to make a JAX fallback to the CPU an error rather than a warning.
|
|
1062
1125
|
|
|
1063
1126
|
`CachePolicy` enables identity-checked file caches on the existing preprocessing and model-input cache stages. The cache
|
|
1064
1127
|
path is a directory whose parent exists. The input identity must cover ordered content. `load_inventory` binds its
|
|
@@ -1263,7 +1326,7 @@ package-support jobs pass. Build artifacts are retained for one week. Tag pipeli
|
|
|
1263
1326
|
to complete a release.
|
|
1264
1327
|
|
|
1265
1328
|
Before tagging, update `pyproject.toml`, `src/justdata/__init__.py`, and `CHANGELOG.md`, then run `uv lock`. Push a tag
|
|
1266
|
-
matching the package version, with an optional `v` prefix (for example, `v1.
|
|
1329
|
+
matching the package version, with an optional `v` prefix (for example, `v1.5.0` or `1.5.0`). Publishing rejects tags
|
|
1267
1330
|
that disagree with either version declaration. Branch pipelines build artifacts without publishing.
|
|
1268
1331
|
|
|
1269
1332
|
Configure a [PyPI GitLab trusted publisher](https://docs.pypi.org/trusted-publishers/adding-a-publisher/) for the
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "justdata"
|
|
3
|
-
version = "1.
|
|
3
|
+
version = "1.5.0"
|
|
4
4
|
description = "A TensorFlow-native data pipeline library with a modality-neutral core and first-class computer vision and acoustic recipes"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "MIT"
|
|
@@ -33,6 +33,7 @@ acoustic = [
|
|
|
33
33
|
"torchcodec==0.10.0",
|
|
34
34
|
]
|
|
35
35
|
golden = [
|
|
36
|
+
"opencv-python-headless==5.0.0.93",
|
|
36
37
|
"torchaudio==2.10.0",
|
|
37
38
|
"torchlibrosa==0.1.0",
|
|
38
39
|
]
|
|
@@ -14,6 +14,11 @@ def configure_tensorflow_cpu(
|
|
|
14
14
|
|
|
15
15
|
Call before constructing tensors, datasets, or invoking other TensorFlow
|
|
16
16
|
operations. A runtime that was initialized too early raises RuntimeError.
|
|
17
|
+
|
|
18
|
+
Hiding GPUs does not isolate TensorFlow's CUDA libraries. With the
|
|
19
|
+
CUDA-enabled ``tensorflow`` wheel, another framework must initialize its
|
|
20
|
+
GPU backend before TensorFlow is first imported; ``tensorflow-cpu`` has no
|
|
21
|
+
such ordering requirement.
|
|
17
22
|
"""
|
|
18
23
|
for name, value in (
|
|
19
24
|
("intra_op_threads", intra_op_threads),
|
|
@@ -305,13 +305,15 @@ def finalize_dataset(
|
|
|
305
305
|
deterministic=deterministic,
|
|
306
306
|
)
|
|
307
307
|
elif rng is not None:
|
|
308
|
+
# Parallel calls draw from the shared generator in scheduling order,
|
|
309
|
+
# so batches would trade seeds between otherwise identical runs.
|
|
308
310
|
ds = ds.map(
|
|
309
311
|
lambda batch: late_augment_fn(
|
|
310
312
|
batch,
|
|
311
313
|
num_classes=num_classes,
|
|
312
314
|
seed=rng.make_seeds(1)[:, 0],
|
|
313
315
|
),
|
|
314
|
-
num_parallel_calls=parallel_calls,
|
|
316
|
+
num_parallel_calls=1 if deterministic else parallel_calls,
|
|
315
317
|
deterministic=deterministic,
|
|
316
318
|
)
|
|
317
319
|
|
|
@@ -478,6 +478,10 @@ def _prepare_ds(
|
|
|
478
478
|
):
|
|
479
479
|
"""Shared pipeline, resolving its source only after validating options."""
|
|
480
480
|
rng = tf.random.Generator.from_seed(seed)
|
|
481
|
+
# Late augmentation draws in a later tf.data stage that runs concurrently with
|
|
482
|
+
# the per-sample augment map; sharing one generator would interleave the two
|
|
483
|
+
# stages' draws differently on every run.
|
|
484
|
+
late_augment_rng = tf.random.Generator.from_seed(seed).split(1)[0]
|
|
481
485
|
if metadata_mode is None:
|
|
482
486
|
metadata_mode = (
|
|
483
487
|
pipeline.kwargs.get("metadata_mode", "full")
|
|
@@ -685,7 +689,7 @@ def _prepare_ds(
|
|
|
685
689
|
"is_training": is_training,
|
|
686
690
|
"apply_late_augment": apply_augmentation,
|
|
687
691
|
"late_augment_fn": late_augment_fn,
|
|
688
|
-
"rng":
|
|
692
|
+
"rng": late_augment_rng,
|
|
689
693
|
"deterministic": deterministic,
|
|
690
694
|
"shuffle_buffer": shuffle_buffer if is_training else None,
|
|
691
695
|
"shuffle_seed": seed,
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""Float32 RGB colour-space conversions on values in ``[0, 1]``.
|
|
2
|
+
|
|
3
|
+
sRGB follows IEC 61966-2-1 with the D65 white point; CIELAB uses the same
|
|
4
|
+
white, so ``lab_to_rgb(rgb_to_lab(x))`` returns ``x`` up to float32 rounding.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import tensorflow as tf
|
|
8
|
+
|
|
9
|
+
_RGB_TO_XYZ = (
|
|
10
|
+
(0.4124564, 0.3575761, 0.1804375),
|
|
11
|
+
(0.2126729, 0.7151522, 0.0721750),
|
|
12
|
+
(0.0193339, 0.1191920, 0.9503041),
|
|
13
|
+
)
|
|
14
|
+
_D65_WHITE_XYZ = (0.95047, 1.0, 1.08883)
|
|
15
|
+
_LAB_EPSILON = 216.0 / 24389.0
|
|
16
|
+
_LAB_KAPPA = 24389.0 / 27.0
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def srgb_to_linear(image: tf.Tensor) -> tf.Tensor:
|
|
20
|
+
image = tf.cast(image, tf.float32)
|
|
21
|
+
return tf.where(
|
|
22
|
+
image <= 0.04045,
|
|
23
|
+
image / 12.92,
|
|
24
|
+
tf.pow((tf.maximum(image, 0.04045) + 0.055) / 1.055, 2.4),
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def linear_to_srgb(image: tf.Tensor) -> tf.Tensor:
|
|
29
|
+
image = tf.maximum(tf.cast(image, tf.float32), 0.0)
|
|
30
|
+
return tf.where(
|
|
31
|
+
image <= 0.0031308,
|
|
32
|
+
image * 12.92,
|
|
33
|
+
1.055 * tf.pow(tf.maximum(image, 0.0031308), 1.0 / 2.4) - 0.055,
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def rgb_to_lab(image: tf.Tensor) -> tf.Tensor:
|
|
38
|
+
"""Convert sRGB ``[..., 3]`` to CIELAB with L in ``[0, 100]``."""
|
|
39
|
+
linear = srgb_to_linear(image)
|
|
40
|
+
xyz = tf.linalg.matvec(tf.constant(_RGB_TO_XYZ, tf.float32), linear)
|
|
41
|
+
ratio = xyz / tf.constant(_D65_WHITE_XYZ, tf.float32)
|
|
42
|
+
f = tf.where(
|
|
43
|
+
ratio > _LAB_EPSILON,
|
|
44
|
+
tf.pow(tf.maximum(ratio, _LAB_EPSILON), 1.0 / 3.0),
|
|
45
|
+
(_LAB_KAPPA * ratio + 16.0) / 116.0,
|
|
46
|
+
)
|
|
47
|
+
lightness = 116.0 * f[..., 1] - 16.0
|
|
48
|
+
a = 500.0 * (f[..., 0] - f[..., 1])
|
|
49
|
+
b = 200.0 * (f[..., 1] - f[..., 2])
|
|
50
|
+
return tf.stack([lightness, a, b], axis=-1)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def lab_to_rgb(lab: tf.Tensor) -> tf.Tensor:
|
|
54
|
+
"""Convert CIELAB ``[..., 3]`` to sRGB, clipped to ``[0, 1]``."""
|
|
55
|
+
lab = tf.cast(lab, tf.float32)
|
|
56
|
+
fy = (lab[..., 0] + 16.0) / 116.0
|
|
57
|
+
fx = fy + lab[..., 1] / 500.0
|
|
58
|
+
fz = fy - lab[..., 2] / 200.0
|
|
59
|
+
f = tf.stack([fx, fy, fz], axis=-1)
|
|
60
|
+
ratio = tf.where(
|
|
61
|
+
tf.pow(f, 3.0) > _LAB_EPSILON,
|
|
62
|
+
tf.pow(f, 3.0),
|
|
63
|
+
(116.0 * f - 16.0) / _LAB_KAPPA,
|
|
64
|
+
)
|
|
65
|
+
xyz = ratio * tf.constant(_D65_WHITE_XYZ, tf.float32)
|
|
66
|
+
xyz_to_rgb = tf.linalg.inv(tf.constant(_RGB_TO_XYZ, tf.float64))
|
|
67
|
+
linear = tf.linalg.matvec(tf.cast(xyz_to_rgb, tf.float32), xyz)
|
|
68
|
+
return tf.clip_by_value(linear_to_srgb(linear), 0.0, 1.0)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
__all__ = ["lab_to_rgb", "linear_to_srgb", "rgb_to_lab", "srgb_to_linear"]
|