justdata 1.2.1__tar.gz → 1.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. {justdata-1.2.1 → justdata-1.5.0}/PKG-INFO +69 -5
  2. {justdata-1.2.1 → justdata-1.5.0}/README.md +67 -4
  3. {justdata-1.2.1 → justdata-1.5.0}/pyproject.toml +2 -1
  4. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/__init__.py +1 -1
  5. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/core/execution.py +5 -0
  6. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/core/finalization.py +3 -1
  7. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/core/loader.py +5 -1
  8. justdata-1.5.0/src/justdata/vision/colorspace.py +71 -0
  9. justdata-1.5.0/src/justdata/vision/compositing.py +717 -0
  10. justdata-1.5.0/src/justdata/vision/corruptions/__init__.py +65 -0
  11. justdata-1.5.0/src/justdata/vision/corruptions/_conditions.py +73 -0
  12. justdata-1.5.0/src/justdata/vision/corruptions/atmosphere.py +206 -0
  13. justdata-1.5.0/src/justdata/vision/corruptions/continuous.py +467 -0
  14. justdata-1.5.0/src/justdata/vision/corruptions/exposure.py +235 -0
  15. justdata-1.5.0/src/justdata/vision/corruptions/lens.py +183 -0
  16. justdata-1.5.0/src/justdata/vision/corruptions/motion.py +86 -0
  17. justdata-1.5.0/src/justdata/vision/corruptions/precipitation.py +153 -0
  18. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/corruptions/registry.py +328 -9
  19. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/corruptions/standard.py +8 -38
  20. justdata-1.5.0/src/justdata/vision/corruptions/water.py +175 -0
  21. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/encodings/panoptic_targets.py +121 -34
  22. justdata-1.5.0/src/justdata/vision/filters.py +154 -0
  23. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/geometry.py +267 -35
  24. justdata-1.5.0/src/justdata/vision/horizon.py +81 -0
  25. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/pipelines.py +15 -4
  26. justdata-1.5.0/src/justdata/vision/poisson.py +177 -0
  27. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/stages.py +2 -2
  28. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/tasks/panoptic.py +7 -1
  29. justdata-1.2.1/src/justdata/vision/corruptions/__init__.py +0 -27
  30. {justdata-1.2.1 → justdata-1.5.0}/LICENSE +0 -0
  31. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/__init__.py +0 -0
  32. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/_dcase.py +0 -0
  33. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/_manifests.py +0 -0
  34. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/_random.py +0 -0
  35. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/_signal.py +0 -0
  36. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/adapters.py +0 -0
  37. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/augment/__init__.py +0 -0
  38. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/augment/_common.py +0 -0
  39. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/augment/batch.py +0 -0
  40. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/augment/configs.py +0 -0
  41. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/augment/patchout.py +0 -0
  42. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/augment/spectrogram.py +0 -0
  43. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/augment/waveform.py +0 -0
  44. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/batching.py +0 -0
  45. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/channel.py +0 -0
  46. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/compat/__init__.py +0 -0
  47. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/compat/ast.py +0 -0
  48. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/compat/ced.py +0 -0
  49. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/compat/efficientat.py +0 -0
  50. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/compat/panns.py +0 -0
  51. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/compat/passt.py +0 -0
  52. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/configs.py +0 -0
  53. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/corruptions/__init__.py +0 -0
  54. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/corruptions/codec.py +0 -0
  55. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/corruptions/datasets.py +0 -0
  56. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/corruptions/device.py +0 -0
  57. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/corruptions/dynamics.py +0 -0
  58. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/corruptions/filters.py +0 -0
  59. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/corruptions/noise.py +0 -0
  60. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/corruptions/registry.py +0 -0
  61. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/corruptions/reverb.py +0 -0
  62. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/corruptions/timefreq.py +0 -0
  63. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/datasets.py +0 -0
  64. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/dcase2025.py +0 -0
  65. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/decoding.py +0 -0
  66. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/eval_views.py +0 -0
  67. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/evals.py +0 -0
  68. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/filters.py +0 -0
  69. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/frontends/__init__.py +0 -0
  70. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/frontends/ast.py +0 -0
  71. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/frontends/kaldi.py +0 -0
  72. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/frontends/log.py +0 -0
  73. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/frontends/mel.py +0 -0
  74. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/frontends/mfcc.py +0 -0
  75. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/frontends/pcen.py +0 -0
  76. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/frontends/stft.py +0 -0
  77. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/jax.py +0 -0
  78. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/labels.py +0 -0
  79. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/layouts.py +0 -0
  80. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/metadata.py +0 -0
  81. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/normalization.py +0 -0
  82. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/pipelines.py +0 -0
  83. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/postprocessing.py +0 -0
  84. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/preprocessing.py +0 -0
  85. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/presets.py +0 -0
  86. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/registry.py +0 -0
  87. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/resampling.py +0 -0
  88. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/schema.py +0 -0
  89. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/segment.py +0 -0
  90. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/sources.py +0 -0
  91. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/stages.py +0 -0
  92. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/stats.py +0 -0
  93. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/acoustic/tasks.py +0 -0
  94. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/audio/__init__.py +0 -0
  95. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/core/__init__.py +0 -0
  96. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/core/_records.py +0 -0
  97. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/core/adapters.py +0 -0
  98. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/core/augmentations.py +0 -0
  99. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/core/cache.py +0 -0
  100. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/core/config_resolution.py +0 -0
  101. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/core/executed_config.py +0 -0
  102. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/core/filters.py +0 -0
  103. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/core/inventory.py +0 -0
  104. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/core/label_mixing.py +0 -0
  105. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/core/metadata.py +0 -0
  106. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/core/presets.py +0 -0
  107. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/core/registry.py +0 -0
  108. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/core/replay.py +0 -0
  109. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/core/sources.py +0 -0
  110. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/core/stats.py +0 -0
  111. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/corruptions.py +0 -0
  112. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/py.typed +0 -0
  113. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/__init__.py +0 -0
  114. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/augmentations/__init__.py +0 -0
  115. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/augmentations/auto.py +0 -0
  116. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/augmentations/color.py +0 -0
  117. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/augmentations/composed.py +0 -0
  118. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/augmentations/geometric.py +0 -0
  119. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/augmentations/mixing.py +0 -0
  120. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/augmentations/registry.py +0 -0
  121. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/corruptions/blur.py +0 -0
  122. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/corruptions/digital.py +0 -0
  123. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/corruptions/noise.py +0 -0
  124. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/corruptions/weather.py +0 -0
  125. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/datasets.py +0 -0
  126. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/encodings/__init__.py +0 -0
  127. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/encodings/heatmaps.py +0 -0
  128. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/encodings/semantic_targets.py +0 -0
  129. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/minic.py +0 -0
  130. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/presets.py +0 -0
  131. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/sources.py +0 -0
  132. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/tasks/__init__.py +0 -0
  133. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/tasks/classification.py +0 -0
  134. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/tasks/depth.py +0 -0
  135. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/tasks/segmentation.py +0 -0
  136. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/transforms.py +0 -0
  137. {justdata-1.2.1 → justdata-1.5.0}/src/justdata/vision/utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: justdata
3
- Version: 1.2.1
3
+ Version: 1.5.0
4
4
  Summary: A TensorFlow-native data pipeline library with a modality-neutral core and first-class computer vision and acoustic recipes
5
5
  Author: clementpoiret
6
6
  Author-email: clementpoiret <clement@linux.com>
@@ -27,6 +27,7 @@ Requires-Dist: soundfile>=0.12 ; extra == 'dev'
27
27
  Requires-Dist: soxr>=0.5 ; extra == 'dev'
28
28
  Requires-Dist: torch==2.10.0 ; extra == 'dev'
29
29
  Requires-Dist: torchcodec==0.10.0 ; extra == 'dev'
30
+ Requires-Dist: opencv-python-headless==5.0.0.93 ; extra == 'golden'
30
31
  Requires-Dist: torchaudio==2.10.0 ; extra == 'golden'
31
32
  Requires-Dist: torchlibrosa==0.1.0 ; extra == 'golden'
32
33
  Requires-Dist: datasets[vision]>=4.5.0 ; extra == 'vision'
@@ -124,6 +125,8 @@ enabled.
124
125
  | `justdata.core.load_inventory(admitted, ...)` | Apply the shared pipeline to an admitted inventory, including raw epoch finalization. |
125
126
  | `justdata.core.ExecutedConfig` | Immutable, canonical snapshot returned by loaders when `return_config=True`. |
126
127
  | `justdata.vision.minic.create_minic_datasets(...)` | Constructs Mini-C corruption benchmark datasets from a shared preprocessed base dataset. |
128
+ | `justdata.vision.corruptions.apply_corruption_at(...)` | Renders a scene condition at a continuous strength and returns its record. |
129
+ | `justdata.vision.corruptions.apply_corruption_stack(...)` | Applies ordered conditions in float, rounds once, and returns the stack record. |
127
130
  | `justdata.acoustic.corruptions.create_audio_corruption_datasets(...)` | Constructs acoustic corruption benchmark datasets from a shared preprocessed base dataset. |
128
131
  | `justdata.acoustic.dcase2025.make_source_dataset(...)` | Builds DCASE Task 1 source-domain datasets with split-safety checks. |
129
132
 
@@ -546,7 +549,9 @@ For semantic segmentation with replayable geometry, select `vision/segmentation`
546
549
  `geometry_kwargs`. It supports paired aspect-preserving training resize and crops, longer-side evaluation caps,
547
550
  independent RGB/ignore padding, per-pixel validity, and original-coordinate score restoration. See
548
551
  [the dense geometry contract](docs/vision.md#12-replayable-dense-segmentation) for configuration, record fields, replay,
549
- and batching requirements.
552
+ and batching requirements. For ordered image/mask file lists without a registered source,
553
+ `justdata.vision.geometry.dense_views_from_files` builds the same views with explicit per-example seeds in a parallel,
554
+ order-preserving `tf.data` map.
550
555
 
551
556
  For instance-aware labels, select `vision/panoptic_segmentation` with a segment-ID map and category table. It preserves
552
557
  thing instances, merges stuff by category, and excludes crowd pixels from panoptic supervision. The
@@ -961,6 +966,43 @@ severity; `none` removes metadata according to the shared core contract. The dat
961
966
  enumeration-position seed salting. Consumers with their own sample-level PRNG lineage should call `apply_corruption`
962
967
  directly.
963
968
 
969
+ ### Scene conditions at continuous strength
970
+
971
+ Ten photometric conditions declare a continuous-strength contract: `haze`, `low_light`, `sun_glitter`, `lens_droplets`,
972
+ `rain`, `motion_blur`, `overexposure`, `white_balance_warm`, `white_balance_cool`, and `shot_noise`.
973
+ `apply_corruption_at` renders one of them at any strength between 0 (identity) and 5 (the last severity row). It takes
974
+ an optional episode seed for structure that persists across frames, and optional `depth_m`, `horizon`, or `region`
975
+ inputs. It returns the image and a record of the resolved parameters and mode. `apply_corruption_stack` applies ordered
976
+ steps in float and rounds once.
977
+
978
+ ```python
979
+ import tensorflow as tf
980
+
981
+ from justdata.vision.corruptions import apply_corruption_at, apply_corruption_stack
982
+
983
+ frame = apply_corruption_at(
984
+ decoded_rgb_uint8,
985
+ name="lens_droplets",
986
+ version="1.0.0",
987
+ strength=1.7,
988
+ seed=tf.constant([frame_index, 0], tf.int32),
989
+ episode_seed=tf.constant([episode_index, 0], tf.int32),
990
+ )
991
+ dusk = apply_corruption_stack(
992
+ decoded_rgb_uint8,
993
+ steps=[("low_light", "1.0.0", 2.0), ("white_balance_cool", "1.0.0", 1.5)],
994
+ seed=tf.constant([frame_index, 1], tf.int32),
995
+ )
996
+ ```
997
+
998
+ See [continuous strength](docs/vision.md#continuous-strength-episodes-and-scene-inputs) and
999
+ [scene condition corruptions](docs/vision.md#scene-condition-corruptions) for the contract, the tables, and their
1000
+ sources, and `examples/vision/maritime_conditions.py` for a drift over an episode.
1001
+
1002
+ To add rare objects, `composite_instances` pastes annotated crops onto caller-chosen anchor pixels, updates the class
1003
+ and instance maps, and records every request as placed or rejected; see
1004
+ [compositing annotated instances](docs/vision.md#16-compositing-annotated-instances).
1005
+
964
1006
  ______________________________________________________________________
965
1007
 
966
1008
  ## Audio Corruption Benchmark
@@ -1094,14 +1136,36 @@ Replay prefetches after skipping committed batches and defaults to one batch.
1094
1136
  When another framework uses the accelerator, configure TensorFlow before creating tensors or datasets:
1095
1137
 
1096
1138
  ```python
1139
+ import jax
1140
+
1141
+ jax.devices() # initialize JAX's GPU backend before TensorFlow is imported
1142
+
1097
1143
  from justdata.core import configure_tensorflow_cpu
1098
1144
 
1099
1145
  device_report = configure_tensorflow_cpu(intra_op_threads=1, inter_op_threads=1)
1100
1146
  assert device_report["logical_gpus"] == []
1101
1147
  ```
1102
1148
 
1103
- The report includes physical and logical device names and process thread settings. The helper raises if the runtime has
1104
- initialized too early for the requested change. Core, vision, acoustic, and audio imports permit this setup.
1149
+ The report includes physical and logical device names and process thread settings. The helper raises if the TensorFlow
1150
+ runtime has initialized too early for the requested change. Core, vision, acoustic, and audio imports do not initialize
1151
+ it, so the helper can run after them.
1152
+
1153
+ Those imports do import TensorFlow. On Linux, the CUDA-enabled `tensorflow` wheel then exposes stubs for its CUDA
1154
+ libraries (CUDA 12 as of TensorFlow 2.21) to the whole process. A framework built for another CUDA version that
1155
+ initializes its GPU backend afterwards can bind to those stubs: JAX with its CUDA 13 plugin reports an outdated cuBLAS
1156
+ and falls back to the CPU. Hiding TensorFlow's GPUs does not prevent this. Either initialize the other framework's
1157
+ backend before the first TensorFlow import, as above (`import jax` alone is not enough), or install `tensorflow-cpu`,
1158
+ which exports no CUDA symbols, in place of `tensorflow`. With uv, on Linux x86_64:
1159
+
1160
+ ```toml
1161
+ [project]
1162
+ dependencies = ["justdata", "tensorflow-cpu"]
1163
+
1164
+ [tool.uv]
1165
+ exclude-dependencies = ["tensorflow"]
1166
+ ```
1167
+
1168
+ Set `JAX_PLATFORMS=cuda` to make a JAX fallback to the CPU an error rather than a warning.
1105
1169
 
1106
1170
  `CachePolicy` enables identity-checked file caches on the existing preprocessing and model-input cache stages. The cache
1107
1171
  path is a directory whose parent exists. The input identity must cover ordered content. `load_inventory` binds its
@@ -1306,7 +1370,7 @@ package-support jobs pass. Build artifacts are retained for one week. Tag pipeli
1306
1370
  to complete a release.
1307
1371
 
1308
1372
  Before tagging, update `pyproject.toml`, `src/justdata/__init__.py`, and `CHANGELOG.md`, then run `uv lock`. Push a tag
1309
- matching the package version, with an optional `v` prefix (for example, `v1.2.1` or `1.2.1`). Publishing rejects tags
1373
+ matching the package version, with an optional `v` prefix (for example, `v1.5.0` or `1.5.0`). Publishing rejects tags
1310
1374
  that disagree with either version declaration. Branch pipelines build artifacts without publishing.
1311
1375
 
1312
1376
  Configure a [PyPI GitLab trusted publisher](https://docs.pypi.org/trusted-publishers/adding-a-publisher/) for the
@@ -81,6 +81,8 @@ enabled.
81
81
  | `justdata.core.load_inventory(admitted, ...)` | Apply the shared pipeline to an admitted inventory, including raw epoch finalization. |
82
82
  | `justdata.core.ExecutedConfig` | Immutable, canonical snapshot returned by loaders when `return_config=True`. |
83
83
  | `justdata.vision.minic.create_minic_datasets(...)` | Constructs Mini-C corruption benchmark datasets from a shared preprocessed base dataset. |
84
+ | `justdata.vision.corruptions.apply_corruption_at(...)` | Renders a scene condition at a continuous strength and returns its record. |
85
+ | `justdata.vision.corruptions.apply_corruption_stack(...)` | Applies ordered conditions in float, rounds once, and returns the stack record. |
84
86
  | `justdata.acoustic.corruptions.create_audio_corruption_datasets(...)` | Constructs acoustic corruption benchmark datasets from a shared preprocessed base dataset. |
85
87
  | `justdata.acoustic.dcase2025.make_source_dataset(...)` | Builds DCASE Task 1 source-domain datasets with split-safety checks. |
86
88
 
@@ -503,7 +505,9 @@ For semantic segmentation with replayable geometry, select `vision/segmentation`
503
505
  `geometry_kwargs`. It supports paired aspect-preserving training resize and crops, longer-side evaluation caps,
504
506
  independent RGB/ignore padding, per-pixel validity, and original-coordinate score restoration. See
505
507
  [the dense geometry contract](docs/vision.md#12-replayable-dense-segmentation) for configuration, record fields, replay,
506
- and batching requirements.
508
+ and batching requirements. For ordered image/mask file lists without a registered source,
509
+ `justdata.vision.geometry.dense_views_from_files` builds the same views with explicit per-example seeds in a parallel,
510
+ order-preserving `tf.data` map.
507
511
 
508
512
  For instance-aware labels, select `vision/panoptic_segmentation` with a segment-ID map and category table. It preserves
509
513
  thing instances, merges stuff by category, and excludes crowd pixels from panoptic supervision. The
@@ -918,6 +922,43 @@ severity; `none` removes metadata according to the shared core contract. The dat
918
922
  enumeration-position seed salting. Consumers with their own sample-level PRNG lineage should call `apply_corruption`
919
923
  directly.
920
924
 
925
+ ### Scene conditions at continuous strength
926
+
927
+ Ten photometric conditions declare a continuous-strength contract: `haze`, `low_light`, `sun_glitter`, `lens_droplets`,
928
+ `rain`, `motion_blur`, `overexposure`, `white_balance_warm`, `white_balance_cool`, and `shot_noise`.
929
+ `apply_corruption_at` renders one of them at any strength between 0 (identity) and 5 (the last severity row). It takes
930
+ an optional episode seed for structure that persists across frames, and optional `depth_m`, `horizon`, or `region`
931
+ inputs. It returns the image and a record of the resolved parameters and mode. `apply_corruption_stack` applies ordered
932
+ steps in float and rounds once.
933
+
934
+ ```python
935
+ import tensorflow as tf
936
+
937
+ from justdata.vision.corruptions import apply_corruption_at, apply_corruption_stack
938
+
939
+ frame = apply_corruption_at(
940
+ decoded_rgb_uint8,
941
+ name="lens_droplets",
942
+ version="1.0.0",
943
+ strength=1.7,
944
+ seed=tf.constant([frame_index, 0], tf.int32),
945
+ episode_seed=tf.constant([episode_index, 0], tf.int32),
946
+ )
947
+ dusk = apply_corruption_stack(
948
+ decoded_rgb_uint8,
949
+ steps=[("low_light", "1.0.0", 2.0), ("white_balance_cool", "1.0.0", 1.5)],
950
+ seed=tf.constant([frame_index, 1], tf.int32),
951
+ )
952
+ ```
953
+
954
+ See [continuous strength](docs/vision.md#continuous-strength-episodes-and-scene-inputs) and
955
+ [scene condition corruptions](docs/vision.md#scene-condition-corruptions) for the contract, the tables, and their
956
+ sources, and `examples/vision/maritime_conditions.py` for a drift over an episode.
957
+
958
+ To add rare objects, `composite_instances` pastes annotated crops onto caller-chosen anchor pixels, updates the class
959
+ and instance maps, and records every request as placed or rejected; see
960
+ [compositing annotated instances](docs/vision.md#16-compositing-annotated-instances).
961
+
921
962
  ______________________________________________________________________
922
963
 
923
964
  ## Audio Corruption Benchmark
@@ -1051,14 +1092,36 @@ Replay prefetches after skipping committed batches and defaults to one batch.
1051
1092
  When another framework uses the accelerator, configure TensorFlow before creating tensors or datasets:
1052
1093
 
1053
1094
  ```python
1095
+ import jax
1096
+
1097
+ jax.devices() # initialize JAX's GPU backend before TensorFlow is imported
1098
+
1054
1099
  from justdata.core import configure_tensorflow_cpu
1055
1100
 
1056
1101
  device_report = configure_tensorflow_cpu(intra_op_threads=1, inter_op_threads=1)
1057
1102
  assert device_report["logical_gpus"] == []
1058
1103
  ```
1059
1104
 
1060
- The report includes physical and logical device names and process thread settings. The helper raises if the runtime has
1061
- initialized too early for the requested change. Core, vision, acoustic, and audio imports permit this setup.
1105
+ The report includes physical and logical device names and process thread settings. The helper raises if the TensorFlow
1106
+ runtime has initialized too early for the requested change. Core, vision, acoustic, and audio imports do not initialize
1107
+ it, so the helper can run after them.
1108
+
1109
+ Those imports do import TensorFlow. On Linux, the CUDA-enabled `tensorflow` wheel then exposes stubs for its CUDA
1110
+ libraries (CUDA 12 as of TensorFlow 2.21) to the whole process. A framework built for another CUDA version that
1111
+ initializes its GPU backend afterwards can bind to those stubs: JAX with its CUDA 13 plugin reports an outdated cuBLAS
1112
+ and falls back to the CPU. Hiding TensorFlow's GPUs does not prevent this. Either initialize the other framework's
1113
+ backend before the first TensorFlow import, as above (`import jax` alone is not enough), or install `tensorflow-cpu`,
1114
+ which exports no CUDA symbols, in place of `tensorflow`. With uv, on Linux x86_64:
1115
+
1116
+ ```toml
1117
+ [project]
1118
+ dependencies = ["justdata", "tensorflow-cpu"]
1119
+
1120
+ [tool.uv]
1121
+ exclude-dependencies = ["tensorflow"]
1122
+ ```
1123
+
1124
+ Set `JAX_PLATFORMS=cuda` to make a JAX fallback to the CPU an error rather than a warning.
1062
1125
 
1063
1126
  `CachePolicy` enables identity-checked file caches on the existing preprocessing and model-input cache stages. The cache
1064
1127
  path is a directory whose parent exists. The input identity must cover ordered content. `load_inventory` binds its
@@ -1263,7 +1326,7 @@ package-support jobs pass. Build artifacts are retained for one week. Tag pipeli
1263
1326
  to complete a release.
1264
1327
 
1265
1328
  Before tagging, update `pyproject.toml`, `src/justdata/__init__.py`, and `CHANGELOG.md`, then run `uv lock`. Push a tag
1266
- matching the package version, with an optional `v` prefix (for example, `v1.2.1` or `1.2.1`). Publishing rejects tags
1329
+ matching the package version, with an optional `v` prefix (for example, `v1.5.0` or `1.5.0`). Publishing rejects tags
1267
1330
  that disagree with either version declaration. Branch pipelines build artifacts without publishing.
1268
1331
 
1269
1332
  Configure a [PyPI GitLab trusted publisher](https://docs.pypi.org/trusted-publishers/adding-a-publisher/) for the
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "justdata"
3
- version = "1.2.1"
3
+ version = "1.5.0"
4
4
  description = "A TensorFlow-native data pipeline library with a modality-neutral core and first-class computer vision and acoustic recipes"
5
5
  readme = "README.md"
6
6
  license = "MIT"
@@ -33,6 +33,7 @@ acoustic = [
33
33
  "torchcodec==0.10.0",
34
34
  ]
35
35
  golden = [
36
+ "opencv-python-headless==5.0.0.93",
36
37
  "torchaudio==2.10.0",
37
38
  "torchlibrosa==0.1.0",
38
39
  ]
@@ -1,3 +1,3 @@
1
- __version__ = "1.2.1"
1
+ __version__ = "1.5.0"
2
2
 
3
3
  __all__ = ["__version__"]
@@ -14,6 +14,11 @@ def configure_tensorflow_cpu(
14
14
 
15
15
  Call before constructing tensors, datasets, or invoking other TensorFlow
16
16
  operations. A runtime that was initialized too early raises RuntimeError.
17
+
18
+ Hiding GPUs does not isolate TensorFlow's CUDA libraries. With the
19
+ CUDA-enabled ``tensorflow`` wheel, another framework must initialize its
20
+ GPU backend before TensorFlow is first imported; ``tensorflow-cpu`` has no
21
+ such ordering requirement.
17
22
  """
18
23
  for name, value in (
19
24
  ("intra_op_threads", intra_op_threads),
@@ -305,13 +305,15 @@ def finalize_dataset(
305
305
  deterministic=deterministic,
306
306
  )
307
307
  elif rng is not None:
308
+ # Parallel calls draw from the shared generator in scheduling order,
309
+ # so batches would trade seeds between otherwise identical runs.
308
310
  ds = ds.map(
309
311
  lambda batch: late_augment_fn(
310
312
  batch,
311
313
  num_classes=num_classes,
312
314
  seed=rng.make_seeds(1)[:, 0],
313
315
  ),
314
- num_parallel_calls=parallel_calls,
316
+ num_parallel_calls=1 if deterministic else parallel_calls,
315
317
  deterministic=deterministic,
316
318
  )
317
319
 
@@ -478,6 +478,10 @@ def _prepare_ds(
478
478
  ):
479
479
  """Shared pipeline, resolving its source only after validating options."""
480
480
  rng = tf.random.Generator.from_seed(seed)
481
+ # Late augmentation draws in a later tf.data stage that runs concurrently with
482
+ # the per-sample augment map; sharing one generator would interleave the two
483
+ # stages' draws differently on every run.
484
+ late_augment_rng = tf.random.Generator.from_seed(seed).split(1)[0]
481
485
  if metadata_mode is None:
482
486
  metadata_mode = (
483
487
  pipeline.kwargs.get("metadata_mode", "full")
@@ -685,7 +689,7 @@ def _prepare_ds(
685
689
  "is_training": is_training,
686
690
  "apply_late_augment": apply_augmentation,
687
691
  "late_augment_fn": late_augment_fn,
688
- "rng": rng,
692
+ "rng": late_augment_rng,
689
693
  "deterministic": deterministic,
690
694
  "shuffle_buffer": shuffle_buffer if is_training else None,
691
695
  "shuffle_seed": seed,
@@ -0,0 +1,71 @@
1
+ """Float32 RGB colour-space conversions on values in ``[0, 1]``.
2
+
3
+ sRGB follows IEC 61966-2-1 with the D65 white point; CIELAB uses the same
4
+ white, so ``lab_to_rgb(rgb_to_lab(x))`` returns ``x`` up to float32 rounding.
5
+ """
6
+
7
+ import tensorflow as tf
8
+
9
+ _RGB_TO_XYZ = (
10
+ (0.4124564, 0.3575761, 0.1804375),
11
+ (0.2126729, 0.7151522, 0.0721750),
12
+ (0.0193339, 0.1191920, 0.9503041),
13
+ )
14
+ _D65_WHITE_XYZ = (0.95047, 1.0, 1.08883)
15
+ _LAB_EPSILON = 216.0 / 24389.0
16
+ _LAB_KAPPA = 24389.0 / 27.0
17
+
18
+
19
+ def srgb_to_linear(image: tf.Tensor) -> tf.Tensor:
20
+ image = tf.cast(image, tf.float32)
21
+ return tf.where(
22
+ image <= 0.04045,
23
+ image / 12.92,
24
+ tf.pow((tf.maximum(image, 0.04045) + 0.055) / 1.055, 2.4),
25
+ )
26
+
27
+
28
+ def linear_to_srgb(image: tf.Tensor) -> tf.Tensor:
29
+ image = tf.maximum(tf.cast(image, tf.float32), 0.0)
30
+ return tf.where(
31
+ image <= 0.0031308,
32
+ image * 12.92,
33
+ 1.055 * tf.pow(tf.maximum(image, 0.0031308), 1.0 / 2.4) - 0.055,
34
+ )
35
+
36
+
37
+ def rgb_to_lab(image: tf.Tensor) -> tf.Tensor:
38
+ """Convert sRGB ``[..., 3]`` to CIELAB with L in ``[0, 100]``."""
39
+ linear = srgb_to_linear(image)
40
+ xyz = tf.linalg.matvec(tf.constant(_RGB_TO_XYZ, tf.float32), linear)
41
+ ratio = xyz / tf.constant(_D65_WHITE_XYZ, tf.float32)
42
+ f = tf.where(
43
+ ratio > _LAB_EPSILON,
44
+ tf.pow(tf.maximum(ratio, _LAB_EPSILON), 1.0 / 3.0),
45
+ (_LAB_KAPPA * ratio + 16.0) / 116.0,
46
+ )
47
+ lightness = 116.0 * f[..., 1] - 16.0
48
+ a = 500.0 * (f[..., 0] - f[..., 1])
49
+ b = 200.0 * (f[..., 1] - f[..., 2])
50
+ return tf.stack([lightness, a, b], axis=-1)
51
+
52
+
53
+ def lab_to_rgb(lab: tf.Tensor) -> tf.Tensor:
54
+ """Convert CIELAB ``[..., 3]`` to sRGB, clipped to ``[0, 1]``."""
55
+ lab = tf.cast(lab, tf.float32)
56
+ fy = (lab[..., 0] + 16.0) / 116.0
57
+ fx = fy + lab[..., 1] / 500.0
58
+ fz = fy - lab[..., 2] / 200.0
59
+ f = tf.stack([fx, fy, fz], axis=-1)
60
+ ratio = tf.where(
61
+ tf.pow(f, 3.0) > _LAB_EPSILON,
62
+ tf.pow(f, 3.0),
63
+ (116.0 * f - 16.0) / _LAB_KAPPA,
64
+ )
65
+ xyz = ratio * tf.constant(_D65_WHITE_XYZ, tf.float32)
66
+ xyz_to_rgb = tf.linalg.inv(tf.constant(_RGB_TO_XYZ, tf.float64))
67
+ linear = tf.linalg.matvec(tf.cast(xyz_to_rgb, tf.float32), xyz)
68
+ return tf.clip_by_value(linear_to_srgb(linear), 0.0, 1.0)
69
+
70
+
71
+ __all__ = ["lab_to_rgb", "linear_to_srgb", "rgb_to_lab", "srgb_to_linear"]