justdata 1.3.0__tar.gz → 1.6.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. justdata-1.3.0/README.md → justdata-1.6.0/PKG-INFO +84 -1
  2. justdata-1.3.0/PKG-INFO → justdata-1.6.0/README.md +40 -44
  3. {justdata-1.3.0 → justdata-1.6.0}/pyproject.toml +2 -1
  4. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/__init__.py +1 -1
  5. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/core/finalization.py +3 -1
  6. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/core/loader.py +5 -1
  7. justdata-1.6.0/src/justdata/vision/colorspace.py +133 -0
  8. justdata-1.6.0/src/justdata/vision/compositing.py +807 -0
  9. justdata-1.6.0/src/justdata/vision/corruptions/__init__.py +65 -0
  10. justdata-1.6.0/src/justdata/vision/corruptions/_conditions.py +73 -0
  11. justdata-1.6.0/src/justdata/vision/corruptions/atmosphere.py +206 -0
  12. justdata-1.6.0/src/justdata/vision/corruptions/continuous.py +467 -0
  13. justdata-1.6.0/src/justdata/vision/corruptions/exposure.py +235 -0
  14. justdata-1.6.0/src/justdata/vision/corruptions/lens.py +183 -0
  15. justdata-1.6.0/src/justdata/vision/corruptions/motion.py +86 -0
  16. justdata-1.6.0/src/justdata/vision/corruptions/precipitation.py +153 -0
  17. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/corruptions/registry.py +328 -9
  18. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/corruptions/standard.py +8 -38
  19. justdata-1.6.0/src/justdata/vision/corruptions/water.py +175 -0
  20. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/encodings/panoptic_targets.py +121 -34
  21. justdata-1.6.0/src/justdata/vision/filters.py +154 -0
  22. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/geometry.py +134 -18
  23. justdata-1.6.0/src/justdata/vision/horizon.py +81 -0
  24. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/pipelines.py +13 -2
  25. justdata-1.6.0/src/justdata/vision/poisson.py +177 -0
  26. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/tasks/panoptic.py +7 -1
  27. justdata-1.3.0/src/justdata/vision/corruptions/__init__.py +0 -27
  28. {justdata-1.3.0 → justdata-1.6.0}/LICENSE +0 -0
  29. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/__init__.py +0 -0
  30. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/_dcase.py +0 -0
  31. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/_manifests.py +0 -0
  32. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/_random.py +0 -0
  33. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/_signal.py +0 -0
  34. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/adapters.py +0 -0
  35. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/augment/__init__.py +0 -0
  36. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/augment/_common.py +0 -0
  37. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/augment/batch.py +0 -0
  38. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/augment/configs.py +0 -0
  39. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/augment/patchout.py +0 -0
  40. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/augment/spectrogram.py +0 -0
  41. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/augment/waveform.py +0 -0
  42. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/batching.py +0 -0
  43. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/channel.py +0 -0
  44. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/compat/__init__.py +0 -0
  45. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/compat/ast.py +0 -0
  46. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/compat/ced.py +0 -0
  47. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/compat/efficientat.py +0 -0
  48. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/compat/panns.py +0 -0
  49. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/compat/passt.py +0 -0
  50. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/configs.py +0 -0
  51. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/corruptions/__init__.py +0 -0
  52. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/corruptions/codec.py +0 -0
  53. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/corruptions/datasets.py +0 -0
  54. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/corruptions/device.py +0 -0
  55. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/corruptions/dynamics.py +0 -0
  56. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/corruptions/filters.py +0 -0
  57. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/corruptions/noise.py +0 -0
  58. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/corruptions/registry.py +0 -0
  59. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/corruptions/reverb.py +0 -0
  60. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/corruptions/timefreq.py +0 -0
  61. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/datasets.py +0 -0
  62. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/dcase2025.py +0 -0
  63. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/decoding.py +0 -0
  64. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/eval_views.py +0 -0
  65. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/evals.py +0 -0
  66. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/filters.py +0 -0
  67. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/frontends/__init__.py +0 -0
  68. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/frontends/ast.py +0 -0
  69. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/frontends/kaldi.py +0 -0
  70. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/frontends/log.py +0 -0
  71. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/frontends/mel.py +0 -0
  72. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/frontends/mfcc.py +0 -0
  73. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/frontends/pcen.py +0 -0
  74. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/frontends/stft.py +0 -0
  75. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/jax.py +0 -0
  76. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/labels.py +0 -0
  77. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/layouts.py +0 -0
  78. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/metadata.py +0 -0
  79. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/normalization.py +0 -0
  80. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/pipelines.py +0 -0
  81. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/postprocessing.py +0 -0
  82. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/preprocessing.py +0 -0
  83. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/presets.py +0 -0
  84. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/registry.py +0 -0
  85. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/resampling.py +0 -0
  86. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/schema.py +0 -0
  87. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/segment.py +0 -0
  88. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/sources.py +0 -0
  89. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/stages.py +0 -0
  90. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/stats.py +0 -0
  91. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/tasks.py +0 -0
  92. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/audio/__init__.py +0 -0
  93. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/core/__init__.py +0 -0
  94. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/core/_records.py +0 -0
  95. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/core/adapters.py +0 -0
  96. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/core/augmentations.py +0 -0
  97. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/core/cache.py +0 -0
  98. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/core/config_resolution.py +0 -0
  99. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/core/executed_config.py +0 -0
  100. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/core/execution.py +0 -0
  101. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/core/filters.py +0 -0
  102. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/core/inventory.py +0 -0
  103. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/core/label_mixing.py +0 -0
  104. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/core/metadata.py +0 -0
  105. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/core/presets.py +0 -0
  106. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/core/registry.py +0 -0
  107. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/core/replay.py +0 -0
  108. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/core/sources.py +0 -0
  109. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/core/stats.py +0 -0
  110. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/corruptions.py +0 -0
  111. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/py.typed +0 -0
  112. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/__init__.py +0 -0
  113. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/augmentations/__init__.py +0 -0
  114. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/augmentations/auto.py +0 -0
  115. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/augmentations/color.py +0 -0
  116. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/augmentations/composed.py +0 -0
  117. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/augmentations/geometric.py +0 -0
  118. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/augmentations/mixing.py +0 -0
  119. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/augmentations/registry.py +0 -0
  120. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/corruptions/blur.py +0 -0
  121. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/corruptions/digital.py +0 -0
  122. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/corruptions/noise.py +0 -0
  123. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/corruptions/weather.py +0 -0
  124. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/datasets.py +0 -0
  125. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/encodings/__init__.py +0 -0
  126. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/encodings/heatmaps.py +0 -0
  127. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/encodings/semantic_targets.py +0 -0
  128. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/minic.py +0 -0
  129. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/presets.py +0 -0
  130. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/sources.py +0 -0
  131. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/stages.py +0 -0
  132. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/tasks/__init__.py +0 -0
  133. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/tasks/classification.py +0 -0
  134. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/tasks/depth.py +0 -0
  135. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/tasks/segmentation.py +0 -0
  136. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/transforms.py +0 -0
  137. {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/utils.py +0 -0
@@ -1,3 +1,47 @@
1
+ Metadata-Version: 2.4
2
+ Name: justdata
3
+ Version: 1.6.0
4
+ Summary: A TensorFlow-native data pipeline library with a modality-neutral core and first-class computer vision and acoustic recipes
5
+ Author: clementpoiret
6
+ Author-email: clementpoiret <clement@linux.com>
7
+ License-Expression: MIT
8
+ License-File: LICENSE
9
+ Requires-Dist: loguru>=0.7.3
10
+ Requires-Dist: tensorflow>=2.18.1
11
+ Requires-Dist: tensorflow-datasets>=4.9.9
12
+ Requires-Dist: datasets[audio]>=4.5.0 ; extra == 'acoustic'
13
+ Requires-Dist: soundfile>=0.12 ; extra == 'acoustic'
14
+ Requires-Dist: soxr>=0.5 ; extra == 'acoustic'
15
+ Requires-Dist: librosa>=0.11 ; extra == 'acoustic'
16
+ Requires-Dist: torch==2.10.0 ; extra == 'acoustic'
17
+ Requires-Dist: torchcodec==0.10.0 ; extra == 'acoustic'
18
+ Requires-Dist: datasets[audio]>=4.5.0 ; extra == 'dev'
19
+ Requires-Dist: datasets[vision]>=4.5.0 ; extra == 'dev'
20
+ Requires-Dist: filelock>=3.20 ; extra == 'dev'
21
+ Requires-Dist: librosa>=0.11 ; extra == 'dev'
22
+ Requires-Dist: matplotlib>=3.10.8 ; extra == 'dev'
23
+ Requires-Dist: pillow>=12.2.0 ; extra == 'dev'
24
+ Requires-Dist: pytest>=9.0.2 ; extra == 'dev'
25
+ Requires-Dist: ruff==0.15.20 ; extra == 'dev'
26
+ Requires-Dist: soundfile>=0.12 ; extra == 'dev'
27
+ Requires-Dist: soxr>=0.5 ; extra == 'dev'
28
+ Requires-Dist: torch==2.10.0 ; extra == 'dev'
29
+ Requires-Dist: torchcodec==0.10.0 ; extra == 'dev'
30
+ Requires-Dist: opencv-python-headless==5.0.0.93 ; extra == 'golden'
31
+ Requires-Dist: torchaudio==2.10.0 ; extra == 'golden'
32
+ Requires-Dist: torchlibrosa==0.1.0 ; extra == 'golden'
33
+ Requires-Dist: datasets[vision]>=4.5.0 ; extra == 'vision'
34
+ Requires-Dist: filelock>=3.20 ; extra == 'vision'
35
+ Requires-Dist: pillow>=12.2.0 ; extra == 'vision'
36
+ Requires-Dist: wilds>=2.0.0 ; extra == 'wilds'
37
+ Requires-Python: >=3.11, <3.14
38
+ Provides-Extra: acoustic
39
+ Provides-Extra: dev
40
+ Provides-Extra: golden
41
+ Provides-Extra: vision
42
+ Provides-Extra: wilds
43
+ Description-Content-Type: text/markdown
44
+
1
45
  # justdata
2
46
 
3
47
  A TensorFlow-native data pipeline library with a modality-neutral core and first-class computer vision and acoustic
@@ -81,6 +125,8 @@ enabled.
81
125
  | `justdata.core.load_inventory(admitted, ...)` | Apply the shared pipeline to an admitted inventory, including raw epoch finalization. |
82
126
  | `justdata.core.ExecutedConfig` | Immutable, canonical snapshot returned by loaders when `return_config=True`. |
83
127
  | `justdata.vision.minic.create_minic_datasets(...)` | Constructs Mini-C corruption benchmark datasets from a shared preprocessed base dataset. |
128
+ | `justdata.vision.corruptions.apply_corruption_at(...)` | Renders a scene condition at a continuous strength and returns its record. |
129
+ | `justdata.vision.corruptions.apply_corruption_stack(...)` | Applies ordered conditions in float, rounds once, and returns the stack record. |
84
130
  | `justdata.acoustic.corruptions.create_audio_corruption_datasets(...)` | Constructs acoustic corruption benchmark datasets from a shared preprocessed base dataset. |
85
131
  | `justdata.acoustic.dcase2025.make_source_dataset(...)` | Builds DCASE Task 1 source-domain datasets with split-safety checks. |
86
132
 
@@ -920,6 +966,43 @@ severity; `none` removes metadata according to the shared core contract. The dat
920
966
  enumeration-position seed salting. Consumers with their own sample-level PRNG lineage should call `apply_corruption`
921
967
  directly.
922
968
 
969
+ ### Scene conditions at continuous strength
970
+
971
+ Ten photometric conditions declare a continuous-strength contract: `haze`, `low_light`, `sun_glitter`, `lens_droplets`,
972
+ `rain`, `motion_blur`, `overexposure`, `white_balance_warm`, `white_balance_cool`, and `shot_noise`.
973
+ `apply_corruption_at` renders one of them at any strength between 0 (identity) and 5 (the last severity row). It takes
974
+ an optional episode seed for structure that persists across frames, and optional `depth_m`, `horizon`, or `region`
975
+ inputs. It returns the image and a record of the resolved parameters and mode. `apply_corruption_stack` applies ordered
976
+ steps in float and rounds once.
977
+
978
+ ```python
979
+ import tensorflow as tf
980
+
981
+ from justdata.vision.corruptions import apply_corruption_at, apply_corruption_stack
982
+
983
+ frame = apply_corruption_at(
984
+ decoded_rgb_uint8,
985
+ name="lens_droplets",
986
+ version="1.0.0",
987
+ strength=1.7,
988
+ seed=tf.constant([frame_index, 0], tf.int32),
989
+ episode_seed=tf.constant([episode_index, 0], tf.int32),
990
+ )
991
+ dusk = apply_corruption_stack(
992
+ decoded_rgb_uint8,
993
+ steps=[("low_light", "1.0.0", 2.0), ("white_balance_cool", "1.0.0", 1.5)],
994
+ seed=tf.constant([frame_index, 1], tf.int32),
995
+ )
996
+ ```
997
+
998
+ See [continuous strength](docs/vision.md#continuous-strength-episodes-and-scene-inputs) and
999
+ [scene condition corruptions](docs/vision.md#scene-condition-corruptions) for the contract, the tables, and their
1000
+ sources, and `examples/vision/maritime_conditions.py` for a drift over an episode.
1001
+
1002
+ To add rare objects, `composite_instances` pastes annotated crops onto caller-chosen anchor pixels, updates the class
1003
+ and instance maps, and records every request as placed or rejected; see
1004
+ [compositing annotated instances](docs/vision.md#16-compositing-annotated-instances).
1005
+
923
1006
  ______________________________________________________________________
924
1007
 
925
1008
  ## Audio Corruption Benchmark
@@ -1287,7 +1370,7 @@ package-support jobs pass. Build artifacts are retained for one week. Tag pipeli
1287
1370
  to complete a release.
1288
1371
 
1289
1372
  Before tagging, update `pyproject.toml`, `src/justdata/__init__.py`, and `CHANGELOG.md`, then run `uv lock`. Push a tag
1290
- matching the package version, with an optional `v` prefix (for example, `v1.3.0` or `1.3.0`). Publishing rejects tags
1373
+ matching the package version, with an optional `v` prefix (for example, `v1.6.0` or `1.6.0`). Publishing rejects tags
1291
1374
  that disagree with either version declaration. Branch pipelines build artifacts without publishing.
1292
1375
 
1293
1376
  Configure a [PyPI GitLab trusted publisher](https://docs.pypi.org/trusted-publishers/adding-a-publisher/) for the
@@ -1,46 +1,3 @@
1
- Metadata-Version: 2.4
2
- Name: justdata
3
- Version: 1.3.0
4
- Summary: A TensorFlow-native data pipeline library with a modality-neutral core and first-class computer vision and acoustic recipes
5
- Author: clementpoiret
6
- Author-email: clementpoiret <clement@linux.com>
7
- License-Expression: MIT
8
- License-File: LICENSE
9
- Requires-Dist: loguru>=0.7.3
10
- Requires-Dist: tensorflow>=2.18.1
11
- Requires-Dist: tensorflow-datasets>=4.9.9
12
- Requires-Dist: datasets[audio]>=4.5.0 ; extra == 'acoustic'
13
- Requires-Dist: soundfile>=0.12 ; extra == 'acoustic'
14
- Requires-Dist: soxr>=0.5 ; extra == 'acoustic'
15
- Requires-Dist: librosa>=0.11 ; extra == 'acoustic'
16
- Requires-Dist: torch==2.10.0 ; extra == 'acoustic'
17
- Requires-Dist: torchcodec==0.10.0 ; extra == 'acoustic'
18
- Requires-Dist: datasets[audio]>=4.5.0 ; extra == 'dev'
19
- Requires-Dist: datasets[vision]>=4.5.0 ; extra == 'dev'
20
- Requires-Dist: filelock>=3.20 ; extra == 'dev'
21
- Requires-Dist: librosa>=0.11 ; extra == 'dev'
22
- Requires-Dist: matplotlib>=3.10.8 ; extra == 'dev'
23
- Requires-Dist: pillow>=12.2.0 ; extra == 'dev'
24
- Requires-Dist: pytest>=9.0.2 ; extra == 'dev'
25
- Requires-Dist: ruff==0.15.20 ; extra == 'dev'
26
- Requires-Dist: soundfile>=0.12 ; extra == 'dev'
27
- Requires-Dist: soxr>=0.5 ; extra == 'dev'
28
- Requires-Dist: torch==2.10.0 ; extra == 'dev'
29
- Requires-Dist: torchcodec==0.10.0 ; extra == 'dev'
30
- Requires-Dist: torchaudio==2.10.0 ; extra == 'golden'
31
- Requires-Dist: torchlibrosa==0.1.0 ; extra == 'golden'
32
- Requires-Dist: datasets[vision]>=4.5.0 ; extra == 'vision'
33
- Requires-Dist: filelock>=3.20 ; extra == 'vision'
34
- Requires-Dist: pillow>=12.2.0 ; extra == 'vision'
35
- Requires-Dist: wilds>=2.0.0 ; extra == 'wilds'
36
- Requires-Python: >=3.11, <3.14
37
- Provides-Extra: acoustic
38
- Provides-Extra: dev
39
- Provides-Extra: golden
40
- Provides-Extra: vision
41
- Provides-Extra: wilds
42
- Description-Content-Type: text/markdown
43
-
44
1
  # justdata
45
2
 
46
3
  A TensorFlow-native data pipeline library with a modality-neutral core and first-class computer vision and acoustic
@@ -124,6 +81,8 @@ enabled.
124
81
  | `justdata.core.load_inventory(admitted, ...)` | Apply the shared pipeline to an admitted inventory, including raw epoch finalization. |
125
82
  | `justdata.core.ExecutedConfig` | Immutable, canonical snapshot returned by loaders when `return_config=True`. |
126
83
  | `justdata.vision.minic.create_minic_datasets(...)` | Constructs Mini-C corruption benchmark datasets from a shared preprocessed base dataset. |
84
+ | `justdata.vision.corruptions.apply_corruption_at(...)` | Renders a scene condition at a continuous strength and returns its record. |
85
+ | `justdata.vision.corruptions.apply_corruption_stack(...)` | Applies ordered conditions in float, rounds once, and returns the stack record. |
127
86
  | `justdata.acoustic.corruptions.create_audio_corruption_datasets(...)` | Constructs acoustic corruption benchmark datasets from a shared preprocessed base dataset. |
128
87
  | `justdata.acoustic.dcase2025.make_source_dataset(...)` | Builds DCASE Task 1 source-domain datasets with split-safety checks. |
129
88
 
@@ -963,6 +922,43 @@ severity; `none` removes metadata according to the shared core contract. The dat
963
922
  enumeration-position seed salting. Consumers with their own sample-level PRNG lineage should call `apply_corruption`
964
923
  directly.
965
924
 
925
+ ### Scene conditions at continuous strength
926
+
927
+ Ten photometric conditions declare a continuous-strength contract: `haze`, `low_light`, `sun_glitter`, `lens_droplets`,
928
+ `rain`, `motion_blur`, `overexposure`, `white_balance_warm`, `white_balance_cool`, and `shot_noise`.
929
+ `apply_corruption_at` renders one of them at any strength between 0 (identity) and 5 (the last severity row). It takes
930
+ an optional episode seed for structure that persists across frames, and optional `depth_m`, `horizon`, or `region`
931
+ inputs. It returns the image and a record of the resolved parameters and mode. `apply_corruption_stack` applies ordered
932
+ steps in float and rounds once.
933
+
934
+ ```python
935
+ import tensorflow as tf
936
+
937
+ from justdata.vision.corruptions import apply_corruption_at, apply_corruption_stack
938
+
939
+ frame = apply_corruption_at(
940
+ decoded_rgb_uint8,
941
+ name="lens_droplets",
942
+ version="1.0.0",
943
+ strength=1.7,
944
+ seed=tf.constant([frame_index, 0], tf.int32),
945
+ episode_seed=tf.constant([episode_index, 0], tf.int32),
946
+ )
947
+ dusk = apply_corruption_stack(
948
+ decoded_rgb_uint8,
949
+ steps=[("low_light", "1.0.0", 2.0), ("white_balance_cool", "1.0.0", 1.5)],
950
+ seed=tf.constant([frame_index, 1], tf.int32),
951
+ )
952
+ ```
953
+
954
+ See [continuous strength](docs/vision.md#continuous-strength-episodes-and-scene-inputs) and
955
+ [scene condition corruptions](docs/vision.md#scene-condition-corruptions) for the contract, the tables, and their
956
+ sources, and `examples/vision/maritime_conditions.py` for a drift over an episode.
957
+
958
+ To add rare objects, `composite_instances` pastes annotated crops onto caller-chosen anchor pixels, updates the class
959
+ and instance maps, and records every request as placed or rejected; see
960
+ [compositing annotated instances](docs/vision.md#16-compositing-annotated-instances).
961
+
966
962
  ______________________________________________________________________
967
963
 
968
964
  ## Audio Corruption Benchmark
@@ -1330,7 +1326,7 @@ package-support jobs pass. Build artifacts are retained for one week. Tag pipeli
1330
1326
  to complete a release.
1331
1327
 
1332
1328
  Before tagging, update `pyproject.toml`, `src/justdata/__init__.py`, and `CHANGELOG.md`, then run `uv lock`. Push a tag
1333
- matching the package version, with an optional `v` prefix (for example, `v1.3.0` or `1.3.0`). Publishing rejects tags
1329
+ matching the package version, with an optional `v` prefix (for example, `v1.6.0` or `1.6.0`). Publishing rejects tags
1334
1330
  that disagree with either version declaration. Branch pipelines build artifacts without publishing.
1335
1331
 
1336
1332
  Configure a [PyPI GitLab trusted publisher](https://docs.pypi.org/trusted-publishers/adding-a-publisher/) for the
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "justdata"
3
- version = "1.3.0"
3
+ version = "1.6.0"
4
4
  description = "A TensorFlow-native data pipeline library with a modality-neutral core and first-class computer vision and acoustic recipes"
5
5
  readme = "README.md"
6
6
  license = "MIT"
@@ -33,6 +33,7 @@ acoustic = [
33
33
  "torchcodec==0.10.0",
34
34
  ]
35
35
  golden = [
36
+ "opencv-python-headless==5.0.0.93",
36
37
  "torchaudio==2.10.0",
37
38
  "torchlibrosa==0.1.0",
38
39
  ]
@@ -1,3 +1,3 @@
1
- __version__ = "1.3.0"
1
+ __version__ = "1.6.0"
2
2
 
3
3
  __all__ = ["__version__"]
@@ -305,13 +305,15 @@ def finalize_dataset(
305
305
  deterministic=deterministic,
306
306
  )
307
307
  elif rng is not None:
308
+ # Parallel calls draw from the shared generator in scheduling order,
309
+ # so batches would trade seeds between otherwise identical runs.
308
310
  ds = ds.map(
309
311
  lambda batch: late_augment_fn(
310
312
  batch,
311
313
  num_classes=num_classes,
312
314
  seed=rng.make_seeds(1)[:, 0],
313
315
  ),
314
- num_parallel_calls=parallel_calls,
316
+ num_parallel_calls=1 if deterministic else parallel_calls,
315
317
  deterministic=deterministic,
316
318
  )
317
319
 
@@ -478,6 +478,10 @@ def _prepare_ds(
478
478
  ):
479
479
  """Shared pipeline, resolving its source only after validating options."""
480
480
  rng = tf.random.Generator.from_seed(seed)
481
+ # Late augmentation draws in a later tf.data stage that runs concurrently with
482
+ # the per-sample augment map; sharing one generator would interleave the two
483
+ # stages' draws differently on every run.
484
+ late_augment_rng = tf.random.Generator.from_seed(seed).split(1)[0]
481
485
  if metadata_mode is None:
482
486
  metadata_mode = (
483
487
  pipeline.kwargs.get("metadata_mode", "full")
@@ -685,7 +689,7 @@ def _prepare_ds(
685
689
  "is_training": is_training,
686
690
  "apply_late_augment": apply_augmentation,
687
691
  "late_augment_fn": late_augment_fn,
688
- "rng": rng,
692
+ "rng": late_augment_rng,
689
693
  "deterministic": deterministic,
690
694
  "shuffle_buffer": shuffle_buffer if is_training else None,
691
695
  "shuffle_seed": seed,
@@ -0,0 +1,133 @@
1
+ """Float32 RGB colour-space conversions on values in ``[0, 1]``.
2
+
3
+ sRGB follows IEC 61966-2-1 with the D65 white point; CIELAB uses the same
4
+ white, so ``lab_to_rgb(rgb_to_lab(x))`` returns ``x`` up to float32 rounding.
5
+ ``estimate_illuminant`` gives a frame's illuminant colour for chromatic
6
+ adaptation between frames.
7
+ """
8
+
9
+ import tensorflow as tf
10
+
11
+ from justdata.vision.filters import gaussian_blur_hwc
12
+
13
+ _RGB_TO_XYZ = (
14
+ (0.4124564, 0.3575761, 0.1804375),
15
+ (0.2126729, 0.7151522, 0.0721750),
16
+ (0.0193339, 0.1191920, 0.9503041),
17
+ )
18
+ _D65_WHITE_XYZ = (0.95047, 1.0, 1.08883)
19
+ _LAB_EPSILON = 216.0 / 24389.0
20
+ _LAB_KAPPA = 24389.0 / 27.0
21
+ MINKOWSKI_ORDER = 6
22
+ # Grey-Edge smoothing in pixels per 1024 pixels of image width.
23
+ GREY_EDGE_SIGMA_PER_1024_PX = 1.0
24
+ _CLIPPED_LEVEL = 254.0 / 255.0
25
+
26
+
27
+ def srgb_to_linear(image: tf.Tensor) -> tf.Tensor:
28
+ image = tf.cast(image, tf.float32)
29
+ return tf.where(
30
+ image <= 0.04045,
31
+ image / 12.92,
32
+ tf.pow((tf.maximum(image, 0.04045) + 0.055) / 1.055, 2.4),
33
+ )
34
+
35
+
36
+ def linear_to_srgb(image: tf.Tensor) -> tf.Tensor:
37
+ image = tf.maximum(tf.cast(image, tf.float32), 0.0)
38
+ return tf.where(
39
+ image <= 0.0031308,
40
+ image * 12.92,
41
+ 1.055 * tf.pow(tf.maximum(image, 0.0031308), 1.0 / 2.4) - 0.055,
42
+ )
43
+
44
+
45
+ def rgb_to_lab(image: tf.Tensor) -> tf.Tensor:
46
+ """Convert sRGB ``[..., 3]`` to CIELAB with L in ``[0, 100]``."""
47
+ linear = srgb_to_linear(image)
48
+ xyz = tf.linalg.matvec(tf.constant(_RGB_TO_XYZ, tf.float32), linear)
49
+ ratio = xyz / tf.constant(_D65_WHITE_XYZ, tf.float32)
50
+ f = tf.where(
51
+ ratio > _LAB_EPSILON,
52
+ tf.pow(tf.maximum(ratio, _LAB_EPSILON), 1.0 / 3.0),
53
+ (_LAB_KAPPA * ratio + 16.0) / 116.0,
54
+ )
55
+ lightness = 116.0 * f[..., 1] - 16.0
56
+ a = 500.0 * (f[..., 0] - f[..., 1])
57
+ b = 200.0 * (f[..., 1] - f[..., 2])
58
+ return tf.stack([lightness, a, b], axis=-1)
59
+
60
+
61
+ def lab_to_rgb(lab: tf.Tensor) -> tf.Tensor:
62
+ """Convert CIELAB ``[..., 3]`` to sRGB, clipped to ``[0, 1]``."""
63
+ lab = tf.cast(lab, tf.float32)
64
+ fy = (lab[..., 0] + 16.0) / 116.0
65
+ fx = fy + lab[..., 1] / 500.0
66
+ fz = fy - lab[..., 2] / 200.0
67
+ f = tf.stack([fx, fy, fz], axis=-1)
68
+ ratio = tf.where(
69
+ tf.pow(f, 3.0) > _LAB_EPSILON,
70
+ tf.pow(f, 3.0),
71
+ (116.0 * f - 16.0) / _LAB_KAPPA,
72
+ )
73
+ xyz = ratio * tf.constant(_D65_WHITE_XYZ, tf.float32)
74
+ xyz_to_rgb = tf.linalg.inv(tf.constant(_RGB_TO_XYZ, tf.float64))
75
+ linear = tf.linalg.matvec(tf.cast(xyz_to_rgb, tf.float32), xyz)
76
+ return tf.clip_by_value(linear_to_srgb(linear), 0.0, 1.0)
77
+
78
+
79
+ def estimate_illuminant(image: tf.Tensor, valid: tf.Tensor | None = None) -> tf.Tensor:
80
+ """Linear-RGB illuminant of an sRGB image in ``[0, 1]``, colour and level.
81
+
82
+ First-order Grey-Edge with Minkowski order 6 (van de Weijer, Gevers and
83
+ Gijsenij, "Edge-Based Color Constancy", IEEE TIP 2007): the Minkowski mean
84
+ of gradient magnitudes of the image smoothed by a Gaussian of
85
+ ``width / 1024`` pixels, with gradients per ``width / 1024`` pixels so the
86
+ level does not depend on resolution. Gradients make large uniform regions,
87
+ such as open water or sky, count little, so the estimate follows the light
88
+ and exposure rather than the dominant surface colour and brightness. Clipped pixels (a channel at or above 254/255)
89
+ and their 3 x 3 neighbours, as in the authors' reference code, border
90
+ pixels, and pixels outside the optional bool ``valid`` map are left out;
91
+ excluding the whole smoothing reach instead drops the bright cloud and glare
92
+ edges that carry most of the light's colour. A diagonal (von Kries) change of
93
+ illumination or exposure scales the estimate by the same per-channel
94
+ factors, so the ratio of two frames' estimates adapts colours from one frame
95
+ to the other.
96
+ """
97
+ image = tf.cast(image, tf.float32)
98
+ if image.shape.rank != 3 or image.shape[1] is None:
99
+ raise ValueError("estimate_illuminant needs an HWC image with a static width.")
100
+ pixels_per_reference = image.shape[1] / 1024.0
101
+ sigma = GREY_EDGE_SIGMA_PER_1024_PX * pixels_per_reference
102
+ smooth = gaussian_blur_hwc(srgb_to_linear(image), sigma, sigma)
103
+ half_step = 0.5 * pixels_per_reference
104
+ gradient_y = half_step * (smooth[2:, 1:-1] - smooth[:-2, 1:-1])
105
+ gradient_x = half_step * (smooth[1:-1, 2:] - smooth[1:-1, :-2])
106
+ magnitude = tf.sqrt(tf.square(gradient_x) + tf.square(gradient_y))
107
+
108
+ clipped = tf.cast(tf.reduce_max(image, axis=-1) >= _CLIPPED_LEVEL, tf.float32)
109
+ near_clipped = tf.nn.max_pool2d(
110
+ clipped[tf.newaxis, :, :, tf.newaxis], ksize=3, strides=1, padding="SAME"
111
+ )[0, 1:-1, 1:-1, 0]
112
+ keep = near_clipped == 0.0
113
+ if valid is not None:
114
+ keep &= tf.cast(valid, tf.bool)[1:-1, 1:-1]
115
+ edges = tf.boolean_mask(magnitude, keep)
116
+ tf.debugging.assert_positive(
117
+ tf.size(edges), message="estimate_illuminant needs unclipped valid pixels."
118
+ )
119
+ order = float(MINKOWSKI_ORDER)
120
+ estimate = tf.pow(tf.reduce_mean(tf.pow(edges, order), axis=0), 1.0 / order)
121
+ tf.debugging.assert_positive(
122
+ estimate, message="estimate_illuminant needs edges in every channel."
123
+ )
124
+ return estimate
125
+
126
+
127
+ __all__ = [
128
+ "estimate_illuminant",
129
+ "lab_to_rgb",
130
+ "linear_to_srgb",
131
+ "rgb_to_lab",
132
+ "srgb_to_linear",
133
+ ]