justdata 1.3.0__tar.gz → 1.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- justdata-1.3.0/README.md → justdata-1.6.0/PKG-INFO +84 -1
- justdata-1.3.0/PKG-INFO → justdata-1.6.0/README.md +40 -44
- {justdata-1.3.0 → justdata-1.6.0}/pyproject.toml +2 -1
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/__init__.py +1 -1
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/core/finalization.py +3 -1
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/core/loader.py +5 -1
- justdata-1.6.0/src/justdata/vision/colorspace.py +133 -0
- justdata-1.6.0/src/justdata/vision/compositing.py +807 -0
- justdata-1.6.0/src/justdata/vision/corruptions/__init__.py +65 -0
- justdata-1.6.0/src/justdata/vision/corruptions/_conditions.py +73 -0
- justdata-1.6.0/src/justdata/vision/corruptions/atmosphere.py +206 -0
- justdata-1.6.0/src/justdata/vision/corruptions/continuous.py +467 -0
- justdata-1.6.0/src/justdata/vision/corruptions/exposure.py +235 -0
- justdata-1.6.0/src/justdata/vision/corruptions/lens.py +183 -0
- justdata-1.6.0/src/justdata/vision/corruptions/motion.py +86 -0
- justdata-1.6.0/src/justdata/vision/corruptions/precipitation.py +153 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/corruptions/registry.py +328 -9
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/corruptions/standard.py +8 -38
- justdata-1.6.0/src/justdata/vision/corruptions/water.py +175 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/encodings/panoptic_targets.py +121 -34
- justdata-1.6.0/src/justdata/vision/filters.py +154 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/geometry.py +134 -18
- justdata-1.6.0/src/justdata/vision/horizon.py +81 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/pipelines.py +13 -2
- justdata-1.6.0/src/justdata/vision/poisson.py +177 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/tasks/panoptic.py +7 -1
- justdata-1.3.0/src/justdata/vision/corruptions/__init__.py +0 -27
- {justdata-1.3.0 → justdata-1.6.0}/LICENSE +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/__init__.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/_dcase.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/_manifests.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/_random.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/_signal.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/adapters.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/augment/__init__.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/augment/_common.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/augment/batch.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/augment/configs.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/augment/patchout.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/augment/spectrogram.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/augment/waveform.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/batching.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/channel.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/compat/__init__.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/compat/ast.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/compat/ced.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/compat/efficientat.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/compat/panns.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/compat/passt.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/configs.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/corruptions/__init__.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/corruptions/codec.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/corruptions/datasets.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/corruptions/device.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/corruptions/dynamics.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/corruptions/filters.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/corruptions/noise.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/corruptions/registry.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/corruptions/reverb.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/corruptions/timefreq.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/datasets.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/dcase2025.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/decoding.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/eval_views.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/evals.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/filters.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/frontends/__init__.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/frontends/ast.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/frontends/kaldi.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/frontends/log.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/frontends/mel.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/frontends/mfcc.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/frontends/pcen.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/frontends/stft.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/jax.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/labels.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/layouts.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/metadata.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/normalization.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/pipelines.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/postprocessing.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/preprocessing.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/presets.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/registry.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/resampling.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/schema.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/segment.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/sources.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/stages.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/stats.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/acoustic/tasks.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/audio/__init__.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/core/__init__.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/core/_records.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/core/adapters.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/core/augmentations.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/core/cache.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/core/config_resolution.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/core/executed_config.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/core/execution.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/core/filters.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/core/inventory.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/core/label_mixing.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/core/metadata.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/core/presets.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/core/registry.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/core/replay.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/core/sources.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/core/stats.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/corruptions.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/py.typed +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/__init__.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/augmentations/__init__.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/augmentations/auto.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/augmentations/color.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/augmentations/composed.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/augmentations/geometric.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/augmentations/mixing.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/augmentations/registry.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/corruptions/blur.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/corruptions/digital.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/corruptions/noise.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/corruptions/weather.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/datasets.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/encodings/__init__.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/encodings/heatmaps.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/encodings/semantic_targets.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/minic.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/presets.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/sources.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/stages.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/tasks/__init__.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/tasks/classification.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/tasks/depth.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/tasks/segmentation.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/transforms.py +0 -0
- {justdata-1.3.0 → justdata-1.6.0}/src/justdata/vision/utils.py +0 -0
|
@@ -1,3 +1,47 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: justdata
|
|
3
|
+
Version: 1.6.0
|
|
4
|
+
Summary: A TensorFlow-native data pipeline library with a modality-neutral core and first-class computer vision and acoustic recipes
|
|
5
|
+
Author: clementpoiret
|
|
6
|
+
Author-email: clementpoiret <clement@linux.com>
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Requires-Dist: loguru>=0.7.3
|
|
10
|
+
Requires-Dist: tensorflow>=2.18.1
|
|
11
|
+
Requires-Dist: tensorflow-datasets>=4.9.9
|
|
12
|
+
Requires-Dist: datasets[audio]>=4.5.0 ; extra == 'acoustic'
|
|
13
|
+
Requires-Dist: soundfile>=0.12 ; extra == 'acoustic'
|
|
14
|
+
Requires-Dist: soxr>=0.5 ; extra == 'acoustic'
|
|
15
|
+
Requires-Dist: librosa>=0.11 ; extra == 'acoustic'
|
|
16
|
+
Requires-Dist: torch==2.10.0 ; extra == 'acoustic'
|
|
17
|
+
Requires-Dist: torchcodec==0.10.0 ; extra == 'acoustic'
|
|
18
|
+
Requires-Dist: datasets[audio]>=4.5.0 ; extra == 'dev'
|
|
19
|
+
Requires-Dist: datasets[vision]>=4.5.0 ; extra == 'dev'
|
|
20
|
+
Requires-Dist: filelock>=3.20 ; extra == 'dev'
|
|
21
|
+
Requires-Dist: librosa>=0.11 ; extra == 'dev'
|
|
22
|
+
Requires-Dist: matplotlib>=3.10.8 ; extra == 'dev'
|
|
23
|
+
Requires-Dist: pillow>=12.2.0 ; extra == 'dev'
|
|
24
|
+
Requires-Dist: pytest>=9.0.2 ; extra == 'dev'
|
|
25
|
+
Requires-Dist: ruff==0.15.20 ; extra == 'dev'
|
|
26
|
+
Requires-Dist: soundfile>=0.12 ; extra == 'dev'
|
|
27
|
+
Requires-Dist: soxr>=0.5 ; extra == 'dev'
|
|
28
|
+
Requires-Dist: torch==2.10.0 ; extra == 'dev'
|
|
29
|
+
Requires-Dist: torchcodec==0.10.0 ; extra == 'dev'
|
|
30
|
+
Requires-Dist: opencv-python-headless==5.0.0.93 ; extra == 'golden'
|
|
31
|
+
Requires-Dist: torchaudio==2.10.0 ; extra == 'golden'
|
|
32
|
+
Requires-Dist: torchlibrosa==0.1.0 ; extra == 'golden'
|
|
33
|
+
Requires-Dist: datasets[vision]>=4.5.0 ; extra == 'vision'
|
|
34
|
+
Requires-Dist: filelock>=3.20 ; extra == 'vision'
|
|
35
|
+
Requires-Dist: pillow>=12.2.0 ; extra == 'vision'
|
|
36
|
+
Requires-Dist: wilds>=2.0.0 ; extra == 'wilds'
|
|
37
|
+
Requires-Python: >=3.11, <3.14
|
|
38
|
+
Provides-Extra: acoustic
|
|
39
|
+
Provides-Extra: dev
|
|
40
|
+
Provides-Extra: golden
|
|
41
|
+
Provides-Extra: vision
|
|
42
|
+
Provides-Extra: wilds
|
|
43
|
+
Description-Content-Type: text/markdown
|
|
44
|
+
|
|
1
45
|
# justdata
|
|
2
46
|
|
|
3
47
|
A TensorFlow-native data pipeline library with a modality-neutral core and first-class computer vision and acoustic
|
|
@@ -81,6 +125,8 @@ enabled.
|
|
|
81
125
|
| `justdata.core.load_inventory(admitted, ...)` | Apply the shared pipeline to an admitted inventory, including raw epoch finalization. |
|
|
82
126
|
| `justdata.core.ExecutedConfig` | Immutable, canonical snapshot returned by loaders when `return_config=True`. |
|
|
83
127
|
| `justdata.vision.minic.create_minic_datasets(...)` | Constructs Mini-C corruption benchmark datasets from a shared preprocessed base dataset. |
|
|
128
|
+
| `justdata.vision.corruptions.apply_corruption_at(...)` | Renders a scene condition at a continuous strength and returns its record. |
|
|
129
|
+
| `justdata.vision.corruptions.apply_corruption_stack(...)` | Applies ordered conditions in float, rounds once, and returns the stack record. |
|
|
84
130
|
| `justdata.acoustic.corruptions.create_audio_corruption_datasets(...)` | Constructs acoustic corruption benchmark datasets from a shared preprocessed base dataset. |
|
|
85
131
|
| `justdata.acoustic.dcase2025.make_source_dataset(...)` | Builds DCASE Task 1 source-domain datasets with split-safety checks. |
|
|
86
132
|
|
|
@@ -920,6 +966,43 @@ severity; `none` removes metadata according to the shared core contract. The dat
|
|
|
920
966
|
enumeration-position seed salting. Consumers with their own sample-level PRNG lineage should call `apply_corruption`
|
|
921
967
|
directly.
|
|
922
968
|
|
|
969
|
+
### Scene conditions at continuous strength
|
|
970
|
+
|
|
971
|
+
Ten photometric conditions declare a continuous-strength contract: `haze`, `low_light`, `sun_glitter`, `lens_droplets`,
|
|
972
|
+
`rain`, `motion_blur`, `overexposure`, `white_balance_warm`, `white_balance_cool`, and `shot_noise`.
|
|
973
|
+
`apply_corruption_at` renders one of them at any strength between 0 (identity) and 5 (the last severity row). It takes
|
|
974
|
+
an optional episode seed for structure that persists across frames, and optional `depth_m`, `horizon`, or `region`
|
|
975
|
+
inputs. It returns the image and a record of the resolved parameters and mode. `apply_corruption_stack` applies ordered
|
|
976
|
+
steps in float and rounds once.
|
|
977
|
+
|
|
978
|
+
```python
|
|
979
|
+
import tensorflow as tf
|
|
980
|
+
|
|
981
|
+
from justdata.vision.corruptions import apply_corruption_at, apply_corruption_stack
|
|
982
|
+
|
|
983
|
+
frame = apply_corruption_at(
|
|
984
|
+
decoded_rgb_uint8,
|
|
985
|
+
name="lens_droplets",
|
|
986
|
+
version="1.0.0",
|
|
987
|
+
strength=1.7,
|
|
988
|
+
seed=tf.constant([frame_index, 0], tf.int32),
|
|
989
|
+
episode_seed=tf.constant([episode_index, 0], tf.int32),
|
|
990
|
+
)
|
|
991
|
+
dusk = apply_corruption_stack(
|
|
992
|
+
decoded_rgb_uint8,
|
|
993
|
+
steps=[("low_light", "1.0.0", 2.0), ("white_balance_cool", "1.0.0", 1.5)],
|
|
994
|
+
seed=tf.constant([frame_index, 1], tf.int32),
|
|
995
|
+
)
|
|
996
|
+
```
|
|
997
|
+
|
|
998
|
+
See [continuous strength](docs/vision.md#continuous-strength-episodes-and-scene-inputs) and
|
|
999
|
+
[scene condition corruptions](docs/vision.md#scene-condition-corruptions) for the contract, the tables, and their
|
|
1000
|
+
sources, and `examples/vision/maritime_conditions.py` for a drift over an episode.
|
|
1001
|
+
|
|
1002
|
+
To add rare objects, `composite_instances` pastes annotated crops onto caller-chosen anchor pixels, updates the class
|
|
1003
|
+
and instance maps, and records every request as placed or rejected; see
|
|
1004
|
+
[compositing annotated instances](docs/vision.md#16-compositing-annotated-instances).
|
|
1005
|
+
|
|
923
1006
|
______________________________________________________________________
|
|
924
1007
|
|
|
925
1008
|
## Audio Corruption Benchmark
|
|
@@ -1287,7 +1370,7 @@ package-support jobs pass. Build artifacts are retained for one week. Tag pipeli
|
|
|
1287
1370
|
to complete a release.
|
|
1288
1371
|
|
|
1289
1372
|
Before tagging, update `pyproject.toml`, `src/justdata/__init__.py`, and `CHANGELOG.md`, then run `uv lock`. Push a tag
|
|
1290
|
-
matching the package version, with an optional `v` prefix (for example, `v1.
|
|
1373
|
+
matching the package version, with an optional `v` prefix (for example, `v1.6.0` or `1.6.0`). Publishing rejects tags
|
|
1291
1374
|
that disagree with either version declaration. Branch pipelines build artifacts without publishing.
|
|
1292
1375
|
|
|
1293
1376
|
Configure a [PyPI GitLab trusted publisher](https://docs.pypi.org/trusted-publishers/adding-a-publisher/) for the
|
|
@@ -1,46 +1,3 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: justdata
|
|
3
|
-
Version: 1.3.0
|
|
4
|
-
Summary: A TensorFlow-native data pipeline library with a modality-neutral core and first-class computer vision and acoustic recipes
|
|
5
|
-
Author: clementpoiret
|
|
6
|
-
Author-email: clementpoiret <clement@linux.com>
|
|
7
|
-
License-Expression: MIT
|
|
8
|
-
License-File: LICENSE
|
|
9
|
-
Requires-Dist: loguru>=0.7.3
|
|
10
|
-
Requires-Dist: tensorflow>=2.18.1
|
|
11
|
-
Requires-Dist: tensorflow-datasets>=4.9.9
|
|
12
|
-
Requires-Dist: datasets[audio]>=4.5.0 ; extra == 'acoustic'
|
|
13
|
-
Requires-Dist: soundfile>=0.12 ; extra == 'acoustic'
|
|
14
|
-
Requires-Dist: soxr>=0.5 ; extra == 'acoustic'
|
|
15
|
-
Requires-Dist: librosa>=0.11 ; extra == 'acoustic'
|
|
16
|
-
Requires-Dist: torch==2.10.0 ; extra == 'acoustic'
|
|
17
|
-
Requires-Dist: torchcodec==0.10.0 ; extra == 'acoustic'
|
|
18
|
-
Requires-Dist: datasets[audio]>=4.5.0 ; extra == 'dev'
|
|
19
|
-
Requires-Dist: datasets[vision]>=4.5.0 ; extra == 'dev'
|
|
20
|
-
Requires-Dist: filelock>=3.20 ; extra == 'dev'
|
|
21
|
-
Requires-Dist: librosa>=0.11 ; extra == 'dev'
|
|
22
|
-
Requires-Dist: matplotlib>=3.10.8 ; extra == 'dev'
|
|
23
|
-
Requires-Dist: pillow>=12.2.0 ; extra == 'dev'
|
|
24
|
-
Requires-Dist: pytest>=9.0.2 ; extra == 'dev'
|
|
25
|
-
Requires-Dist: ruff==0.15.20 ; extra == 'dev'
|
|
26
|
-
Requires-Dist: soundfile>=0.12 ; extra == 'dev'
|
|
27
|
-
Requires-Dist: soxr>=0.5 ; extra == 'dev'
|
|
28
|
-
Requires-Dist: torch==2.10.0 ; extra == 'dev'
|
|
29
|
-
Requires-Dist: torchcodec==0.10.0 ; extra == 'dev'
|
|
30
|
-
Requires-Dist: torchaudio==2.10.0 ; extra == 'golden'
|
|
31
|
-
Requires-Dist: torchlibrosa==0.1.0 ; extra == 'golden'
|
|
32
|
-
Requires-Dist: datasets[vision]>=4.5.0 ; extra == 'vision'
|
|
33
|
-
Requires-Dist: filelock>=3.20 ; extra == 'vision'
|
|
34
|
-
Requires-Dist: pillow>=12.2.0 ; extra == 'vision'
|
|
35
|
-
Requires-Dist: wilds>=2.0.0 ; extra == 'wilds'
|
|
36
|
-
Requires-Python: >=3.11, <3.14
|
|
37
|
-
Provides-Extra: acoustic
|
|
38
|
-
Provides-Extra: dev
|
|
39
|
-
Provides-Extra: golden
|
|
40
|
-
Provides-Extra: vision
|
|
41
|
-
Provides-Extra: wilds
|
|
42
|
-
Description-Content-Type: text/markdown
|
|
43
|
-
|
|
44
1
|
# justdata
|
|
45
2
|
|
|
46
3
|
A TensorFlow-native data pipeline library with a modality-neutral core and first-class computer vision and acoustic
|
|
@@ -124,6 +81,8 @@ enabled.
|
|
|
124
81
|
| `justdata.core.load_inventory(admitted, ...)` | Apply the shared pipeline to an admitted inventory, including raw epoch finalization. |
|
|
125
82
|
| `justdata.core.ExecutedConfig` | Immutable, canonical snapshot returned by loaders when `return_config=True`. |
|
|
126
83
|
| `justdata.vision.minic.create_minic_datasets(...)` | Constructs Mini-C corruption benchmark datasets from a shared preprocessed base dataset. |
|
|
84
|
+
| `justdata.vision.corruptions.apply_corruption_at(...)` | Renders a scene condition at a continuous strength and returns its record. |
|
|
85
|
+
| `justdata.vision.corruptions.apply_corruption_stack(...)` | Applies ordered conditions in float, rounds once, and returns the stack record. |
|
|
127
86
|
| `justdata.acoustic.corruptions.create_audio_corruption_datasets(...)` | Constructs acoustic corruption benchmark datasets from a shared preprocessed base dataset. |
|
|
128
87
|
| `justdata.acoustic.dcase2025.make_source_dataset(...)` | Builds DCASE Task 1 source-domain datasets with split-safety checks. |
|
|
129
88
|
|
|
@@ -963,6 +922,43 @@ severity; `none` removes metadata according to the shared core contract. The dat
|
|
|
963
922
|
enumeration-position seed salting. Consumers with their own sample-level PRNG lineage should call `apply_corruption`
|
|
964
923
|
directly.
|
|
965
924
|
|
|
925
|
+
### Scene conditions at continuous strength
|
|
926
|
+
|
|
927
|
+
Ten photometric conditions declare a continuous-strength contract: `haze`, `low_light`, `sun_glitter`, `lens_droplets`,
|
|
928
|
+
`rain`, `motion_blur`, `overexposure`, `white_balance_warm`, `white_balance_cool`, and `shot_noise`.
|
|
929
|
+
`apply_corruption_at` renders one of them at any strength between 0 (identity) and 5 (the last severity row). It takes
|
|
930
|
+
an optional episode seed for structure that persists across frames, and optional `depth_m`, `horizon`, or `region`
|
|
931
|
+
inputs. It returns the image and a record of the resolved parameters and mode. `apply_corruption_stack` applies ordered
|
|
932
|
+
steps in float and rounds once.
|
|
933
|
+
|
|
934
|
+
```python
|
|
935
|
+
import tensorflow as tf
|
|
936
|
+
|
|
937
|
+
from justdata.vision.corruptions import apply_corruption_at, apply_corruption_stack
|
|
938
|
+
|
|
939
|
+
frame = apply_corruption_at(
|
|
940
|
+
decoded_rgb_uint8,
|
|
941
|
+
name="lens_droplets",
|
|
942
|
+
version="1.0.0",
|
|
943
|
+
strength=1.7,
|
|
944
|
+
seed=tf.constant([frame_index, 0], tf.int32),
|
|
945
|
+
episode_seed=tf.constant([episode_index, 0], tf.int32),
|
|
946
|
+
)
|
|
947
|
+
dusk = apply_corruption_stack(
|
|
948
|
+
decoded_rgb_uint8,
|
|
949
|
+
steps=[("low_light", "1.0.0", 2.0), ("white_balance_cool", "1.0.0", 1.5)],
|
|
950
|
+
seed=tf.constant([frame_index, 1], tf.int32),
|
|
951
|
+
)
|
|
952
|
+
```
|
|
953
|
+
|
|
954
|
+
See [continuous strength](docs/vision.md#continuous-strength-episodes-and-scene-inputs) and
|
|
955
|
+
[scene condition corruptions](docs/vision.md#scene-condition-corruptions) for the contract, the tables, and their
|
|
956
|
+
sources, and `examples/vision/maritime_conditions.py` for a drift over an episode.
|
|
957
|
+
|
|
958
|
+
To add rare objects, `composite_instances` pastes annotated crops onto caller-chosen anchor pixels, updates the class
|
|
959
|
+
and instance maps, and records every request as placed or rejected; see
|
|
960
|
+
[compositing annotated instances](docs/vision.md#16-compositing-annotated-instances).
|
|
961
|
+
|
|
966
962
|
______________________________________________________________________
|
|
967
963
|
|
|
968
964
|
## Audio Corruption Benchmark
|
|
@@ -1330,7 +1326,7 @@ package-support jobs pass. Build artifacts are retained for one week. Tag pipeli
|
|
|
1330
1326
|
to complete a release.
|
|
1331
1327
|
|
|
1332
1328
|
Before tagging, update `pyproject.toml`, `src/justdata/__init__.py`, and `CHANGELOG.md`, then run `uv lock`. Push a tag
|
|
1333
|
-
matching the package version, with an optional `v` prefix (for example, `v1.
|
|
1329
|
+
matching the package version, with an optional `v` prefix (for example, `v1.6.0` or `1.6.0`). Publishing rejects tags
|
|
1334
1330
|
that disagree with either version declaration. Branch pipelines build artifacts without publishing.
|
|
1335
1331
|
|
|
1336
1332
|
Configure a [PyPI GitLab trusted publisher](https://docs.pypi.org/trusted-publishers/adding-a-publisher/) for the
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "justdata"
|
|
3
|
-
version = "1.
|
|
3
|
+
version = "1.6.0"
|
|
4
4
|
description = "A TensorFlow-native data pipeline library with a modality-neutral core and first-class computer vision and acoustic recipes"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "MIT"
|
|
@@ -33,6 +33,7 @@ acoustic = [
|
|
|
33
33
|
"torchcodec==0.10.0",
|
|
34
34
|
]
|
|
35
35
|
golden = [
|
|
36
|
+
"opencv-python-headless==5.0.0.93",
|
|
36
37
|
"torchaudio==2.10.0",
|
|
37
38
|
"torchlibrosa==0.1.0",
|
|
38
39
|
]
|
|
@@ -305,13 +305,15 @@ def finalize_dataset(
|
|
|
305
305
|
deterministic=deterministic,
|
|
306
306
|
)
|
|
307
307
|
elif rng is not None:
|
|
308
|
+
# Parallel calls draw from the shared generator in scheduling order,
|
|
309
|
+
# so batches would trade seeds between otherwise identical runs.
|
|
308
310
|
ds = ds.map(
|
|
309
311
|
lambda batch: late_augment_fn(
|
|
310
312
|
batch,
|
|
311
313
|
num_classes=num_classes,
|
|
312
314
|
seed=rng.make_seeds(1)[:, 0],
|
|
313
315
|
),
|
|
314
|
-
num_parallel_calls=parallel_calls,
|
|
316
|
+
num_parallel_calls=1 if deterministic else parallel_calls,
|
|
315
317
|
deterministic=deterministic,
|
|
316
318
|
)
|
|
317
319
|
|
|
@@ -478,6 +478,10 @@ def _prepare_ds(
|
|
|
478
478
|
):
|
|
479
479
|
"""Shared pipeline, resolving its source only after validating options."""
|
|
480
480
|
rng = tf.random.Generator.from_seed(seed)
|
|
481
|
+
# Late augmentation draws in a later tf.data stage that runs concurrently with
|
|
482
|
+
# the per-sample augment map; sharing one generator would interleave the two
|
|
483
|
+
# stages' draws differently on every run.
|
|
484
|
+
late_augment_rng = tf.random.Generator.from_seed(seed).split(1)[0]
|
|
481
485
|
if metadata_mode is None:
|
|
482
486
|
metadata_mode = (
|
|
483
487
|
pipeline.kwargs.get("metadata_mode", "full")
|
|
@@ -685,7 +689,7 @@ def _prepare_ds(
|
|
|
685
689
|
"is_training": is_training,
|
|
686
690
|
"apply_late_augment": apply_augmentation,
|
|
687
691
|
"late_augment_fn": late_augment_fn,
|
|
688
|
-
"rng":
|
|
692
|
+
"rng": late_augment_rng,
|
|
689
693
|
"deterministic": deterministic,
|
|
690
694
|
"shuffle_buffer": shuffle_buffer if is_training else None,
|
|
691
695
|
"shuffle_seed": seed,
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
"""Float32 RGB colour-space conversions on values in ``[0, 1]``.
|
|
2
|
+
|
|
3
|
+
sRGB follows IEC 61966-2-1 with the D65 white point; CIELAB uses the same
|
|
4
|
+
white, so ``lab_to_rgb(rgb_to_lab(x))`` returns ``x`` up to float32 rounding.
|
|
5
|
+
``estimate_illuminant`` gives a frame's illuminant colour for chromatic
|
|
6
|
+
adaptation between frames.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import tensorflow as tf
|
|
10
|
+
|
|
11
|
+
from justdata.vision.filters import gaussian_blur_hwc
|
|
12
|
+
|
|
13
|
+
_RGB_TO_XYZ = (
|
|
14
|
+
(0.4124564, 0.3575761, 0.1804375),
|
|
15
|
+
(0.2126729, 0.7151522, 0.0721750),
|
|
16
|
+
(0.0193339, 0.1191920, 0.9503041),
|
|
17
|
+
)
|
|
18
|
+
_D65_WHITE_XYZ = (0.95047, 1.0, 1.08883)
|
|
19
|
+
_LAB_EPSILON = 216.0 / 24389.0
|
|
20
|
+
_LAB_KAPPA = 24389.0 / 27.0
|
|
21
|
+
MINKOWSKI_ORDER = 6
|
|
22
|
+
# Grey-Edge smoothing in pixels per 1024 pixels of image width.
|
|
23
|
+
GREY_EDGE_SIGMA_PER_1024_PX = 1.0
|
|
24
|
+
_CLIPPED_LEVEL = 254.0 / 255.0
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def srgb_to_linear(image: tf.Tensor) -> tf.Tensor:
|
|
28
|
+
image = tf.cast(image, tf.float32)
|
|
29
|
+
return tf.where(
|
|
30
|
+
image <= 0.04045,
|
|
31
|
+
image / 12.92,
|
|
32
|
+
tf.pow((tf.maximum(image, 0.04045) + 0.055) / 1.055, 2.4),
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def linear_to_srgb(image: tf.Tensor) -> tf.Tensor:
|
|
37
|
+
image = tf.maximum(tf.cast(image, tf.float32), 0.0)
|
|
38
|
+
return tf.where(
|
|
39
|
+
image <= 0.0031308,
|
|
40
|
+
image * 12.92,
|
|
41
|
+
1.055 * tf.pow(tf.maximum(image, 0.0031308), 1.0 / 2.4) - 0.055,
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def rgb_to_lab(image: tf.Tensor) -> tf.Tensor:
|
|
46
|
+
"""Convert sRGB ``[..., 3]`` to CIELAB with L in ``[0, 100]``."""
|
|
47
|
+
linear = srgb_to_linear(image)
|
|
48
|
+
xyz = tf.linalg.matvec(tf.constant(_RGB_TO_XYZ, tf.float32), linear)
|
|
49
|
+
ratio = xyz / tf.constant(_D65_WHITE_XYZ, tf.float32)
|
|
50
|
+
f = tf.where(
|
|
51
|
+
ratio > _LAB_EPSILON,
|
|
52
|
+
tf.pow(tf.maximum(ratio, _LAB_EPSILON), 1.0 / 3.0),
|
|
53
|
+
(_LAB_KAPPA * ratio + 16.0) / 116.0,
|
|
54
|
+
)
|
|
55
|
+
lightness = 116.0 * f[..., 1] - 16.0
|
|
56
|
+
a = 500.0 * (f[..., 0] - f[..., 1])
|
|
57
|
+
b = 200.0 * (f[..., 1] - f[..., 2])
|
|
58
|
+
return tf.stack([lightness, a, b], axis=-1)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def lab_to_rgb(lab: tf.Tensor) -> tf.Tensor:
|
|
62
|
+
"""Convert CIELAB ``[..., 3]`` to sRGB, clipped to ``[0, 1]``."""
|
|
63
|
+
lab = tf.cast(lab, tf.float32)
|
|
64
|
+
fy = (lab[..., 0] + 16.0) / 116.0
|
|
65
|
+
fx = fy + lab[..., 1] / 500.0
|
|
66
|
+
fz = fy - lab[..., 2] / 200.0
|
|
67
|
+
f = tf.stack([fx, fy, fz], axis=-1)
|
|
68
|
+
ratio = tf.where(
|
|
69
|
+
tf.pow(f, 3.0) > _LAB_EPSILON,
|
|
70
|
+
tf.pow(f, 3.0),
|
|
71
|
+
(116.0 * f - 16.0) / _LAB_KAPPA,
|
|
72
|
+
)
|
|
73
|
+
xyz = ratio * tf.constant(_D65_WHITE_XYZ, tf.float32)
|
|
74
|
+
xyz_to_rgb = tf.linalg.inv(tf.constant(_RGB_TO_XYZ, tf.float64))
|
|
75
|
+
linear = tf.linalg.matvec(tf.cast(xyz_to_rgb, tf.float32), xyz)
|
|
76
|
+
return tf.clip_by_value(linear_to_srgb(linear), 0.0, 1.0)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def estimate_illuminant(image: tf.Tensor, valid: tf.Tensor | None = None) -> tf.Tensor:
|
|
80
|
+
"""Linear-RGB illuminant of an sRGB image in ``[0, 1]``, colour and level.
|
|
81
|
+
|
|
82
|
+
First-order Grey-Edge with Minkowski order 6 (van de Weijer, Gevers and
|
|
83
|
+
Gijsenij, "Edge-Based Color Constancy", IEEE TIP 2007): the Minkowski mean
|
|
84
|
+
of gradient magnitudes of the image smoothed by a Gaussian of
|
|
85
|
+
``width / 1024`` pixels, with gradients per ``width / 1024`` pixels so the
|
|
86
|
+
level does not depend on resolution. Gradients make large uniform regions,
|
|
87
|
+
such as open water or sky, count little, so the estimate follows the light
|
|
88
|
+
and exposure rather than the dominant surface colour and brightness. Clipped pixels (a channel at or above 254/255)
|
|
89
|
+
and their 3 x 3 neighbours, as in the authors' reference code, border
|
|
90
|
+
pixels, and pixels outside the optional bool ``valid`` map are left out;
|
|
91
|
+
excluding the whole smoothing reach instead drops the bright cloud and glare
|
|
92
|
+
edges that carry most of the light's colour. A diagonal (von Kries) change of
|
|
93
|
+
illumination or exposure scales the estimate by the same per-channel
|
|
94
|
+
factors, so the ratio of two frames' estimates adapts colours from one frame
|
|
95
|
+
to the other.
|
|
96
|
+
"""
|
|
97
|
+
image = tf.cast(image, tf.float32)
|
|
98
|
+
if image.shape.rank != 3 or image.shape[1] is None:
|
|
99
|
+
raise ValueError("estimate_illuminant needs an HWC image with a static width.")
|
|
100
|
+
pixels_per_reference = image.shape[1] / 1024.0
|
|
101
|
+
sigma = GREY_EDGE_SIGMA_PER_1024_PX * pixels_per_reference
|
|
102
|
+
smooth = gaussian_blur_hwc(srgb_to_linear(image), sigma, sigma)
|
|
103
|
+
half_step = 0.5 * pixels_per_reference
|
|
104
|
+
gradient_y = half_step * (smooth[2:, 1:-1] - smooth[:-2, 1:-1])
|
|
105
|
+
gradient_x = half_step * (smooth[1:-1, 2:] - smooth[1:-1, :-2])
|
|
106
|
+
magnitude = tf.sqrt(tf.square(gradient_x) + tf.square(gradient_y))
|
|
107
|
+
|
|
108
|
+
clipped = tf.cast(tf.reduce_max(image, axis=-1) >= _CLIPPED_LEVEL, tf.float32)
|
|
109
|
+
near_clipped = tf.nn.max_pool2d(
|
|
110
|
+
clipped[tf.newaxis, :, :, tf.newaxis], ksize=3, strides=1, padding="SAME"
|
|
111
|
+
)[0, 1:-1, 1:-1, 0]
|
|
112
|
+
keep = near_clipped == 0.0
|
|
113
|
+
if valid is not None:
|
|
114
|
+
keep &= tf.cast(valid, tf.bool)[1:-1, 1:-1]
|
|
115
|
+
edges = tf.boolean_mask(magnitude, keep)
|
|
116
|
+
tf.debugging.assert_positive(
|
|
117
|
+
tf.size(edges), message="estimate_illuminant needs unclipped valid pixels."
|
|
118
|
+
)
|
|
119
|
+
order = float(MINKOWSKI_ORDER)
|
|
120
|
+
estimate = tf.pow(tf.reduce_mean(tf.pow(edges, order), axis=0), 1.0 / order)
|
|
121
|
+
tf.debugging.assert_positive(
|
|
122
|
+
estimate, message="estimate_illuminant needs edges in every channel."
|
|
123
|
+
)
|
|
124
|
+
return estimate
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
__all__ = [
|
|
128
|
+
"estimate_illuminant",
|
|
129
|
+
"lab_to_rgb",
|
|
130
|
+
"linear_to_srgb",
|
|
131
|
+
"rgb_to_lab",
|
|
132
|
+
"srgb_to_linear",
|
|
133
|
+
]
|