justdata 1.3.0__tar.gz → 1.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- justdata-1.3.0/README.md → justdata-1.5.0/PKG-INFO +84 -1
- justdata-1.3.0/PKG-INFO → justdata-1.5.0/README.md +40 -44
- {justdata-1.3.0 → justdata-1.5.0}/pyproject.toml +2 -1
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/__init__.py +1 -1
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/core/finalization.py +3 -1
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/core/loader.py +5 -1
- justdata-1.5.0/src/justdata/vision/colorspace.py +71 -0
- justdata-1.5.0/src/justdata/vision/compositing.py +717 -0
- justdata-1.5.0/src/justdata/vision/corruptions/__init__.py +65 -0
- justdata-1.5.0/src/justdata/vision/corruptions/_conditions.py +73 -0
- justdata-1.5.0/src/justdata/vision/corruptions/atmosphere.py +206 -0
- justdata-1.5.0/src/justdata/vision/corruptions/continuous.py +467 -0
- justdata-1.5.0/src/justdata/vision/corruptions/exposure.py +235 -0
- justdata-1.5.0/src/justdata/vision/corruptions/lens.py +183 -0
- justdata-1.5.0/src/justdata/vision/corruptions/motion.py +86 -0
- justdata-1.5.0/src/justdata/vision/corruptions/precipitation.py +153 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/vision/corruptions/registry.py +328 -9
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/vision/corruptions/standard.py +8 -38
- justdata-1.5.0/src/justdata/vision/corruptions/water.py +175 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/vision/encodings/panoptic_targets.py +121 -34
- justdata-1.5.0/src/justdata/vision/filters.py +154 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/vision/geometry.py +134 -18
- justdata-1.5.0/src/justdata/vision/horizon.py +81 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/vision/pipelines.py +13 -2
- justdata-1.5.0/src/justdata/vision/poisson.py +177 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/vision/tasks/panoptic.py +7 -1
- justdata-1.3.0/src/justdata/vision/corruptions/__init__.py +0 -27
- {justdata-1.3.0 → justdata-1.5.0}/LICENSE +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/__init__.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/_dcase.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/_manifests.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/_random.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/_signal.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/adapters.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/augment/__init__.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/augment/_common.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/augment/batch.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/augment/configs.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/augment/patchout.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/augment/spectrogram.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/augment/waveform.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/batching.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/channel.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/compat/__init__.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/compat/ast.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/compat/ced.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/compat/efficientat.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/compat/panns.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/compat/passt.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/configs.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/corruptions/__init__.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/corruptions/codec.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/corruptions/datasets.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/corruptions/device.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/corruptions/dynamics.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/corruptions/filters.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/corruptions/noise.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/corruptions/registry.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/corruptions/reverb.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/corruptions/timefreq.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/datasets.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/dcase2025.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/decoding.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/eval_views.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/evals.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/filters.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/frontends/__init__.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/frontends/ast.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/frontends/kaldi.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/frontends/log.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/frontends/mel.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/frontends/mfcc.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/frontends/pcen.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/frontends/stft.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/jax.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/labels.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/layouts.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/metadata.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/normalization.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/pipelines.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/postprocessing.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/preprocessing.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/presets.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/registry.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/resampling.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/schema.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/segment.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/sources.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/stages.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/stats.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/acoustic/tasks.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/audio/__init__.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/core/__init__.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/core/_records.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/core/adapters.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/core/augmentations.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/core/cache.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/core/config_resolution.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/core/executed_config.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/core/execution.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/core/filters.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/core/inventory.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/core/label_mixing.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/core/metadata.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/core/presets.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/core/registry.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/core/replay.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/core/sources.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/core/stats.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/corruptions.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/py.typed +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/vision/__init__.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/vision/augmentations/__init__.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/vision/augmentations/auto.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/vision/augmentations/color.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/vision/augmentations/composed.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/vision/augmentations/geometric.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/vision/augmentations/mixing.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/vision/augmentations/registry.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/vision/corruptions/blur.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/vision/corruptions/digital.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/vision/corruptions/noise.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/vision/corruptions/weather.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/vision/datasets.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/vision/encodings/__init__.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/vision/encodings/heatmaps.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/vision/encodings/semantic_targets.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/vision/minic.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/vision/presets.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/vision/sources.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/vision/stages.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/vision/tasks/__init__.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/vision/tasks/classification.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/vision/tasks/depth.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/vision/tasks/segmentation.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/vision/transforms.py +0 -0
- {justdata-1.3.0 → justdata-1.5.0}/src/justdata/vision/utils.py +0 -0
|
@@ -1,3 +1,47 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: justdata
|
|
3
|
+
Version: 1.5.0
|
|
4
|
+
Summary: A TensorFlow-native data pipeline library with a modality-neutral core and first-class computer vision and acoustic recipes
|
|
5
|
+
Author: clementpoiret
|
|
6
|
+
Author-email: clementpoiret <clement@linux.com>
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Requires-Dist: loguru>=0.7.3
|
|
10
|
+
Requires-Dist: tensorflow>=2.18.1
|
|
11
|
+
Requires-Dist: tensorflow-datasets>=4.9.9
|
|
12
|
+
Requires-Dist: datasets[audio]>=4.5.0 ; extra == 'acoustic'
|
|
13
|
+
Requires-Dist: soundfile>=0.12 ; extra == 'acoustic'
|
|
14
|
+
Requires-Dist: soxr>=0.5 ; extra == 'acoustic'
|
|
15
|
+
Requires-Dist: librosa>=0.11 ; extra == 'acoustic'
|
|
16
|
+
Requires-Dist: torch==2.10.0 ; extra == 'acoustic'
|
|
17
|
+
Requires-Dist: torchcodec==0.10.0 ; extra == 'acoustic'
|
|
18
|
+
Requires-Dist: datasets[audio]>=4.5.0 ; extra == 'dev'
|
|
19
|
+
Requires-Dist: datasets[vision]>=4.5.0 ; extra == 'dev'
|
|
20
|
+
Requires-Dist: filelock>=3.20 ; extra == 'dev'
|
|
21
|
+
Requires-Dist: librosa>=0.11 ; extra == 'dev'
|
|
22
|
+
Requires-Dist: matplotlib>=3.10.8 ; extra == 'dev'
|
|
23
|
+
Requires-Dist: pillow>=12.2.0 ; extra == 'dev'
|
|
24
|
+
Requires-Dist: pytest>=9.0.2 ; extra == 'dev'
|
|
25
|
+
Requires-Dist: ruff==0.15.20 ; extra == 'dev'
|
|
26
|
+
Requires-Dist: soundfile>=0.12 ; extra == 'dev'
|
|
27
|
+
Requires-Dist: soxr>=0.5 ; extra == 'dev'
|
|
28
|
+
Requires-Dist: torch==2.10.0 ; extra == 'dev'
|
|
29
|
+
Requires-Dist: torchcodec==0.10.0 ; extra == 'dev'
|
|
30
|
+
Requires-Dist: opencv-python-headless==5.0.0.93 ; extra == 'golden'
|
|
31
|
+
Requires-Dist: torchaudio==2.10.0 ; extra == 'golden'
|
|
32
|
+
Requires-Dist: torchlibrosa==0.1.0 ; extra == 'golden'
|
|
33
|
+
Requires-Dist: datasets[vision]>=4.5.0 ; extra == 'vision'
|
|
34
|
+
Requires-Dist: filelock>=3.20 ; extra == 'vision'
|
|
35
|
+
Requires-Dist: pillow>=12.2.0 ; extra == 'vision'
|
|
36
|
+
Requires-Dist: wilds>=2.0.0 ; extra == 'wilds'
|
|
37
|
+
Requires-Python: >=3.11, <3.14
|
|
38
|
+
Provides-Extra: acoustic
|
|
39
|
+
Provides-Extra: dev
|
|
40
|
+
Provides-Extra: golden
|
|
41
|
+
Provides-Extra: vision
|
|
42
|
+
Provides-Extra: wilds
|
|
43
|
+
Description-Content-Type: text/markdown
|
|
44
|
+
|
|
1
45
|
# justdata
|
|
2
46
|
|
|
3
47
|
A TensorFlow-native data pipeline library with a modality-neutral core and first-class computer vision and acoustic
|
|
@@ -81,6 +125,8 @@ enabled.
|
|
|
81
125
|
| `justdata.core.load_inventory(admitted, ...)` | Apply the shared pipeline to an admitted inventory, including raw epoch finalization. |
|
|
82
126
|
| `justdata.core.ExecutedConfig` | Immutable, canonical snapshot returned by loaders when `return_config=True`. |
|
|
83
127
|
| `justdata.vision.minic.create_minic_datasets(...)` | Constructs Mini-C corruption benchmark datasets from a shared preprocessed base dataset. |
|
|
128
|
+
| `justdata.vision.corruptions.apply_corruption_at(...)` | Renders a scene condition at a continuous strength and returns its record. |
|
|
129
|
+
| `justdata.vision.corruptions.apply_corruption_stack(...)` | Applies ordered conditions in float, rounds once, and returns the stack record. |
|
|
84
130
|
| `justdata.acoustic.corruptions.create_audio_corruption_datasets(...)` | Constructs acoustic corruption benchmark datasets from a shared preprocessed base dataset. |
|
|
85
131
|
| `justdata.acoustic.dcase2025.make_source_dataset(...)` | Builds DCASE Task 1 source-domain datasets with split-safety checks. |
|
|
86
132
|
|
|
@@ -920,6 +966,43 @@ severity; `none` removes metadata according to the shared core contract. The dat
|
|
|
920
966
|
enumeration-position seed salting. Consumers with their own sample-level PRNG lineage should call `apply_corruption`
|
|
921
967
|
directly.
|
|
922
968
|
|
|
969
|
+
### Scene conditions at continuous strength
|
|
970
|
+
|
|
971
|
+
Ten photometric conditions declare a continuous-strength contract: `haze`, `low_light`, `sun_glitter`, `lens_droplets`,
|
|
972
|
+
`rain`, `motion_blur`, `overexposure`, `white_balance_warm`, `white_balance_cool`, and `shot_noise`.
|
|
973
|
+
`apply_corruption_at` renders one of them at any strength between 0 (identity) and 5 (the last severity row). It takes
|
|
974
|
+
an optional episode seed for structure that persists across frames, and optional `depth_m`, `horizon`, or `region`
|
|
975
|
+
inputs. It returns the image and a record of the resolved parameters and mode. `apply_corruption_stack` applies ordered
|
|
976
|
+
steps in float and rounds once.
|
|
977
|
+
|
|
978
|
+
```python
|
|
979
|
+
import tensorflow as tf
|
|
980
|
+
|
|
981
|
+
from justdata.vision.corruptions import apply_corruption_at, apply_corruption_stack
|
|
982
|
+
|
|
983
|
+
frame = apply_corruption_at(
|
|
984
|
+
decoded_rgb_uint8,
|
|
985
|
+
name="lens_droplets",
|
|
986
|
+
version="1.0.0",
|
|
987
|
+
strength=1.7,
|
|
988
|
+
seed=tf.constant([frame_index, 0], tf.int32),
|
|
989
|
+
episode_seed=tf.constant([episode_index, 0], tf.int32),
|
|
990
|
+
)
|
|
991
|
+
dusk = apply_corruption_stack(
|
|
992
|
+
decoded_rgb_uint8,
|
|
993
|
+
steps=[("low_light", "1.0.0", 2.0), ("white_balance_cool", "1.0.0", 1.5)],
|
|
994
|
+
seed=tf.constant([frame_index, 1], tf.int32),
|
|
995
|
+
)
|
|
996
|
+
```
|
|
997
|
+
|
|
998
|
+
See [continuous strength](docs/vision.md#continuous-strength-episodes-and-scene-inputs) and
|
|
999
|
+
[scene condition corruptions](docs/vision.md#scene-condition-corruptions) for the contract, the tables, and their
|
|
1000
|
+
sources, and `examples/vision/maritime_conditions.py` for a drift over an episode.
|
|
1001
|
+
|
|
1002
|
+
To add rare objects, `composite_instances` pastes annotated crops onto caller-chosen anchor pixels, updates the class
|
|
1003
|
+
and instance maps, and records every request as placed or rejected; see
|
|
1004
|
+
[compositing annotated instances](docs/vision.md#16-compositing-annotated-instances).
|
|
1005
|
+
|
|
923
1006
|
______________________________________________________________________
|
|
924
1007
|
|
|
925
1008
|
## Audio Corruption Benchmark
|
|
@@ -1287,7 +1370,7 @@ package-support jobs pass. Build artifacts are retained for one week. Tag pipeli
|
|
|
1287
1370
|
to complete a release.
|
|
1288
1371
|
|
|
1289
1372
|
Before tagging, update `pyproject.toml`, `src/justdata/__init__.py`, and `CHANGELOG.md`, then run `uv lock`. Push a tag
|
|
1290
|
-
matching the package version, with an optional `v` prefix (for example, `v1.
|
|
1373
|
+
matching the package version, with an optional `v` prefix (for example, `v1.5.0` or `1.5.0`). Publishing rejects tags
|
|
1291
1374
|
that disagree with either version declaration. Branch pipelines build artifacts without publishing.
|
|
1292
1375
|
|
|
1293
1376
|
Configure a [PyPI GitLab trusted publisher](https://docs.pypi.org/trusted-publishers/adding-a-publisher/) for the
|
|
@@ -1,46 +1,3 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: justdata
|
|
3
|
-
Version: 1.3.0
|
|
4
|
-
Summary: A TensorFlow-native data pipeline library with a modality-neutral core and first-class computer vision and acoustic recipes
|
|
5
|
-
Author: clementpoiret
|
|
6
|
-
Author-email: clementpoiret <clement@linux.com>
|
|
7
|
-
License-Expression: MIT
|
|
8
|
-
License-File: LICENSE
|
|
9
|
-
Requires-Dist: loguru>=0.7.3
|
|
10
|
-
Requires-Dist: tensorflow>=2.18.1
|
|
11
|
-
Requires-Dist: tensorflow-datasets>=4.9.9
|
|
12
|
-
Requires-Dist: datasets[audio]>=4.5.0 ; extra == 'acoustic'
|
|
13
|
-
Requires-Dist: soundfile>=0.12 ; extra == 'acoustic'
|
|
14
|
-
Requires-Dist: soxr>=0.5 ; extra == 'acoustic'
|
|
15
|
-
Requires-Dist: librosa>=0.11 ; extra == 'acoustic'
|
|
16
|
-
Requires-Dist: torch==2.10.0 ; extra == 'acoustic'
|
|
17
|
-
Requires-Dist: torchcodec==0.10.0 ; extra == 'acoustic'
|
|
18
|
-
Requires-Dist: datasets[audio]>=4.5.0 ; extra == 'dev'
|
|
19
|
-
Requires-Dist: datasets[vision]>=4.5.0 ; extra == 'dev'
|
|
20
|
-
Requires-Dist: filelock>=3.20 ; extra == 'dev'
|
|
21
|
-
Requires-Dist: librosa>=0.11 ; extra == 'dev'
|
|
22
|
-
Requires-Dist: matplotlib>=3.10.8 ; extra == 'dev'
|
|
23
|
-
Requires-Dist: pillow>=12.2.0 ; extra == 'dev'
|
|
24
|
-
Requires-Dist: pytest>=9.0.2 ; extra == 'dev'
|
|
25
|
-
Requires-Dist: ruff==0.15.20 ; extra == 'dev'
|
|
26
|
-
Requires-Dist: soundfile>=0.12 ; extra == 'dev'
|
|
27
|
-
Requires-Dist: soxr>=0.5 ; extra == 'dev'
|
|
28
|
-
Requires-Dist: torch==2.10.0 ; extra == 'dev'
|
|
29
|
-
Requires-Dist: torchcodec==0.10.0 ; extra == 'dev'
|
|
30
|
-
Requires-Dist: torchaudio==2.10.0 ; extra == 'golden'
|
|
31
|
-
Requires-Dist: torchlibrosa==0.1.0 ; extra == 'golden'
|
|
32
|
-
Requires-Dist: datasets[vision]>=4.5.0 ; extra == 'vision'
|
|
33
|
-
Requires-Dist: filelock>=3.20 ; extra == 'vision'
|
|
34
|
-
Requires-Dist: pillow>=12.2.0 ; extra == 'vision'
|
|
35
|
-
Requires-Dist: wilds>=2.0.0 ; extra == 'wilds'
|
|
36
|
-
Requires-Python: >=3.11, <3.14
|
|
37
|
-
Provides-Extra: acoustic
|
|
38
|
-
Provides-Extra: dev
|
|
39
|
-
Provides-Extra: golden
|
|
40
|
-
Provides-Extra: vision
|
|
41
|
-
Provides-Extra: wilds
|
|
42
|
-
Description-Content-Type: text/markdown
|
|
43
|
-
|
|
44
1
|
# justdata
|
|
45
2
|
|
|
46
3
|
A TensorFlow-native data pipeline library with a modality-neutral core and first-class computer vision and acoustic
|
|
@@ -124,6 +81,8 @@ enabled.
|
|
|
124
81
|
| `justdata.core.load_inventory(admitted, ...)` | Apply the shared pipeline to an admitted inventory, including raw epoch finalization. |
|
|
125
82
|
| `justdata.core.ExecutedConfig` | Immutable, canonical snapshot returned by loaders when `return_config=True`. |
|
|
126
83
|
| `justdata.vision.minic.create_minic_datasets(...)` | Constructs Mini-C corruption benchmark datasets from a shared preprocessed base dataset. |
|
|
84
|
+
| `justdata.vision.corruptions.apply_corruption_at(...)` | Renders a scene condition at a continuous strength and returns its record. |
|
|
85
|
+
| `justdata.vision.corruptions.apply_corruption_stack(...)` | Applies ordered conditions in float, rounds once, and returns the stack record. |
|
|
127
86
|
| `justdata.acoustic.corruptions.create_audio_corruption_datasets(...)` | Constructs acoustic corruption benchmark datasets from a shared preprocessed base dataset. |
|
|
128
87
|
| `justdata.acoustic.dcase2025.make_source_dataset(...)` | Builds DCASE Task 1 source-domain datasets with split-safety checks. |
|
|
129
88
|
|
|
@@ -963,6 +922,43 @@ severity; `none` removes metadata according to the shared core contract. The dat
|
|
|
963
922
|
enumeration-position seed salting. Consumers with their own sample-level PRNG lineage should call `apply_corruption`
|
|
964
923
|
directly.
|
|
965
924
|
|
|
925
|
+
### Scene conditions at continuous strength
|
|
926
|
+
|
|
927
|
+
Ten photometric conditions declare a continuous-strength contract: `haze`, `low_light`, `sun_glitter`, `lens_droplets`,
|
|
928
|
+
`rain`, `motion_blur`, `overexposure`, `white_balance_warm`, `white_balance_cool`, and `shot_noise`.
|
|
929
|
+
`apply_corruption_at` renders one of them at any strength between 0 (identity) and 5 (the last severity row). It takes
|
|
930
|
+
an optional episode seed for structure that persists across frames, and optional `depth_m`, `horizon`, or `region`
|
|
931
|
+
inputs. It returns the image and a record of the resolved parameters and mode. `apply_corruption_stack` applies ordered
|
|
932
|
+
steps in float and rounds once.
|
|
933
|
+
|
|
934
|
+
```python
|
|
935
|
+
import tensorflow as tf
|
|
936
|
+
|
|
937
|
+
from justdata.vision.corruptions import apply_corruption_at, apply_corruption_stack
|
|
938
|
+
|
|
939
|
+
frame = apply_corruption_at(
|
|
940
|
+
decoded_rgb_uint8,
|
|
941
|
+
name="lens_droplets",
|
|
942
|
+
version="1.0.0",
|
|
943
|
+
strength=1.7,
|
|
944
|
+
seed=tf.constant([frame_index, 0], tf.int32),
|
|
945
|
+
episode_seed=tf.constant([episode_index, 0], tf.int32),
|
|
946
|
+
)
|
|
947
|
+
dusk = apply_corruption_stack(
|
|
948
|
+
decoded_rgb_uint8,
|
|
949
|
+
steps=[("low_light", "1.0.0", 2.0), ("white_balance_cool", "1.0.0", 1.5)],
|
|
950
|
+
seed=tf.constant([frame_index, 1], tf.int32),
|
|
951
|
+
)
|
|
952
|
+
```
|
|
953
|
+
|
|
954
|
+
See [continuous strength](docs/vision.md#continuous-strength-episodes-and-scene-inputs) and
|
|
955
|
+
[scene condition corruptions](docs/vision.md#scene-condition-corruptions) for the contract, the tables, and their
|
|
956
|
+
sources, and `examples/vision/maritime_conditions.py` for a drift over an episode.
|
|
957
|
+
|
|
958
|
+
To add rare objects, `composite_instances` pastes annotated crops onto caller-chosen anchor pixels, updates the class
|
|
959
|
+
and instance maps, and records every request as placed or rejected; see
|
|
960
|
+
[compositing annotated instances](docs/vision.md#16-compositing-annotated-instances).
|
|
961
|
+
|
|
966
962
|
______________________________________________________________________
|
|
967
963
|
|
|
968
964
|
## Audio Corruption Benchmark
|
|
@@ -1330,7 +1326,7 @@ package-support jobs pass. Build artifacts are retained for one week. Tag pipeli
|
|
|
1330
1326
|
to complete a release.
|
|
1331
1327
|
|
|
1332
1328
|
Before tagging, update `pyproject.toml`, `src/justdata/__init__.py`, and `CHANGELOG.md`, then run `uv lock`. Push a tag
|
|
1333
|
-
matching the package version, with an optional `v` prefix (for example, `v1.
|
|
1329
|
+
matching the package version, with an optional `v` prefix (for example, `v1.5.0` or `1.5.0`). Publishing rejects tags
|
|
1334
1330
|
that disagree with either version declaration. Branch pipelines build artifacts without publishing.
|
|
1335
1331
|
|
|
1336
1332
|
Configure a [PyPI GitLab trusted publisher](https://docs.pypi.org/trusted-publishers/adding-a-publisher/) for the
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "justdata"
|
|
3
|
-
version = "1.
|
|
3
|
+
version = "1.5.0"
|
|
4
4
|
description = "A TensorFlow-native data pipeline library with a modality-neutral core and first-class computer vision and acoustic recipes"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "MIT"
|
|
@@ -33,6 +33,7 @@ acoustic = [
|
|
|
33
33
|
"torchcodec==0.10.0",
|
|
34
34
|
]
|
|
35
35
|
golden = [
|
|
36
|
+
"opencv-python-headless==5.0.0.93",
|
|
36
37
|
"torchaudio==2.10.0",
|
|
37
38
|
"torchlibrosa==0.1.0",
|
|
38
39
|
]
|
|
@@ -305,13 +305,15 @@ def finalize_dataset(
|
|
|
305
305
|
deterministic=deterministic,
|
|
306
306
|
)
|
|
307
307
|
elif rng is not None:
|
|
308
|
+
# Parallel calls draw from the shared generator in scheduling order,
|
|
309
|
+
# so batches would trade seeds between otherwise identical runs.
|
|
308
310
|
ds = ds.map(
|
|
309
311
|
lambda batch: late_augment_fn(
|
|
310
312
|
batch,
|
|
311
313
|
num_classes=num_classes,
|
|
312
314
|
seed=rng.make_seeds(1)[:, 0],
|
|
313
315
|
),
|
|
314
|
-
num_parallel_calls=parallel_calls,
|
|
316
|
+
num_parallel_calls=1 if deterministic else parallel_calls,
|
|
315
317
|
deterministic=deterministic,
|
|
316
318
|
)
|
|
317
319
|
|
|
@@ -478,6 +478,10 @@ def _prepare_ds(
|
|
|
478
478
|
):
|
|
479
479
|
"""Shared pipeline, resolving its source only after validating options."""
|
|
480
480
|
rng = tf.random.Generator.from_seed(seed)
|
|
481
|
+
# Late augmentation draws in a later tf.data stage that runs concurrently with
|
|
482
|
+
# the per-sample augment map; sharing one generator would interleave the two
|
|
483
|
+
# stages' draws differently on every run.
|
|
484
|
+
late_augment_rng = tf.random.Generator.from_seed(seed).split(1)[0]
|
|
481
485
|
if metadata_mode is None:
|
|
482
486
|
metadata_mode = (
|
|
483
487
|
pipeline.kwargs.get("metadata_mode", "full")
|
|
@@ -685,7 +689,7 @@ def _prepare_ds(
|
|
|
685
689
|
"is_training": is_training,
|
|
686
690
|
"apply_late_augment": apply_augmentation,
|
|
687
691
|
"late_augment_fn": late_augment_fn,
|
|
688
|
-
"rng":
|
|
692
|
+
"rng": late_augment_rng,
|
|
689
693
|
"deterministic": deterministic,
|
|
690
694
|
"shuffle_buffer": shuffle_buffer if is_training else None,
|
|
691
695
|
"shuffle_seed": seed,
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""Float32 RGB colour-space conversions on values in ``[0, 1]``.
|
|
2
|
+
|
|
3
|
+
sRGB follows IEC 61966-2-1 with the D65 white point; CIELAB uses the same
|
|
4
|
+
white, so ``lab_to_rgb(rgb_to_lab(x))`` returns ``x`` up to float32 rounding.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import tensorflow as tf
|
|
8
|
+
|
|
9
|
+
_RGB_TO_XYZ = (
|
|
10
|
+
(0.4124564, 0.3575761, 0.1804375),
|
|
11
|
+
(0.2126729, 0.7151522, 0.0721750),
|
|
12
|
+
(0.0193339, 0.1191920, 0.9503041),
|
|
13
|
+
)
|
|
14
|
+
_D65_WHITE_XYZ = (0.95047, 1.0, 1.08883)
|
|
15
|
+
_LAB_EPSILON = 216.0 / 24389.0
|
|
16
|
+
_LAB_KAPPA = 24389.0 / 27.0
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def srgb_to_linear(image: tf.Tensor) -> tf.Tensor:
|
|
20
|
+
image = tf.cast(image, tf.float32)
|
|
21
|
+
return tf.where(
|
|
22
|
+
image <= 0.04045,
|
|
23
|
+
image / 12.92,
|
|
24
|
+
tf.pow((tf.maximum(image, 0.04045) + 0.055) / 1.055, 2.4),
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def linear_to_srgb(image: tf.Tensor) -> tf.Tensor:
|
|
29
|
+
image = tf.maximum(tf.cast(image, tf.float32), 0.0)
|
|
30
|
+
return tf.where(
|
|
31
|
+
image <= 0.0031308,
|
|
32
|
+
image * 12.92,
|
|
33
|
+
1.055 * tf.pow(tf.maximum(image, 0.0031308), 1.0 / 2.4) - 0.055,
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def rgb_to_lab(image: tf.Tensor) -> tf.Tensor:
|
|
38
|
+
"""Convert sRGB ``[..., 3]`` to CIELAB with L in ``[0, 100]``."""
|
|
39
|
+
linear = srgb_to_linear(image)
|
|
40
|
+
xyz = tf.linalg.matvec(tf.constant(_RGB_TO_XYZ, tf.float32), linear)
|
|
41
|
+
ratio = xyz / tf.constant(_D65_WHITE_XYZ, tf.float32)
|
|
42
|
+
f = tf.where(
|
|
43
|
+
ratio > _LAB_EPSILON,
|
|
44
|
+
tf.pow(tf.maximum(ratio, _LAB_EPSILON), 1.0 / 3.0),
|
|
45
|
+
(_LAB_KAPPA * ratio + 16.0) / 116.0,
|
|
46
|
+
)
|
|
47
|
+
lightness = 116.0 * f[..., 1] - 16.0
|
|
48
|
+
a = 500.0 * (f[..., 0] - f[..., 1])
|
|
49
|
+
b = 200.0 * (f[..., 1] - f[..., 2])
|
|
50
|
+
return tf.stack([lightness, a, b], axis=-1)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def lab_to_rgb(lab: tf.Tensor) -> tf.Tensor:
|
|
54
|
+
"""Convert CIELAB ``[..., 3]`` to sRGB, clipped to ``[0, 1]``."""
|
|
55
|
+
lab = tf.cast(lab, tf.float32)
|
|
56
|
+
fy = (lab[..., 0] + 16.0) / 116.0
|
|
57
|
+
fx = fy + lab[..., 1] / 500.0
|
|
58
|
+
fz = fy - lab[..., 2] / 200.0
|
|
59
|
+
f = tf.stack([fx, fy, fz], axis=-1)
|
|
60
|
+
ratio = tf.where(
|
|
61
|
+
tf.pow(f, 3.0) > _LAB_EPSILON,
|
|
62
|
+
tf.pow(f, 3.0),
|
|
63
|
+
(116.0 * f - 16.0) / _LAB_KAPPA,
|
|
64
|
+
)
|
|
65
|
+
xyz = ratio * tf.constant(_D65_WHITE_XYZ, tf.float32)
|
|
66
|
+
xyz_to_rgb = tf.linalg.inv(tf.constant(_RGB_TO_XYZ, tf.float64))
|
|
67
|
+
linear = tf.linalg.matvec(tf.cast(xyz_to_rgb, tf.float32), xyz)
|
|
68
|
+
return tf.clip_by_value(linear_to_srgb(linear), 0.0, 1.0)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
__all__ = ["lab_to_rgb", "linear_to_srgb", "rgb_to_lab", "srgb_to_linear"]
|