justdata 1.2.0__tar.gz → 1.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (123) hide show
  1. {justdata-1.2.0 → justdata-1.3.0}/PKG-INFO +36 -8
  2. {justdata-1.2.0 → justdata-1.3.0}/README.md +35 -7
  3. {justdata-1.2.0 → justdata-1.3.0}/pyproject.toml +8 -1
  4. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/__init__.py +1 -1
  5. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/compat/ast.py +34 -15
  6. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/compat/efficientat.py +5 -1
  7. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/compat/passt.py +5 -1
  8. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/configs.py +24 -20
  9. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/evals.py +3 -3
  10. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/metadata.py +7 -7
  11. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/pipelines.py +15 -15
  12. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/presets.py +4 -1
  13. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/registry.py +3 -4
  14. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/resampling.py +0 -1
  15. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/stages.py +8 -3
  16. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/tasks.py +8 -2
  17. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/core/adapters.py +3 -1
  18. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/core/config_resolution.py +2 -2
  19. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/core/execution.py +5 -0
  20. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/core/finalization.py +1 -1
  21. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/core/inventory.py +3 -3
  22. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/core/loader.py +3 -3
  23. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/core/metadata.py +16 -5
  24. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/core/registry.py +6 -1
  25. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/core/stats.py +3 -3
  26. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/augmentations/composed.py +1 -1
  27. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/augmentations/registry.py +3 -2
  28. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/corruptions/registry.py +3 -2
  29. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/geometry.py +145 -29
  30. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/pipelines.py +10 -10
  31. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/sources.py +18 -13
  32. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/stages.py +2 -2
  33. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/tasks/classification.py +8 -8
  34. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/tasks/segmentation.py +2 -2
  35. {justdata-1.2.0 → justdata-1.3.0}/LICENSE +0 -0
  36. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/__init__.py +0 -0
  37. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/_dcase.py +0 -0
  38. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/_manifests.py +0 -0
  39. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/_random.py +0 -0
  40. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/_signal.py +0 -0
  41. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/adapters.py +0 -0
  42. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/augment/__init__.py +0 -0
  43. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/augment/_common.py +0 -0
  44. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/augment/batch.py +0 -0
  45. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/augment/configs.py +0 -0
  46. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/augment/patchout.py +0 -0
  47. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/augment/spectrogram.py +0 -0
  48. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/augment/waveform.py +0 -0
  49. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/batching.py +0 -0
  50. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/channel.py +0 -0
  51. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/compat/__init__.py +0 -0
  52. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/compat/ced.py +0 -0
  53. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/compat/panns.py +0 -0
  54. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/corruptions/__init__.py +0 -0
  55. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/corruptions/codec.py +0 -0
  56. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/corruptions/datasets.py +0 -0
  57. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/corruptions/device.py +0 -0
  58. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/corruptions/dynamics.py +0 -0
  59. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/corruptions/filters.py +0 -0
  60. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/corruptions/noise.py +0 -0
  61. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/corruptions/registry.py +0 -0
  62. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/corruptions/reverb.py +0 -0
  63. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/corruptions/timefreq.py +0 -0
  64. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/datasets.py +0 -0
  65. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/dcase2025.py +0 -0
  66. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/decoding.py +0 -0
  67. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/eval_views.py +0 -0
  68. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/filters.py +0 -0
  69. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/frontends/__init__.py +0 -0
  70. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/frontends/ast.py +0 -0
  71. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/frontends/kaldi.py +0 -0
  72. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/frontends/log.py +0 -0
  73. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/frontends/mel.py +0 -0
  74. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/frontends/mfcc.py +0 -0
  75. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/frontends/pcen.py +0 -0
  76. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/frontends/stft.py +0 -0
  77. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/jax.py +0 -0
  78. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/labels.py +0 -0
  79. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/layouts.py +0 -0
  80. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/normalization.py +0 -0
  81. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/postprocessing.py +0 -0
  82. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/preprocessing.py +0 -0
  83. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/schema.py +0 -0
  84. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/segment.py +0 -0
  85. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/sources.py +0 -0
  86. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/stats.py +0 -0
  87. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/audio/__init__.py +0 -0
  88. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/core/__init__.py +0 -0
  89. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/core/_records.py +0 -0
  90. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/core/augmentations.py +0 -0
  91. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/core/cache.py +0 -0
  92. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/core/executed_config.py +0 -0
  93. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/core/filters.py +0 -0
  94. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/core/label_mixing.py +0 -0
  95. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/core/presets.py +0 -0
  96. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/core/replay.py +0 -0
  97. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/core/sources.py +0 -0
  98. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/corruptions.py +0 -0
  99. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/py.typed +0 -0
  100. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/__init__.py +0 -0
  101. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/augmentations/__init__.py +0 -0
  102. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/augmentations/auto.py +0 -0
  103. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/augmentations/color.py +0 -0
  104. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/augmentations/geometric.py +0 -0
  105. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/augmentations/mixing.py +0 -0
  106. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/corruptions/__init__.py +0 -0
  107. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/corruptions/blur.py +0 -0
  108. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/corruptions/digital.py +0 -0
  109. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/corruptions/noise.py +0 -0
  110. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/corruptions/standard.py +0 -0
  111. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/corruptions/weather.py +0 -0
  112. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/datasets.py +0 -0
  113. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/encodings/__init__.py +0 -0
  114. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/encodings/heatmaps.py +0 -0
  115. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/encodings/panoptic_targets.py +0 -0
  116. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/encodings/semantic_targets.py +0 -0
  117. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/minic.py +0 -0
  118. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/presets.py +0 -0
  119. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/tasks/__init__.py +0 -0
  120. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/tasks/depth.py +0 -0
  121. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/tasks/panoptic.py +0 -0
  122. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/transforms.py +0 -0
  123. {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: justdata
3
- Version: 1.2.0
3
+ Version: 1.3.0
4
4
  Summary: A TensorFlow-native data pipeline library with a modality-neutral core and first-class computer vision and acoustic recipes
5
5
  Author: clementpoiret
6
6
  Author-email: clementpoiret <clement@linux.com>
@@ -546,7 +546,9 @@ For semantic segmentation with replayable geometry, select `vision/segmentation`
546
546
  `geometry_kwargs`. It supports paired aspect-preserving training resize and crops, longer-side evaluation caps,
547
547
  independent RGB/ignore padding, per-pixel validity, and original-coordinate score restoration. See
548
548
  [the dense geometry contract](docs/vision.md#12-replayable-dense-segmentation) for configuration, record fields, replay,
549
- and batching requirements.
549
+ and batching requirements. For ordered image/mask file lists without a registered source,
550
+ `justdata.vision.geometry.dense_views_from_files` builds the same views with explicit per-example seeds in a parallel,
551
+ order-preserving `tf.data` map.
550
552
 
551
553
  For instance-aware labels, select `vision/panoptic_segmentation` with a segment-ID map and category table. It preserves
552
554
  thing instances, merges stuff by category, and excludes crowd pixels from panoptic supervision. The
@@ -1094,14 +1096,36 @@ Replay prefetches after skipping committed batches and defaults to one batch.
1094
1096
  When another framework uses the accelerator, configure TensorFlow before creating tensors or datasets:
1095
1097
 
1096
1098
  ```python
1099
+ import jax
1100
+
1101
+ jax.devices() # initialize JAX's GPU backend before TensorFlow is imported
1102
+
1097
1103
  from justdata.core import configure_tensorflow_cpu
1098
1104
 
1099
1105
  device_report = configure_tensorflow_cpu(intra_op_threads=1, inter_op_threads=1)
1100
1106
  assert device_report["logical_gpus"] == []
1101
1107
  ```
1102
1108
 
1103
- The report includes physical and logical device names and process thread settings. The helper raises if the runtime has
1104
- initialized too early for the requested change. Core, vision, acoustic, and audio imports permit this setup.
1109
+ The report includes physical and logical device names and process thread settings. The helper raises if the TensorFlow
1110
+ runtime has initialized too early for the requested change. Core, vision, acoustic, and audio imports do not initialize
1111
+ it, so the helper can run after them.
1112
+
1113
+ Those imports do import TensorFlow. On Linux, the CUDA-enabled `tensorflow` wheel then exposes stubs for its CUDA
1114
+ libraries (CUDA 12 as of TensorFlow 2.21) to the whole process. A framework built for another CUDA version that
1115
+ initializes its GPU backend afterwards can bind to those stubs: JAX with its CUDA 13 plugin reports an outdated cuBLAS
1116
+ and falls back to the CPU. Hiding TensorFlow's GPUs does not prevent this. Either initialize the other framework's
1117
+ backend before the first TensorFlow import, as above (`import jax` alone is not enough), or install `tensorflow-cpu`,
1118
+ which exports no CUDA symbols, in place of `tensorflow`. With uv, on Linux x86_64:
1119
+
1120
+ ```toml
1121
+ [project]
1122
+ dependencies = ["justdata", "tensorflow-cpu"]
1123
+
1124
+ [tool.uv]
1125
+ exclude-dependencies = ["tensorflow"]
1126
+ ```
1127
+
1128
+ Set `JAX_PLATFORMS=cuda` to make a JAX fallback to the CPU an error rather than a warning.
1105
1129
 
1106
1130
  `CachePolicy` enables identity-checked file caches on the existing preprocessing and model-input cache stages. The cache
1107
1131
  path is a directory whose parent exists. The input identity must cover ordered content. `load_inventory` binds its
@@ -1282,6 +1306,7 @@ uv sync # install/update dependencies
1282
1306
  uv run pytest # run all tests
1283
1307
  uv run ruff check . # run lint checks
1284
1308
  uv run ruff format --check . # check formatting
1309
+ uv run ty check src/ # type check the package
1285
1310
  uv run mdformat --check --wrap 120 ./*.md benchmarks docs examples src tests # check Markdown formatting
1286
1311
  uv run pytest tests/path/to/test_file.py::test_name # run a single test
1287
1312
  ```
@@ -1290,9 +1315,12 @@ The project targets Python 3.12 (see `.python-version`). `LD_LIBRARY_PATH` is co
1290
1315
 
1291
1316
  See [Performance checks](benchmarks/README.md) for reproducible CPU benchmarks and the performance audit results.
1292
1317
 
1293
- GitLab runs `sh tests/doctor/test_lint.sh` in a pinned Debian-based uv image. This installs only the `lint` dependency
1294
- group and checks Python with Ruff and Markdown with mdformat using the GFM, footnote, and MyST plugins. The image
1295
- provides the standard Linux dynamic loader required by Ruff's PyPI executable, which the Nix image lacks.
1318
+ GitLab runs `sh tests/doctor/test_lint.sh` in a pinned Debian-based uv image. It checks Python with Ruff and Markdown
1319
+ with mdformat using the GFM, footnote, and MyST plugins from the `lint` dependency group alone, then type checks `src/`
1320
+ with ty against the locked development environment so that third-party imports resolve. TensorFlow attaches `Tensor`
1321
+ operators at runtime, so ty treats `tensorflow` imports as `Any` (see `[tool.ty.analysis]` in `pyproject.toml`). The
1322
+ image provides the standard Linux dynamic loader required by the Ruff and ty PyPI executables, which the Nix image
1323
+ lacks.
1296
1324
 
1297
1325
  ### Releases
1298
1326
 
@@ -1302,7 +1330,7 @@ package-support jobs pass. Build artifacts are retained for one week. Tag pipeli
1302
1330
  to complete a release.
1303
1331
 
1304
1332
  Before tagging, update `pyproject.toml`, `src/justdata/__init__.py`, and `CHANGELOG.md`, then run `uv lock`. Push a tag
1305
- matching the package version, with an optional `v` prefix (for example, `v1.2.0` or `1.2.0`). Publishing rejects tags
1333
+ matching the package version, with an optional `v` prefix (for example, `v1.3.0` or `1.3.0`). Publishing rejects tags
1306
1334
  that disagree with either version declaration. Branch pipelines build artifacts without publishing.
1307
1335
 
1308
1336
  Configure a [PyPI GitLab trusted publisher](https://docs.pypi.org/trusted-publishers/adding-a-publisher/) for the
@@ -503,7 +503,9 @@ For semantic segmentation with replayable geometry, select `vision/segmentation`
503
503
  `geometry_kwargs`. It supports paired aspect-preserving training resize and crops, longer-side evaluation caps,
504
504
  independent RGB/ignore padding, per-pixel validity, and original-coordinate score restoration. See
505
505
  [the dense geometry contract](docs/vision.md#12-replayable-dense-segmentation) for configuration, record fields, replay,
506
- and batching requirements.
506
+ and batching requirements. For ordered image/mask file lists without a registered source,
507
+ `justdata.vision.geometry.dense_views_from_files` builds the same views with explicit per-example seeds in a parallel,
508
+ order-preserving `tf.data` map.
507
509
 
508
510
  For instance-aware labels, select `vision/panoptic_segmentation` with a segment-ID map and category table. It preserves
509
511
  thing instances, merges stuff by category, and excludes crowd pixels from panoptic supervision. The
@@ -1051,14 +1053,36 @@ Replay prefetches after skipping committed batches and defaults to one batch.
1051
1053
  When another framework uses the accelerator, configure TensorFlow before creating tensors or datasets:
1052
1054
 
1053
1055
  ```python
1056
+ import jax
1057
+
1058
+ jax.devices() # initialize JAX's GPU backend before TensorFlow is imported
1059
+
1054
1060
  from justdata.core import configure_tensorflow_cpu
1055
1061
 
1056
1062
  device_report = configure_tensorflow_cpu(intra_op_threads=1, inter_op_threads=1)
1057
1063
  assert device_report["logical_gpus"] == []
1058
1064
  ```
1059
1065
 
1060
- The report includes physical and logical device names and process thread settings. The helper raises if the runtime has
1061
- initialized too early for the requested change. Core, vision, acoustic, and audio imports permit this setup.
1066
+ The report includes physical and logical device names and process thread settings. The helper raises if the TensorFlow
1067
+ runtime has initialized too early for the requested change. Core, vision, acoustic, and audio imports do not initialize
1068
+ it, so the helper can run after them.
1069
+
1070
+ Those imports do import TensorFlow. On Linux, the CUDA-enabled `tensorflow` wheel then exposes stubs for its CUDA
1071
+ libraries (CUDA 12 as of TensorFlow 2.21) to the whole process. A framework built for another CUDA version that
1072
+ initializes its GPU backend afterwards can bind to those stubs: JAX with its CUDA 13 plugin reports an outdated cuBLAS
1073
+ and falls back to the CPU. Hiding TensorFlow's GPUs does not prevent this. Either initialize the other framework's
1074
+ backend before the first TensorFlow import, as above (`import jax` alone is not enough), or install `tensorflow-cpu`,
1075
+ which exports no CUDA symbols, in place of `tensorflow`. With uv, on Linux x86_64:
1076
+
1077
+ ```toml
1078
+ [project]
1079
+ dependencies = ["justdata", "tensorflow-cpu"]
1080
+
1081
+ [tool.uv]
1082
+ exclude-dependencies = ["tensorflow"]
1083
+ ```
1084
+
1085
+ Set `JAX_PLATFORMS=cuda` to make a JAX fallback to the CPU an error rather than a warning.
1062
1086
 
1063
1087
  `CachePolicy` enables identity-checked file caches on the existing preprocessing and model-input cache stages. The cache
1064
1088
  path is a directory whose parent exists. The input identity must cover ordered content. `load_inventory` binds its
@@ -1239,6 +1263,7 @@ uv sync # install/update dependencies
1239
1263
  uv run pytest # run all tests
1240
1264
  uv run ruff check . # run lint checks
1241
1265
  uv run ruff format --check . # check formatting
1266
+ uv run ty check src/ # type check the package
1242
1267
  uv run mdformat --check --wrap 120 ./*.md benchmarks docs examples src tests # check Markdown formatting
1243
1268
  uv run pytest tests/path/to/test_file.py::test_name # run a single test
1244
1269
  ```
@@ -1247,9 +1272,12 @@ The project targets Python 3.12 (see `.python-version`). `LD_LIBRARY_PATH` is co
1247
1272
 
1248
1273
  See [Performance checks](benchmarks/README.md) for reproducible CPU benchmarks and the performance audit results.
1249
1274
 
1250
- GitLab runs `sh tests/doctor/test_lint.sh` in a pinned Debian-based uv image. This installs only the `lint` dependency
1251
- group and checks Python with Ruff and Markdown with mdformat using the GFM, footnote, and MyST plugins. The image
1252
- provides the standard Linux dynamic loader required by Ruff's PyPI executable, which the Nix image lacks.
1275
+ GitLab runs `sh tests/doctor/test_lint.sh` in a pinned Debian-based uv image. It checks Python with Ruff and Markdown
1276
+ with mdformat using the GFM, footnote, and MyST plugins from the `lint` dependency group alone, then type checks `src/`
1277
+ with ty against the locked development environment so that third-party imports resolve. TensorFlow attaches `Tensor`
1278
+ operators at runtime, so ty treats `tensorflow` imports as `Any` (see `[tool.ty.analysis]` in `pyproject.toml`). The
1279
+ image provides the standard Linux dynamic loader required by the Ruff and ty PyPI executables, which the Nix image
1280
+ lacks.
1253
1281
 
1254
1282
  ### Releases
1255
1283
 
@@ -1259,7 +1287,7 @@ package-support jobs pass. Build artifacts are retained for one week. Tag pipeli
1259
1287
  to complete a release.
1260
1288
 
1261
1289
  Before tagging, update `pyproject.toml`, `src/justdata/__init__.py`, and `CHANGELOG.md`, then run `uv lock`. Push a tag
1262
- matching the package version, with an optional `v` prefix (for example, `v1.2.0` or `1.2.0`). Publishing rejects tags
1290
+ matching the package version, with an optional `v` prefix (for example, `v1.3.0` or `1.3.0`). Publishing rejects tags
1263
1291
  that disagree with either version declaration. Branch pipelines build artifacts without publishing.
1264
1292
 
1265
1293
  Configure a [PyPI GitLab trusted publisher](https://docs.pypi.org/trusted-publishers/adding-a-publisher/) for the
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "justdata"
3
- version = "1.2.0"
3
+ version = "1.3.0"
4
4
  description = "A TensorFlow-native data pipeline library with a modality-neutral core and first-class computer vision and acoustic recipes"
5
5
  readme = "README.md"
6
6
  license = "MIT"
@@ -62,6 +62,7 @@ lint = [
62
62
  "mdformat-gfm==1.0.0",
63
63
  "mdformat-myst==0.3.0",
64
64
  "ruff==0.15.20",
65
+ "ty==0.0.83",
65
66
  ]
66
67
  dev = [
67
68
  { include-group = "lint" },
@@ -100,3 +101,9 @@ target-version = "py311"
100
101
 
101
102
  [tool.ruff.lint]
102
103
  select = ["E4", "E7", "E9", "F"]
104
+
105
+ [tool.ty.analysis]
106
+ # TensorFlow attaches Tensor operators and indexing at runtime, so its inferred types
107
+ # are unusable; the optional `wilds` extra is absent from the development environment.
108
+ replace-imports-with-any = ["tensorflow.**"]
109
+ allowed-unresolved-imports = ["wilds"]
@@ -1,3 +1,3 @@
1
- __version__ = "1.2.0"
1
+ __version__ = "1.3.0"
2
2
 
3
3
  __all__ = ["__version__"]
@@ -1,6 +1,6 @@
1
1
  from __future__ import annotations
2
2
 
3
- from typing import Any, Mapping
3
+ from typing import Any, Mapping, TypedDict
4
4
 
5
5
  import tensorflow as tf
6
6
 
@@ -16,7 +16,7 @@ from justdata.acoustic.configs import (
16
16
  SegmentStrategyConfig,
17
17
  )
18
18
  from justdata.acoustic.labels import transform_label
19
- from justdata.acoustic.layouts import convert_audio_layout
19
+ from justdata.acoustic.layouts import AudioLayout, convert_audio_layout
20
20
  from justdata.acoustic.preprocessing import make_preprocessing
21
21
  from justdata.acoustic.schema import FEATURES, LABEL, METADATA, SAMPLE_RATE, WAVEFORM
22
22
  from justdata.acoustic.segment import segment_waveform
@@ -28,7 +28,20 @@ AST_MIX_WAVEFORM = "ast_mix_waveform"
28
28
  AST_MIX_LABEL = "ast_mix_label"
29
29
  AST_MIX_LAMBDA = "ast_mix_lambda"
30
30
 
31
- AST_RECIPE_CONFIGS = {
31
+
32
+ class _ASTRecipe(TypedDict):
33
+ target_length: int
34
+ freqm: int
35
+ timem: int
36
+ mixup: float
37
+ mean: float
38
+ std: float
39
+ noise: bool
40
+ input_duration: float
41
+ num_classes: int
42
+
43
+
44
+ AST_RECIPE_CONFIGS: dict[str, _ASTRecipe] = {
32
45
  "audioset": {
33
46
  "target_length": 1024,
34
47
  "freqm": 48,
@@ -351,7 +364,9 @@ def _ast_config(
351
364
 
352
365
 
353
366
  def _target_length(
354
- layout: str, static_shape: tuple[int | None, ...] | None, ast: Mapping[str, Any]
367
+ layout: AudioLayout,
368
+ static_shape: tuple[int | None, ...] | None,
369
+ ast: Mapping[str, Any],
355
370
  ) -> int:
356
371
  if "target_length" in ast:
357
372
  return int(ast["target_length"])
@@ -359,15 +374,19 @@ def _target_length(
359
374
  raise ValueError(
360
375
  "AST pipeline requires static_shape or metadata['ast']['target_length']"
361
376
  )
362
- if layout == "btf":
363
- return int(static_shape[0])
364
- if layout == "bft":
365
- return int(static_shape[1])
366
- if layout == "bcft":
367
- return int(static_shape[2])
368
- if layout == "btfc":
369
- return int(static_shape[0])
370
- raise ValueError(f"Unsupported AST layout: {layout!r}")
377
+ if layout in ("btf", "btfc"):
378
+ target_length = static_shape[0]
379
+ elif layout == "bft":
380
+ target_length = static_shape[1]
381
+ elif layout == "bcft":
382
+ target_length = static_shape[2]
383
+ else:
384
+ raise ValueError(f"Unsupported AST layout: {layout!r}")
385
+ if target_length is None:
386
+ raise ValueError(
387
+ "AST pipeline requires a static time dimension in static_shape"
388
+ )
389
+ return int(target_length)
371
390
 
372
391
 
373
392
  def _transform_sample_label(
@@ -454,7 +473,7 @@ def _maybe_mix_waveform_and_label(
454
473
  def _make_ast_feature_stage(
455
474
  *,
456
475
  frontend: FrontendConfig,
457
- layout: str,
476
+ layout: AudioLayout,
458
477
  dtype: str,
459
478
  output_key: str,
460
479
  static_shape: tuple[int | None, ...] | None,
@@ -529,7 +548,7 @@ def ast_pipeline(
529
548
  preprocess: dict | None = None,
530
549
  segment: dict | None = None,
531
550
  frontend: dict | FrontendConfig | None = None,
532
- layout: str = "btf",
551
+ layout: AudioLayout = "btf",
533
552
  dtype: str = "float32",
534
553
  output_key: str | None = None,
535
554
  static_shape: tuple[int | None, ...] | None = None,
@@ -7,12 +7,14 @@ from justdata.acoustic._dcase import _dcase_label_transform
7
7
  from justdata.acoustic.configs import (
8
8
  AudioPreprocessConfig,
9
9
  AudioPreset,
10
+ DurationPolicy,
10
11
  FeatureNormConfig,
11
12
  FrontendConfig,
12
13
  LabelTransformConfig,
13
14
  LogCompressionConfig,
14
15
  MelConfig,
15
16
  STFTConfig,
17
+ SegmentPadMode,
16
18
  SegmentStrategyConfig,
17
19
  )
18
20
  from justdata.acoustic.frontends.log import compress_log
@@ -174,7 +176,9 @@ def dcase2025_task1_dymn_32k_1s() -> AudioPreset:
174
176
  )
175
177
 
176
178
 
177
- def _dcase_10s(name: str, *, duration_policy: str, pad_mode: str) -> AudioPreset:
179
+ def _dcase_10s(
180
+ name: str, *, duration_policy: DurationPolicy, pad_mode: SegmentPadMode
181
+ ) -> AudioPreset:
178
182
  metadata = _metadata("efficientat", source_duration=1.0)
179
183
  metadata["model_input_duration"] = 10.0
180
184
  return _preset(
@@ -5,12 +5,14 @@ from justdata.acoustic._dcase import _dcase_label_transform
5
5
  from justdata.acoustic.configs import (
6
6
  AudioPreprocessConfig,
7
7
  AudioPreset,
8
+ DurationPolicy,
8
9
  FeatureNormConfig,
9
10
  FrontendConfig,
10
11
  LabelTransformConfig,
11
12
  LogCompressionConfig,
12
13
  MelConfig,
13
14
  STFTConfig,
15
+ SegmentPadMode,
14
16
  SegmentStrategyConfig,
15
17
  )
16
18
 
@@ -137,7 +139,9 @@ def dcase2025_task1_passt_32k_1s() -> AudioPreset:
137
139
  )
138
140
 
139
141
 
140
- def _dcase_10s(name: str, *, duration_policy: str, pad_mode: str) -> AudioPreset:
142
+ def _dcase_10s(
143
+ name: str, *, duration_policy: DurationPolicy, pad_mode: SegmentPadMode
144
+ ) -> AudioPreset:
141
145
  metadata = _metadata(source_duration=1.0)
142
146
  metadata["model_input_duration"] = 10.0
143
147
  return _preset(
@@ -5,10 +5,19 @@ import hashlib
5
5
  import json
6
6
  from collections.abc import Mapping
7
7
  from dataclasses import dataclass, field
8
- from typing import Any, Literal, TypeVar
8
+ from typing import Any, ClassVar, Literal, Self
9
9
 
10
10
 
11
- ConfigT = TypeVar("ConfigT", bound="_SerializableConfig")
11
+ SegmentPadMode = Literal["zero", "repeat", "reflect"]
12
+ DurationPolicy = Literal[
13
+ "keep_1s",
14
+ "pad_to_model_duration",
15
+ "tile_to_model_duration",
16
+ "repeat_pad_to_model_duration",
17
+ "sliding_windows",
18
+ "none",
19
+ ]
20
+ LabelTransformMode = Literal["index", "one_hot", "multi_hot", "text", "event_frames"]
12
21
 
13
22
 
14
23
  def _canonical_json(config: Any) -> str:
@@ -48,6 +57,8 @@ def _tuple_or_scalar_value(value: Any) -> Any:
48
57
 
49
58
 
50
59
  class _SerializableConfig:
60
+ __dataclass_fields__: ClassVar[dict[str, dataclasses.Field[Any]]]
61
+
51
62
  def __post_init__(self) -> None:
52
63
  self.validate()
53
64
 
@@ -58,13 +69,15 @@ class _SerializableConfig:
58
69
  return _canonical_json(self)
59
70
 
60
71
  @classmethod
61
- def from_dict(cls: type[ConfigT], data: Mapping[str, Any] | ConfigT) -> ConfigT:
72
+ def from_dict(cls, data: Mapping[str, Any] | _SerializableConfig) -> Self:
62
73
  if isinstance(data, cls):
63
74
  return data
64
- return cls(**dict(data))
75
+ if not isinstance(data, Mapping):
76
+ raise TypeError(f"{cls.__name__} requires a mapping or {cls.__name__}")
77
+ return cls(**data)
65
78
 
66
79
  @classmethod
67
- def from_json(cls: type[ConfigT], data: str) -> ConfigT:
80
+ def from_json(cls, data: str) -> Self:
68
81
  return cls.from_dict(json.loads(data))
69
82
 
70
83
  def hash(self) -> str:
@@ -119,21 +132,14 @@ class SegmentStrategyConfig(_SerializableConfig):
119
132
  clip_duration: float
120
133
  train_mode: Literal["random_crop", "center_crop", "full", "sliding", "pad_or_crop"]
121
134
  eval_mode: Literal["center_crop", "full", "sliding", "multi_crop"]
122
- pad_mode: Literal["zero", "repeat", "reflect"]
135
+ pad_mode: SegmentPadMode
123
136
  pad_position: Literal["right", "center", "random"]
124
137
  num_views: int = 1
125
138
  sliding_hop_duration: float | None = None
126
139
  allow_train_sliding: bool = False
127
140
  drop_short: bool = False
128
141
  min_duration: float | None = None
129
- duration_policy: Literal[
130
- "keep_1s",
131
- "pad_to_model_duration",
132
- "tile_to_model_duration",
133
- "repeat_pad_to_model_duration",
134
- "sliding_windows",
135
- "none",
136
- ] = "none"
142
+ duration_policy: DurationPolicy = "none"
137
143
 
138
144
  def validate(self) -> SegmentStrategyConfig:
139
145
  _ensure_positive("clip_duration", self.clip_duration)
@@ -401,7 +407,7 @@ class FrontendConfig(_SerializableConfig):
401
407
 
402
408
  @dataclass(frozen=True)
403
409
  class LabelTransformConfig(_SerializableConfig):
404
- mode: Literal["index", "one_hot", "multi_hot", "text", "event_frames"]
410
+ mode: LabelTransformMode
405
411
  num_classes: int | None = None
406
412
  class_names: tuple[str, ...] | None = None
407
413
  smoothing: float = 0.0
@@ -480,12 +486,10 @@ class AudioPreset(_SerializableConfig):
480
486
  self.validate()
481
487
 
482
488
  @classmethod
483
- def from_dict(cls, data: Mapping[str, Any] | AudioPreset) -> AudioPreset:
484
- if isinstance(data, cls):
485
- return data
486
- if "target_sample_rate" not in data:
489
+ def from_dict(cls, data: Mapping[str, Any] | _SerializableConfig) -> Self:
490
+ if isinstance(data, Mapping) and "target_sample_rate" not in data:
487
491
  raise ValueError("AudioPreset requires target_sample_rate")
488
- return cls(**dict(data))
492
+ return super().from_dict(data)
489
493
 
490
494
  def validate(self) -> AudioPreset:
491
495
  if not self.name:
@@ -37,10 +37,10 @@ def macro_classwise_accuracy(y_true, y_pred, num_classes: int = 10) -> float:
37
37
  def _group_accuracy(y_true, y_pred, groups: Sequence[Any]) -> dict[Any, float]:
38
38
  true = _labels(y_true)
39
39
  pred = _predictions(y_pred)
40
- groups = np.asarray(groups)
40
+ group_values = np.asarray(groups)
41
41
  result = {}
42
- for group in sorted(set(groups.tolist()), key=str):
43
- mask = groups == group
42
+ for group in sorted(set(group_values.tolist()), key=str):
43
+ mask = group_values == group
44
44
  result[group] = float(np.mean(pred[mask] == true[mask]))
45
45
  return result
46
46
 
@@ -32,12 +32,12 @@ class MetadataEncoder:
32
32
  examples_or_vocab
33
33
  ):
34
34
  self.vocab = {
35
- field: {str(value): int(i) for i, value in enumerate(values)}
36
- for field, values in examples_or_vocab.items()
37
- if field in self._FIELD_KEYS
35
+ field: {
36
+ str(value): i
37
+ for i, value in enumerate(examples_or_vocab.get(field, ()))
38
+ }
39
+ for field in self._FIELD_KEYS
38
40
  }
39
- for field in self._FIELD_KEYS:
40
- self.vocab.setdefault(field, {})
41
41
  return self
42
42
 
43
43
  examples = (
@@ -99,12 +99,12 @@ class MetadataEncoder:
99
99
  return decoded
100
100
 
101
101
  @staticmethod
102
- def _looks_like_vocab(value: Mapping[str, Any]) -> bool:
102
+ def _looks_like_vocab(value: Mapping[Any, Any]) -> bool:
103
103
  return all(isinstance(v, (list, tuple, set)) for v in value.values())
104
104
 
105
105
  @staticmethod
106
106
  def _first_present(
107
- metadata: Mapping[str, Any], keys: tuple[str, ...]
107
+ metadata: Mapping[Any, Any], keys: tuple[str, ...]
108
108
  ) -> Any | None:
109
109
  for key in keys:
110
110
  if key in metadata:
@@ -460,25 +460,25 @@ def _resolve_ast_config(config, is_training):
460
460
 
461
461
 
462
462
  def default_pipeline(
463
- preproc_kwargs: dict = None,
464
- aug_kwargs: dict = None,
465
- laug_kwargs: dict = None,
466
- postproc_kwargs: dict = None,
467
- preprocess: dict = None,
468
- segment: dict = None,
469
- frontend: dict = None,
470
- layout: str = None,
463
+ preproc_kwargs: dict | None = None,
464
+ aug_kwargs: dict | None = None,
465
+ laug_kwargs: dict | None = None,
466
+ postproc_kwargs: dict | None = None,
467
+ preprocess: dict | None = None,
468
+ segment: dict | None = None,
469
+ frontend: dict | None = None,
470
+ layout: str | None = None,
471
471
  dtype: str = "float32",
472
- output_key: str = None,
473
- static_shape: tuple[int | None, ...] = None,
474
- input_duration: float = None,
475
- target_sample_rate: int = None,
476
- label_transform: dict = None,
477
- train_augment: dict = None,
472
+ output_key: str | None = None,
473
+ static_shape: tuple[int | None, ...] | None = None,
474
+ input_duration: float | None = None,
475
+ target_sample_rate: int | None = None,
476
+ label_transform: dict | None = None,
477
+ train_augment: dict | None = None,
478
478
  waveform_augmentations=None,
479
479
  spectrogram_augmentations=None,
480
480
  batch_augmentations=None,
481
- spectrogram_layout: str = None,
481
+ spectrogram_layout: str | None = None,
482
482
  augment_eval: bool = False,
483
483
  **kwargs,
484
484
  ) -> PipelineFuncs:
@@ -36,6 +36,7 @@ from justdata.acoustic.configs import (
36
36
  AudioPreset,
37
37
  FrontendConfig,
38
38
  LabelTransformConfig,
39
+ LabelTransformMode,
39
40
  LogCompressionConfig,
40
41
  MelConfig,
41
42
  STFTConfig,
@@ -93,7 +94,9 @@ def _preprocess(sample_rate: int) -> AudioPreprocessConfig:
93
94
  )
94
95
 
95
96
 
96
- def _label(mode: str, num_classes: int | None = None) -> LabelTransformConfig:
97
+ def _label(
98
+ mode: LabelTransformMode, num_classes: int | None = None
99
+ ) -> LabelTransformConfig:
97
100
  return LabelTransformConfig(mode=mode, num_classes=num_classes)
98
101
 
99
102
 
@@ -7,6 +7,7 @@ from justdata.core.augmentations import (
7
7
  AugmentationMetadata,
8
8
  attach_augmentation_metadata,
9
9
  )
10
+ from justdata.core.registry import qualified_name
10
11
 
11
12
 
12
13
  RegistryFn = TypeVar("RegistryFn", bound=Callable)
@@ -37,8 +38,7 @@ def _register(registry: dict[str, Callable], label: str, name: str):
37
38
  if name in registry:
38
39
  existing = registry[name]
39
40
  raise ValueError(
40
- f"{label} '{name}' already registered by "
41
- f"{existing.__module__}.{existing.__qualname__}"
41
+ f"{label} '{name}' already registered by {qualified_name(existing)}"
42
42
  )
43
43
  registry[name] = fn
44
44
  return fn
@@ -68,8 +68,7 @@ def _register_augment(
68
68
  if name in registry:
69
69
  existing = registry[name]
70
70
  raise ValueError(
71
- f"{label} '{name}' already registered by "
72
- f"{existing.__module__}.{existing.__qualname__}"
71
+ f"{label} '{name}' already registered by {qualified_name(existing)}"
73
72
  )
74
73
  registry[name] = fn
75
74
  metadata_registry[name] = metadata
@@ -53,7 +53,6 @@ def _soxr_resample_numpy(
53
53
  waveform.astype(np.float32, copy=False),
54
54
  int(original_sample_rate),
55
55
  int(target_sample_rate),
56
- axis=0,
57
56
  )
58
57
  return result.astype(np.float32, copy=False)
59
58
 
@@ -1,6 +1,7 @@
1
1
  from __future__ import annotations
2
2
 
3
- from collections.abc import Callable
3
+ from collections.abc import Callable, Mapping
4
+ from typing import Any, Protocol
4
5
 
5
6
  import tensorflow as tf
6
7
 
@@ -10,12 +11,16 @@ from justdata.acoustic.segment import segment_waveform
10
11
  from justdata.acoustic.schema import DURATION, METADATA, SAMPLE_RATE, WAVEFORM
11
12
 
12
13
 
14
+ class SeededSampleStage(Protocol):
15
+ def __call__(self, sample: dict, seed: tf.Tensor | int | None = None) -> dict: ...
16
+
17
+
13
18
  def make_segment_stage(
14
- config: SegmentStrategyConfig,
19
+ config: SegmentStrategyConfig | Mapping[str, Any],
15
20
  *,
16
21
  is_training: bool,
17
22
  audio_key: str = WAVEFORM,
18
- ) -> Callable[[dict, tf.Tensor | int | None], dict]:
23
+ ) -> SeededSampleStage:
19
24
  config = SegmentStrategyConfig.from_dict(config)
20
25
 
21
26
  def stage(sample: dict, seed: tf.Tensor | int | None = None) -> dict: