justdata 1.2.0__tar.gz → 1.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {justdata-1.2.0 → justdata-1.3.0}/PKG-INFO +36 -8
- {justdata-1.2.0 → justdata-1.3.0}/README.md +35 -7
- {justdata-1.2.0 → justdata-1.3.0}/pyproject.toml +8 -1
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/__init__.py +1 -1
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/compat/ast.py +34 -15
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/compat/efficientat.py +5 -1
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/compat/passt.py +5 -1
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/configs.py +24 -20
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/evals.py +3 -3
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/metadata.py +7 -7
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/pipelines.py +15 -15
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/presets.py +4 -1
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/registry.py +3 -4
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/resampling.py +0 -1
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/stages.py +8 -3
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/tasks.py +8 -2
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/core/adapters.py +3 -1
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/core/config_resolution.py +2 -2
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/core/execution.py +5 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/core/finalization.py +1 -1
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/core/inventory.py +3 -3
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/core/loader.py +3 -3
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/core/metadata.py +16 -5
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/core/registry.py +6 -1
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/core/stats.py +3 -3
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/augmentations/composed.py +1 -1
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/augmentations/registry.py +3 -2
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/corruptions/registry.py +3 -2
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/geometry.py +145 -29
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/pipelines.py +10 -10
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/sources.py +18 -13
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/stages.py +2 -2
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/tasks/classification.py +8 -8
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/tasks/segmentation.py +2 -2
- {justdata-1.2.0 → justdata-1.3.0}/LICENSE +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/__init__.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/_dcase.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/_manifests.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/_random.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/_signal.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/adapters.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/augment/__init__.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/augment/_common.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/augment/batch.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/augment/configs.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/augment/patchout.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/augment/spectrogram.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/augment/waveform.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/batching.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/channel.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/compat/__init__.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/compat/ced.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/compat/panns.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/corruptions/__init__.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/corruptions/codec.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/corruptions/datasets.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/corruptions/device.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/corruptions/dynamics.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/corruptions/filters.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/corruptions/noise.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/corruptions/registry.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/corruptions/reverb.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/corruptions/timefreq.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/datasets.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/dcase2025.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/decoding.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/eval_views.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/filters.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/frontends/__init__.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/frontends/ast.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/frontends/kaldi.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/frontends/log.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/frontends/mel.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/frontends/mfcc.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/frontends/pcen.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/frontends/stft.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/jax.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/labels.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/layouts.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/normalization.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/postprocessing.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/preprocessing.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/schema.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/segment.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/sources.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/acoustic/stats.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/audio/__init__.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/core/__init__.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/core/_records.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/core/augmentations.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/core/cache.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/core/executed_config.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/core/filters.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/core/label_mixing.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/core/presets.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/core/replay.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/core/sources.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/corruptions.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/py.typed +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/__init__.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/augmentations/__init__.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/augmentations/auto.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/augmentations/color.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/augmentations/geometric.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/augmentations/mixing.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/corruptions/__init__.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/corruptions/blur.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/corruptions/digital.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/corruptions/noise.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/corruptions/standard.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/corruptions/weather.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/datasets.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/encodings/__init__.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/encodings/heatmaps.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/encodings/panoptic_targets.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/encodings/semantic_targets.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/minic.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/presets.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/tasks/__init__.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/tasks/depth.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/tasks/panoptic.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/transforms.py +0 -0
- {justdata-1.2.0 → justdata-1.3.0}/src/justdata/vision/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: justdata
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.3.0
|
|
4
4
|
Summary: A TensorFlow-native data pipeline library with a modality-neutral core and first-class computer vision and acoustic recipes
|
|
5
5
|
Author: clementpoiret
|
|
6
6
|
Author-email: clementpoiret <clement@linux.com>
|
|
@@ -546,7 +546,9 @@ For semantic segmentation with replayable geometry, select `vision/segmentation`
|
|
|
546
546
|
`geometry_kwargs`. It supports paired aspect-preserving training resize and crops, longer-side evaluation caps,
|
|
547
547
|
independent RGB/ignore padding, per-pixel validity, and original-coordinate score restoration. See
|
|
548
548
|
[the dense geometry contract](docs/vision.md#12-replayable-dense-segmentation) for configuration, record fields, replay,
|
|
549
|
-
and batching requirements.
|
|
549
|
+
and batching requirements. For ordered image/mask file lists without a registered source,
|
|
550
|
+
`justdata.vision.geometry.dense_views_from_files` builds the same views with explicit per-example seeds in a parallel,
|
|
551
|
+
order-preserving `tf.data` map.
|
|
550
552
|
|
|
551
553
|
For instance-aware labels, select `vision/panoptic_segmentation` with a segment-ID map and category table. It preserves
|
|
552
554
|
thing instances, merges stuff by category, and excludes crowd pixels from panoptic supervision. The
|
|
@@ -1094,14 +1096,36 @@ Replay prefetches after skipping committed batches and defaults to one batch.
|
|
|
1094
1096
|
When another framework uses the accelerator, configure TensorFlow before creating tensors or datasets:
|
|
1095
1097
|
|
|
1096
1098
|
```python
|
|
1099
|
+
import jax
|
|
1100
|
+
|
|
1101
|
+
jax.devices() # initialize JAX's GPU backend before TensorFlow is imported
|
|
1102
|
+
|
|
1097
1103
|
from justdata.core import configure_tensorflow_cpu
|
|
1098
1104
|
|
|
1099
1105
|
device_report = configure_tensorflow_cpu(intra_op_threads=1, inter_op_threads=1)
|
|
1100
1106
|
assert device_report["logical_gpus"] == []
|
|
1101
1107
|
```
|
|
1102
1108
|
|
|
1103
|
-
The report includes physical and logical device names and process thread settings. The helper raises if the
|
|
1104
|
-
initialized too early for the requested change. Core, vision, acoustic, and audio imports
|
|
1109
|
+
The report includes physical and logical device names and process thread settings. The helper raises if the TensorFlow
|
|
1110
|
+
runtime has initialized too early for the requested change. Core, vision, acoustic, and audio imports do not initialize
|
|
1111
|
+
it, so the helper can run after them.
|
|
1112
|
+
|
|
1113
|
+
Those imports do import TensorFlow. On Linux, the CUDA-enabled `tensorflow` wheel then exposes stubs for its CUDA
|
|
1114
|
+
libraries (CUDA 12 as of TensorFlow 2.21) to the whole process. A framework built for another CUDA version that
|
|
1115
|
+
initializes its GPU backend afterwards can bind to those stubs: JAX with its CUDA 13 plugin reports an outdated cuBLAS
|
|
1116
|
+
and falls back to the CPU. Hiding TensorFlow's GPUs does not prevent this. Either initialize the other framework's
|
|
1117
|
+
backend before the first TensorFlow import, as above (`import jax` alone is not enough), or install `tensorflow-cpu`,
|
|
1118
|
+
which exports no CUDA symbols, in place of `tensorflow`. With uv, on Linux x86_64:
|
|
1119
|
+
|
|
1120
|
+
```toml
|
|
1121
|
+
[project]
|
|
1122
|
+
dependencies = ["justdata", "tensorflow-cpu"]
|
|
1123
|
+
|
|
1124
|
+
[tool.uv]
|
|
1125
|
+
exclude-dependencies = ["tensorflow"]
|
|
1126
|
+
```
|
|
1127
|
+
|
|
1128
|
+
Set `JAX_PLATFORMS=cuda` to make a JAX fallback to the CPU an error rather than a warning.
|
|
1105
1129
|
|
|
1106
1130
|
`CachePolicy` enables identity-checked file caches on the existing preprocessing and model-input cache stages. The cache
|
|
1107
1131
|
path is a directory whose parent exists. The input identity must cover ordered content. `load_inventory` binds its
|
|
@@ -1282,6 +1306,7 @@ uv sync # install/update dependencies
|
|
|
1282
1306
|
uv run pytest # run all tests
|
|
1283
1307
|
uv run ruff check . # run lint checks
|
|
1284
1308
|
uv run ruff format --check . # check formatting
|
|
1309
|
+
uv run ty check src/ # type check the package
|
|
1285
1310
|
uv run mdformat --check --wrap 120 ./*.md benchmarks docs examples src tests # check Markdown formatting
|
|
1286
1311
|
uv run pytest tests/path/to/test_file.py::test_name # run a single test
|
|
1287
1312
|
```
|
|
@@ -1290,9 +1315,12 @@ The project targets Python 3.12 (see `.python-version`). `LD_LIBRARY_PATH` is co
|
|
|
1290
1315
|
|
|
1291
1316
|
See [Performance checks](benchmarks/README.md) for reproducible CPU benchmarks and the performance audit results.
|
|
1292
1317
|
|
|
1293
|
-
GitLab runs `sh tests/doctor/test_lint.sh` in a pinned Debian-based uv image.
|
|
1294
|
-
|
|
1295
|
-
|
|
1318
|
+
GitLab runs `sh tests/doctor/test_lint.sh` in a pinned Debian-based uv image. It checks Python with Ruff and Markdown
|
|
1319
|
+
with mdformat using the GFM, footnote, and MyST plugins from the `lint` dependency group alone, then type checks `src/`
|
|
1320
|
+
with ty against the locked development environment so that third-party imports resolve. TensorFlow attaches `Tensor`
|
|
1321
|
+
operators at runtime, so ty treats `tensorflow` imports as `Any` (see `[tool.ty.analysis]` in `pyproject.toml`). The
|
|
1322
|
+
image provides the standard Linux dynamic loader required by the Ruff and ty PyPI executables, which the Nix image
|
|
1323
|
+
lacks.
|
|
1296
1324
|
|
|
1297
1325
|
### Releases
|
|
1298
1326
|
|
|
@@ -1302,7 +1330,7 @@ package-support jobs pass. Build artifacts are retained for one week. Tag pipeli
|
|
|
1302
1330
|
to complete a release.
|
|
1303
1331
|
|
|
1304
1332
|
Before tagging, update `pyproject.toml`, `src/justdata/__init__.py`, and `CHANGELOG.md`, then run `uv lock`. Push a tag
|
|
1305
|
-
matching the package version, with an optional `v` prefix (for example, `v1.
|
|
1333
|
+
matching the package version, with an optional `v` prefix (for example, `v1.3.0` or `1.3.0`). Publishing rejects tags
|
|
1306
1334
|
that disagree with either version declaration. Branch pipelines build artifacts without publishing.
|
|
1307
1335
|
|
|
1308
1336
|
Configure a [PyPI GitLab trusted publisher](https://docs.pypi.org/trusted-publishers/adding-a-publisher/) for the
|
|
@@ -503,7 +503,9 @@ For semantic segmentation with replayable geometry, select `vision/segmentation`
|
|
|
503
503
|
`geometry_kwargs`. It supports paired aspect-preserving training resize and crops, longer-side evaluation caps,
|
|
504
504
|
independent RGB/ignore padding, per-pixel validity, and original-coordinate score restoration. See
|
|
505
505
|
[the dense geometry contract](docs/vision.md#12-replayable-dense-segmentation) for configuration, record fields, replay,
|
|
506
|
-
and batching requirements.
|
|
506
|
+
and batching requirements. For ordered image/mask file lists without a registered source,
|
|
507
|
+
`justdata.vision.geometry.dense_views_from_files` builds the same views with explicit per-example seeds in a parallel,
|
|
508
|
+
order-preserving `tf.data` map.
|
|
507
509
|
|
|
508
510
|
For instance-aware labels, select `vision/panoptic_segmentation` with a segment-ID map and category table. It preserves
|
|
509
511
|
thing instances, merges stuff by category, and excludes crowd pixels from panoptic supervision. The
|
|
@@ -1051,14 +1053,36 @@ Replay prefetches after skipping committed batches and defaults to one batch.
|
|
|
1051
1053
|
When another framework uses the accelerator, configure TensorFlow before creating tensors or datasets:
|
|
1052
1054
|
|
|
1053
1055
|
```python
|
|
1056
|
+
import jax
|
|
1057
|
+
|
|
1058
|
+
jax.devices() # initialize JAX's GPU backend before TensorFlow is imported
|
|
1059
|
+
|
|
1054
1060
|
from justdata.core import configure_tensorflow_cpu
|
|
1055
1061
|
|
|
1056
1062
|
device_report = configure_tensorflow_cpu(intra_op_threads=1, inter_op_threads=1)
|
|
1057
1063
|
assert device_report["logical_gpus"] == []
|
|
1058
1064
|
```
|
|
1059
1065
|
|
|
1060
|
-
The report includes physical and logical device names and process thread settings. The helper raises if the
|
|
1061
|
-
initialized too early for the requested change. Core, vision, acoustic, and audio imports
|
|
1066
|
+
The report includes physical and logical device names and process thread settings. The helper raises if the TensorFlow
|
|
1067
|
+
runtime has initialized too early for the requested change. Core, vision, acoustic, and audio imports do not initialize
|
|
1068
|
+
it, so the helper can run after them.
|
|
1069
|
+
|
|
1070
|
+
Those imports do import TensorFlow. On Linux, the CUDA-enabled `tensorflow` wheel then exposes stubs for its CUDA
|
|
1071
|
+
libraries (CUDA 12 as of TensorFlow 2.21) to the whole process. A framework built for another CUDA version that
|
|
1072
|
+
initializes its GPU backend afterwards can bind to those stubs: JAX with its CUDA 13 plugin reports an outdated cuBLAS
|
|
1073
|
+
and falls back to the CPU. Hiding TensorFlow's GPUs does not prevent this. Either initialize the other framework's
|
|
1074
|
+
backend before the first TensorFlow import, as above (`import jax` alone is not enough), or install `tensorflow-cpu`,
|
|
1075
|
+
which exports no CUDA symbols, in place of `tensorflow`. With uv, on Linux x86_64:
|
|
1076
|
+
|
|
1077
|
+
```toml
|
|
1078
|
+
[project]
|
|
1079
|
+
dependencies = ["justdata", "tensorflow-cpu"]
|
|
1080
|
+
|
|
1081
|
+
[tool.uv]
|
|
1082
|
+
exclude-dependencies = ["tensorflow"]
|
|
1083
|
+
```
|
|
1084
|
+
|
|
1085
|
+
Set `JAX_PLATFORMS=cuda` to make a JAX fallback to the CPU an error rather than a warning.
|
|
1062
1086
|
|
|
1063
1087
|
`CachePolicy` enables identity-checked file caches on the existing preprocessing and model-input cache stages. The cache
|
|
1064
1088
|
path is a directory whose parent exists. The input identity must cover ordered content. `load_inventory` binds its
|
|
@@ -1239,6 +1263,7 @@ uv sync # install/update dependencies
|
|
|
1239
1263
|
uv run pytest # run all tests
|
|
1240
1264
|
uv run ruff check . # run lint checks
|
|
1241
1265
|
uv run ruff format --check . # check formatting
|
|
1266
|
+
uv run ty check src/ # type check the package
|
|
1242
1267
|
uv run mdformat --check --wrap 120 ./*.md benchmarks docs examples src tests # check Markdown formatting
|
|
1243
1268
|
uv run pytest tests/path/to/test_file.py::test_name # run a single test
|
|
1244
1269
|
```
|
|
@@ -1247,9 +1272,12 @@ The project targets Python 3.12 (see `.python-version`). `LD_LIBRARY_PATH` is co
|
|
|
1247
1272
|
|
|
1248
1273
|
See [Performance checks](benchmarks/README.md) for reproducible CPU benchmarks and the performance audit results.
|
|
1249
1274
|
|
|
1250
|
-
GitLab runs `sh tests/doctor/test_lint.sh` in a pinned Debian-based uv image.
|
|
1251
|
-
|
|
1252
|
-
|
|
1275
|
+
GitLab runs `sh tests/doctor/test_lint.sh` in a pinned Debian-based uv image. It checks Python with Ruff and Markdown
|
|
1276
|
+
with mdformat using the GFM, footnote, and MyST plugins from the `lint` dependency group alone, then type checks `src/`
|
|
1277
|
+
with ty against the locked development environment so that third-party imports resolve. TensorFlow attaches `Tensor`
|
|
1278
|
+
operators at runtime, so ty treats `tensorflow` imports as `Any` (see `[tool.ty.analysis]` in `pyproject.toml`). The
|
|
1279
|
+
image provides the standard Linux dynamic loader required by the Ruff and ty PyPI executables, which the Nix image
|
|
1280
|
+
lacks.
|
|
1253
1281
|
|
|
1254
1282
|
### Releases
|
|
1255
1283
|
|
|
@@ -1259,7 +1287,7 @@ package-support jobs pass. Build artifacts are retained for one week. Tag pipeli
|
|
|
1259
1287
|
to complete a release.
|
|
1260
1288
|
|
|
1261
1289
|
Before tagging, update `pyproject.toml`, `src/justdata/__init__.py`, and `CHANGELOG.md`, then run `uv lock`. Push a tag
|
|
1262
|
-
matching the package version, with an optional `v` prefix (for example, `v1.
|
|
1290
|
+
matching the package version, with an optional `v` prefix (for example, `v1.3.0` or `1.3.0`). Publishing rejects tags
|
|
1263
1291
|
that disagree with either version declaration. Branch pipelines build artifacts without publishing.
|
|
1264
1292
|
|
|
1265
1293
|
Configure a [PyPI GitLab trusted publisher](https://docs.pypi.org/trusted-publishers/adding-a-publisher/) for the
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "justdata"
|
|
3
|
-
version = "1.
|
|
3
|
+
version = "1.3.0"
|
|
4
4
|
description = "A TensorFlow-native data pipeline library with a modality-neutral core and first-class computer vision and acoustic recipes"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "MIT"
|
|
@@ -62,6 +62,7 @@ lint = [
|
|
|
62
62
|
"mdformat-gfm==1.0.0",
|
|
63
63
|
"mdformat-myst==0.3.0",
|
|
64
64
|
"ruff==0.15.20",
|
|
65
|
+
"ty==0.0.83",
|
|
65
66
|
]
|
|
66
67
|
dev = [
|
|
67
68
|
{ include-group = "lint" },
|
|
@@ -100,3 +101,9 @@ target-version = "py311"
|
|
|
100
101
|
|
|
101
102
|
[tool.ruff.lint]
|
|
102
103
|
select = ["E4", "E7", "E9", "F"]
|
|
104
|
+
|
|
105
|
+
[tool.ty.analysis]
|
|
106
|
+
# TensorFlow attaches Tensor operators and indexing at runtime, so its inferred types
|
|
107
|
+
# are unusable; the optional `wilds` extra is absent from the development environment.
|
|
108
|
+
replace-imports-with-any = ["tensorflow.**"]
|
|
109
|
+
allowed-unresolved-imports = ["wilds"]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
from typing import Any, Mapping
|
|
3
|
+
from typing import Any, Mapping, TypedDict
|
|
4
4
|
|
|
5
5
|
import tensorflow as tf
|
|
6
6
|
|
|
@@ -16,7 +16,7 @@ from justdata.acoustic.configs import (
|
|
|
16
16
|
SegmentStrategyConfig,
|
|
17
17
|
)
|
|
18
18
|
from justdata.acoustic.labels import transform_label
|
|
19
|
-
from justdata.acoustic.layouts import convert_audio_layout
|
|
19
|
+
from justdata.acoustic.layouts import AudioLayout, convert_audio_layout
|
|
20
20
|
from justdata.acoustic.preprocessing import make_preprocessing
|
|
21
21
|
from justdata.acoustic.schema import FEATURES, LABEL, METADATA, SAMPLE_RATE, WAVEFORM
|
|
22
22
|
from justdata.acoustic.segment import segment_waveform
|
|
@@ -28,7 +28,20 @@ AST_MIX_WAVEFORM = "ast_mix_waveform"
|
|
|
28
28
|
AST_MIX_LABEL = "ast_mix_label"
|
|
29
29
|
AST_MIX_LAMBDA = "ast_mix_lambda"
|
|
30
30
|
|
|
31
|
-
|
|
31
|
+
|
|
32
|
+
class _ASTRecipe(TypedDict):
|
|
33
|
+
target_length: int
|
|
34
|
+
freqm: int
|
|
35
|
+
timem: int
|
|
36
|
+
mixup: float
|
|
37
|
+
mean: float
|
|
38
|
+
std: float
|
|
39
|
+
noise: bool
|
|
40
|
+
input_duration: float
|
|
41
|
+
num_classes: int
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
AST_RECIPE_CONFIGS: dict[str, _ASTRecipe] = {
|
|
32
45
|
"audioset": {
|
|
33
46
|
"target_length": 1024,
|
|
34
47
|
"freqm": 48,
|
|
@@ -351,7 +364,9 @@ def _ast_config(
|
|
|
351
364
|
|
|
352
365
|
|
|
353
366
|
def _target_length(
|
|
354
|
-
layout:
|
|
367
|
+
layout: AudioLayout,
|
|
368
|
+
static_shape: tuple[int | None, ...] | None,
|
|
369
|
+
ast: Mapping[str, Any],
|
|
355
370
|
) -> int:
|
|
356
371
|
if "target_length" in ast:
|
|
357
372
|
return int(ast["target_length"])
|
|
@@ -359,15 +374,19 @@ def _target_length(
|
|
|
359
374
|
raise ValueError(
|
|
360
375
|
"AST pipeline requires static_shape or metadata['ast']['target_length']"
|
|
361
376
|
)
|
|
362
|
-
if layout
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
377
|
+
if layout in ("btf", "btfc"):
|
|
378
|
+
target_length = static_shape[0]
|
|
379
|
+
elif layout == "bft":
|
|
380
|
+
target_length = static_shape[1]
|
|
381
|
+
elif layout == "bcft":
|
|
382
|
+
target_length = static_shape[2]
|
|
383
|
+
else:
|
|
384
|
+
raise ValueError(f"Unsupported AST layout: {layout!r}")
|
|
385
|
+
if target_length is None:
|
|
386
|
+
raise ValueError(
|
|
387
|
+
"AST pipeline requires a static time dimension in static_shape"
|
|
388
|
+
)
|
|
389
|
+
return int(target_length)
|
|
371
390
|
|
|
372
391
|
|
|
373
392
|
def _transform_sample_label(
|
|
@@ -454,7 +473,7 @@ def _maybe_mix_waveform_and_label(
|
|
|
454
473
|
def _make_ast_feature_stage(
|
|
455
474
|
*,
|
|
456
475
|
frontend: FrontendConfig,
|
|
457
|
-
layout:
|
|
476
|
+
layout: AudioLayout,
|
|
458
477
|
dtype: str,
|
|
459
478
|
output_key: str,
|
|
460
479
|
static_shape: tuple[int | None, ...] | None,
|
|
@@ -529,7 +548,7 @@ def ast_pipeline(
|
|
|
529
548
|
preprocess: dict | None = None,
|
|
530
549
|
segment: dict | None = None,
|
|
531
550
|
frontend: dict | FrontendConfig | None = None,
|
|
532
|
-
layout:
|
|
551
|
+
layout: AudioLayout = "btf",
|
|
533
552
|
dtype: str = "float32",
|
|
534
553
|
output_key: str | None = None,
|
|
535
554
|
static_shape: tuple[int | None, ...] | None = None,
|
|
@@ -7,12 +7,14 @@ from justdata.acoustic._dcase import _dcase_label_transform
|
|
|
7
7
|
from justdata.acoustic.configs import (
|
|
8
8
|
AudioPreprocessConfig,
|
|
9
9
|
AudioPreset,
|
|
10
|
+
DurationPolicy,
|
|
10
11
|
FeatureNormConfig,
|
|
11
12
|
FrontendConfig,
|
|
12
13
|
LabelTransformConfig,
|
|
13
14
|
LogCompressionConfig,
|
|
14
15
|
MelConfig,
|
|
15
16
|
STFTConfig,
|
|
17
|
+
SegmentPadMode,
|
|
16
18
|
SegmentStrategyConfig,
|
|
17
19
|
)
|
|
18
20
|
from justdata.acoustic.frontends.log import compress_log
|
|
@@ -174,7 +176,9 @@ def dcase2025_task1_dymn_32k_1s() -> AudioPreset:
|
|
|
174
176
|
)
|
|
175
177
|
|
|
176
178
|
|
|
177
|
-
def _dcase_10s(
|
|
179
|
+
def _dcase_10s(
|
|
180
|
+
name: str, *, duration_policy: DurationPolicy, pad_mode: SegmentPadMode
|
|
181
|
+
) -> AudioPreset:
|
|
178
182
|
metadata = _metadata("efficientat", source_duration=1.0)
|
|
179
183
|
metadata["model_input_duration"] = 10.0
|
|
180
184
|
return _preset(
|
|
@@ -5,12 +5,14 @@ from justdata.acoustic._dcase import _dcase_label_transform
|
|
|
5
5
|
from justdata.acoustic.configs import (
|
|
6
6
|
AudioPreprocessConfig,
|
|
7
7
|
AudioPreset,
|
|
8
|
+
DurationPolicy,
|
|
8
9
|
FeatureNormConfig,
|
|
9
10
|
FrontendConfig,
|
|
10
11
|
LabelTransformConfig,
|
|
11
12
|
LogCompressionConfig,
|
|
12
13
|
MelConfig,
|
|
13
14
|
STFTConfig,
|
|
15
|
+
SegmentPadMode,
|
|
14
16
|
SegmentStrategyConfig,
|
|
15
17
|
)
|
|
16
18
|
|
|
@@ -137,7 +139,9 @@ def dcase2025_task1_passt_32k_1s() -> AudioPreset:
|
|
|
137
139
|
)
|
|
138
140
|
|
|
139
141
|
|
|
140
|
-
def _dcase_10s(
|
|
142
|
+
def _dcase_10s(
|
|
143
|
+
name: str, *, duration_policy: DurationPolicy, pad_mode: SegmentPadMode
|
|
144
|
+
) -> AudioPreset:
|
|
141
145
|
metadata = _metadata(source_duration=1.0)
|
|
142
146
|
metadata["model_input_duration"] = 10.0
|
|
143
147
|
return _preset(
|
|
@@ -5,10 +5,19 @@ import hashlib
|
|
|
5
5
|
import json
|
|
6
6
|
from collections.abc import Mapping
|
|
7
7
|
from dataclasses import dataclass, field
|
|
8
|
-
from typing import Any, Literal,
|
|
8
|
+
from typing import Any, ClassVar, Literal, Self
|
|
9
9
|
|
|
10
10
|
|
|
11
|
-
|
|
11
|
+
SegmentPadMode = Literal["zero", "repeat", "reflect"]
|
|
12
|
+
DurationPolicy = Literal[
|
|
13
|
+
"keep_1s",
|
|
14
|
+
"pad_to_model_duration",
|
|
15
|
+
"tile_to_model_duration",
|
|
16
|
+
"repeat_pad_to_model_duration",
|
|
17
|
+
"sliding_windows",
|
|
18
|
+
"none",
|
|
19
|
+
]
|
|
20
|
+
LabelTransformMode = Literal["index", "one_hot", "multi_hot", "text", "event_frames"]
|
|
12
21
|
|
|
13
22
|
|
|
14
23
|
def _canonical_json(config: Any) -> str:
|
|
@@ -48,6 +57,8 @@ def _tuple_or_scalar_value(value: Any) -> Any:
|
|
|
48
57
|
|
|
49
58
|
|
|
50
59
|
class _SerializableConfig:
|
|
60
|
+
__dataclass_fields__: ClassVar[dict[str, dataclasses.Field[Any]]]
|
|
61
|
+
|
|
51
62
|
def __post_init__(self) -> None:
|
|
52
63
|
self.validate()
|
|
53
64
|
|
|
@@ -58,13 +69,15 @@ class _SerializableConfig:
|
|
|
58
69
|
return _canonical_json(self)
|
|
59
70
|
|
|
60
71
|
@classmethod
|
|
61
|
-
def from_dict(cls
|
|
72
|
+
def from_dict(cls, data: Mapping[str, Any] | _SerializableConfig) -> Self:
|
|
62
73
|
if isinstance(data, cls):
|
|
63
74
|
return data
|
|
64
|
-
|
|
75
|
+
if not isinstance(data, Mapping):
|
|
76
|
+
raise TypeError(f"{cls.__name__} requires a mapping or {cls.__name__}")
|
|
77
|
+
return cls(**data)
|
|
65
78
|
|
|
66
79
|
@classmethod
|
|
67
|
-
def from_json(cls
|
|
80
|
+
def from_json(cls, data: str) -> Self:
|
|
68
81
|
return cls.from_dict(json.loads(data))
|
|
69
82
|
|
|
70
83
|
def hash(self) -> str:
|
|
@@ -119,21 +132,14 @@ class SegmentStrategyConfig(_SerializableConfig):
|
|
|
119
132
|
clip_duration: float
|
|
120
133
|
train_mode: Literal["random_crop", "center_crop", "full", "sliding", "pad_or_crop"]
|
|
121
134
|
eval_mode: Literal["center_crop", "full", "sliding", "multi_crop"]
|
|
122
|
-
pad_mode:
|
|
135
|
+
pad_mode: SegmentPadMode
|
|
123
136
|
pad_position: Literal["right", "center", "random"]
|
|
124
137
|
num_views: int = 1
|
|
125
138
|
sliding_hop_duration: float | None = None
|
|
126
139
|
allow_train_sliding: bool = False
|
|
127
140
|
drop_short: bool = False
|
|
128
141
|
min_duration: float | None = None
|
|
129
|
-
duration_policy:
|
|
130
|
-
"keep_1s",
|
|
131
|
-
"pad_to_model_duration",
|
|
132
|
-
"tile_to_model_duration",
|
|
133
|
-
"repeat_pad_to_model_duration",
|
|
134
|
-
"sliding_windows",
|
|
135
|
-
"none",
|
|
136
|
-
] = "none"
|
|
142
|
+
duration_policy: DurationPolicy = "none"
|
|
137
143
|
|
|
138
144
|
def validate(self) -> SegmentStrategyConfig:
|
|
139
145
|
_ensure_positive("clip_duration", self.clip_duration)
|
|
@@ -401,7 +407,7 @@ class FrontendConfig(_SerializableConfig):
|
|
|
401
407
|
|
|
402
408
|
@dataclass(frozen=True)
|
|
403
409
|
class LabelTransformConfig(_SerializableConfig):
|
|
404
|
-
mode:
|
|
410
|
+
mode: LabelTransformMode
|
|
405
411
|
num_classes: int | None = None
|
|
406
412
|
class_names: tuple[str, ...] | None = None
|
|
407
413
|
smoothing: float = 0.0
|
|
@@ -480,12 +486,10 @@ class AudioPreset(_SerializableConfig):
|
|
|
480
486
|
self.validate()
|
|
481
487
|
|
|
482
488
|
@classmethod
|
|
483
|
-
def from_dict(cls, data: Mapping[str, Any] |
|
|
484
|
-
if isinstance(data,
|
|
485
|
-
return data
|
|
486
|
-
if "target_sample_rate" not in data:
|
|
489
|
+
def from_dict(cls, data: Mapping[str, Any] | _SerializableConfig) -> Self:
|
|
490
|
+
if isinstance(data, Mapping) and "target_sample_rate" not in data:
|
|
487
491
|
raise ValueError("AudioPreset requires target_sample_rate")
|
|
488
|
-
return
|
|
492
|
+
return super().from_dict(data)
|
|
489
493
|
|
|
490
494
|
def validate(self) -> AudioPreset:
|
|
491
495
|
if not self.name:
|
|
@@ -37,10 +37,10 @@ def macro_classwise_accuracy(y_true, y_pred, num_classes: int = 10) -> float:
|
|
|
37
37
|
def _group_accuracy(y_true, y_pred, groups: Sequence[Any]) -> dict[Any, float]:
|
|
38
38
|
true = _labels(y_true)
|
|
39
39
|
pred = _predictions(y_pred)
|
|
40
|
-
|
|
40
|
+
group_values = np.asarray(groups)
|
|
41
41
|
result = {}
|
|
42
|
-
for group in sorted(set(
|
|
43
|
-
mask =
|
|
42
|
+
for group in sorted(set(group_values.tolist()), key=str):
|
|
43
|
+
mask = group_values == group
|
|
44
44
|
result[group] = float(np.mean(pred[mask] == true[mask]))
|
|
45
45
|
return result
|
|
46
46
|
|
|
@@ -32,12 +32,12 @@ class MetadataEncoder:
|
|
|
32
32
|
examples_or_vocab
|
|
33
33
|
):
|
|
34
34
|
self.vocab = {
|
|
35
|
-
field: {
|
|
36
|
-
|
|
37
|
-
|
|
35
|
+
field: {
|
|
36
|
+
str(value): i
|
|
37
|
+
for i, value in enumerate(examples_or_vocab.get(field, ()))
|
|
38
|
+
}
|
|
39
|
+
for field in self._FIELD_KEYS
|
|
38
40
|
}
|
|
39
|
-
for field in self._FIELD_KEYS:
|
|
40
|
-
self.vocab.setdefault(field, {})
|
|
41
41
|
return self
|
|
42
42
|
|
|
43
43
|
examples = (
|
|
@@ -99,12 +99,12 @@ class MetadataEncoder:
|
|
|
99
99
|
return decoded
|
|
100
100
|
|
|
101
101
|
@staticmethod
|
|
102
|
-
def _looks_like_vocab(value: Mapping[
|
|
102
|
+
def _looks_like_vocab(value: Mapping[Any, Any]) -> bool:
|
|
103
103
|
return all(isinstance(v, (list, tuple, set)) for v in value.values())
|
|
104
104
|
|
|
105
105
|
@staticmethod
|
|
106
106
|
def _first_present(
|
|
107
|
-
metadata: Mapping[
|
|
107
|
+
metadata: Mapping[Any, Any], keys: tuple[str, ...]
|
|
108
108
|
) -> Any | None:
|
|
109
109
|
for key in keys:
|
|
110
110
|
if key in metadata:
|
|
@@ -460,25 +460,25 @@ def _resolve_ast_config(config, is_training):
|
|
|
460
460
|
|
|
461
461
|
|
|
462
462
|
def default_pipeline(
|
|
463
|
-
preproc_kwargs: dict = None,
|
|
464
|
-
aug_kwargs: dict = None,
|
|
465
|
-
laug_kwargs: dict = None,
|
|
466
|
-
postproc_kwargs: dict = None,
|
|
467
|
-
preprocess: dict = None,
|
|
468
|
-
segment: dict = None,
|
|
469
|
-
frontend: dict = None,
|
|
470
|
-
layout: str = None,
|
|
463
|
+
preproc_kwargs: dict | None = None,
|
|
464
|
+
aug_kwargs: dict | None = None,
|
|
465
|
+
laug_kwargs: dict | None = None,
|
|
466
|
+
postproc_kwargs: dict | None = None,
|
|
467
|
+
preprocess: dict | None = None,
|
|
468
|
+
segment: dict | None = None,
|
|
469
|
+
frontend: dict | None = None,
|
|
470
|
+
layout: str | None = None,
|
|
471
471
|
dtype: str = "float32",
|
|
472
|
-
output_key: str = None,
|
|
473
|
-
static_shape: tuple[int | None, ...] = None,
|
|
474
|
-
input_duration: float = None,
|
|
475
|
-
target_sample_rate: int = None,
|
|
476
|
-
label_transform: dict = None,
|
|
477
|
-
train_augment: dict = None,
|
|
472
|
+
output_key: str | None = None,
|
|
473
|
+
static_shape: tuple[int | None, ...] | None = None,
|
|
474
|
+
input_duration: float | None = None,
|
|
475
|
+
target_sample_rate: int | None = None,
|
|
476
|
+
label_transform: dict | None = None,
|
|
477
|
+
train_augment: dict | None = None,
|
|
478
478
|
waveform_augmentations=None,
|
|
479
479
|
spectrogram_augmentations=None,
|
|
480
480
|
batch_augmentations=None,
|
|
481
|
-
spectrogram_layout: str = None,
|
|
481
|
+
spectrogram_layout: str | None = None,
|
|
482
482
|
augment_eval: bool = False,
|
|
483
483
|
**kwargs,
|
|
484
484
|
) -> PipelineFuncs:
|
|
@@ -36,6 +36,7 @@ from justdata.acoustic.configs import (
|
|
|
36
36
|
AudioPreset,
|
|
37
37
|
FrontendConfig,
|
|
38
38
|
LabelTransformConfig,
|
|
39
|
+
LabelTransformMode,
|
|
39
40
|
LogCompressionConfig,
|
|
40
41
|
MelConfig,
|
|
41
42
|
STFTConfig,
|
|
@@ -93,7 +94,9 @@ def _preprocess(sample_rate: int) -> AudioPreprocessConfig:
|
|
|
93
94
|
)
|
|
94
95
|
|
|
95
96
|
|
|
96
|
-
def _label(
|
|
97
|
+
def _label(
|
|
98
|
+
mode: LabelTransformMode, num_classes: int | None = None
|
|
99
|
+
) -> LabelTransformConfig:
|
|
97
100
|
return LabelTransformConfig(mode=mode, num_classes=num_classes)
|
|
98
101
|
|
|
99
102
|
|
|
@@ -7,6 +7,7 @@ from justdata.core.augmentations import (
|
|
|
7
7
|
AugmentationMetadata,
|
|
8
8
|
attach_augmentation_metadata,
|
|
9
9
|
)
|
|
10
|
+
from justdata.core.registry import qualified_name
|
|
10
11
|
|
|
11
12
|
|
|
12
13
|
RegistryFn = TypeVar("RegistryFn", bound=Callable)
|
|
@@ -37,8 +38,7 @@ def _register(registry: dict[str, Callable], label: str, name: str):
|
|
|
37
38
|
if name in registry:
|
|
38
39
|
existing = registry[name]
|
|
39
40
|
raise ValueError(
|
|
40
|
-
f"{label} '{name}' already registered by "
|
|
41
|
-
f"{existing.__module__}.{existing.__qualname__}"
|
|
41
|
+
f"{label} '{name}' already registered by {qualified_name(existing)}"
|
|
42
42
|
)
|
|
43
43
|
registry[name] = fn
|
|
44
44
|
return fn
|
|
@@ -68,8 +68,7 @@ def _register_augment(
|
|
|
68
68
|
if name in registry:
|
|
69
69
|
existing = registry[name]
|
|
70
70
|
raise ValueError(
|
|
71
|
-
f"{label} '{name}' already registered by "
|
|
72
|
-
f"{existing.__module__}.{existing.__qualname__}"
|
|
71
|
+
f"{label} '{name}' already registered by {qualified_name(existing)}"
|
|
73
72
|
)
|
|
74
73
|
registry[name] = fn
|
|
75
74
|
metadata_registry[name] = metadata
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
from collections.abc import Callable
|
|
3
|
+
from collections.abc import Callable, Mapping
|
|
4
|
+
from typing import Any, Protocol
|
|
4
5
|
|
|
5
6
|
import tensorflow as tf
|
|
6
7
|
|
|
@@ -10,12 +11,16 @@ from justdata.acoustic.segment import segment_waveform
|
|
|
10
11
|
from justdata.acoustic.schema import DURATION, METADATA, SAMPLE_RATE, WAVEFORM
|
|
11
12
|
|
|
12
13
|
|
|
14
|
+
class SeededSampleStage(Protocol):
|
|
15
|
+
def __call__(self, sample: dict, seed: tf.Tensor | int | None = None) -> dict: ...
|
|
16
|
+
|
|
17
|
+
|
|
13
18
|
def make_segment_stage(
|
|
14
|
-
config: SegmentStrategyConfig,
|
|
19
|
+
config: SegmentStrategyConfig | Mapping[str, Any],
|
|
15
20
|
*,
|
|
16
21
|
is_training: bool,
|
|
17
22
|
audio_key: str = WAVEFORM,
|
|
18
|
-
) ->
|
|
23
|
+
) -> SeededSampleStage:
|
|
19
24
|
config = SegmentStrategyConfig.from_dict(config)
|
|
20
25
|
|
|
21
26
|
def stage(sample: dict, seed: tf.Tensor | int | None = None) -> dict:
|