justdata 1.2.0__tar.gz → 1.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {justdata-1.2.0 → justdata-1.2.1}/PKG-INFO +9 -5
- {justdata-1.2.0 → justdata-1.2.1}/README.md +8 -4
- {justdata-1.2.0 → justdata-1.2.1}/pyproject.toml +8 -1
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/__init__.py +1 -1
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/compat/ast.py +34 -15
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/compat/efficientat.py +5 -1
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/compat/passt.py +5 -1
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/configs.py +24 -20
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/evals.py +3 -3
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/metadata.py +7 -7
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/pipelines.py +15 -15
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/presets.py +4 -1
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/registry.py +3 -4
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/resampling.py +0 -1
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/stages.py +8 -3
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/tasks.py +8 -2
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/core/adapters.py +3 -1
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/core/config_resolution.py +2 -2
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/core/finalization.py +1 -1
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/core/inventory.py +3 -3
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/core/loader.py +3 -3
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/core/metadata.py +16 -5
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/core/registry.py +6 -1
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/core/stats.py +3 -3
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/vision/augmentations/composed.py +1 -1
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/vision/augmentations/registry.py +3 -2
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/vision/corruptions/registry.py +3 -2
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/vision/geometry.py +2 -2
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/vision/pipelines.py +8 -8
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/vision/sources.py +18 -13
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/vision/tasks/classification.py +8 -8
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/vision/tasks/segmentation.py +2 -2
- {justdata-1.2.0 → justdata-1.2.1}/LICENSE +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/__init__.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/_dcase.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/_manifests.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/_random.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/_signal.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/adapters.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/augment/__init__.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/augment/_common.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/augment/batch.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/augment/configs.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/augment/patchout.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/augment/spectrogram.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/augment/waveform.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/batching.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/channel.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/compat/__init__.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/compat/ced.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/compat/panns.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/corruptions/__init__.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/corruptions/codec.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/corruptions/datasets.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/corruptions/device.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/corruptions/dynamics.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/corruptions/filters.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/corruptions/noise.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/corruptions/registry.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/corruptions/reverb.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/corruptions/timefreq.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/datasets.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/dcase2025.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/decoding.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/eval_views.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/filters.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/frontends/__init__.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/frontends/ast.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/frontends/kaldi.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/frontends/log.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/frontends/mel.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/frontends/mfcc.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/frontends/pcen.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/frontends/stft.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/jax.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/labels.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/layouts.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/normalization.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/postprocessing.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/preprocessing.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/schema.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/segment.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/sources.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/acoustic/stats.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/audio/__init__.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/core/__init__.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/core/_records.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/core/augmentations.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/core/cache.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/core/executed_config.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/core/execution.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/core/filters.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/core/label_mixing.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/core/presets.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/core/replay.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/core/sources.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/corruptions.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/py.typed +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/vision/__init__.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/vision/augmentations/__init__.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/vision/augmentations/auto.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/vision/augmentations/color.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/vision/augmentations/geometric.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/vision/augmentations/mixing.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/vision/corruptions/__init__.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/vision/corruptions/blur.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/vision/corruptions/digital.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/vision/corruptions/noise.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/vision/corruptions/standard.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/vision/corruptions/weather.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/vision/datasets.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/vision/encodings/__init__.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/vision/encodings/heatmaps.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/vision/encodings/panoptic_targets.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/vision/encodings/semantic_targets.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/vision/minic.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/vision/presets.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/vision/stages.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/vision/tasks/__init__.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/vision/tasks/depth.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/vision/tasks/panoptic.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/vision/transforms.py +0 -0
- {justdata-1.2.0 → justdata-1.2.1}/src/justdata/vision/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: justdata
|
|
3
|
-
Version: 1.2.
|
|
3
|
+
Version: 1.2.1
|
|
4
4
|
Summary: A TensorFlow-native data pipeline library with a modality-neutral core and first-class computer vision and acoustic recipes
|
|
5
5
|
Author: clementpoiret
|
|
6
6
|
Author-email: clementpoiret <clement@linux.com>
|
|
@@ -1282,6 +1282,7 @@ uv sync # install/update dependencies
|
|
|
1282
1282
|
uv run pytest # run all tests
|
|
1283
1283
|
uv run ruff check . # run lint checks
|
|
1284
1284
|
uv run ruff format --check . # check formatting
|
|
1285
|
+
uv run ty check src/ # type check the package
|
|
1285
1286
|
uv run mdformat --check --wrap 120 ./*.md benchmarks docs examples src tests # check Markdown formatting
|
|
1286
1287
|
uv run pytest tests/path/to/test_file.py::test_name # run a single test
|
|
1287
1288
|
```
|
|
@@ -1290,9 +1291,12 @@ The project targets Python 3.12 (see `.python-version`). `LD_LIBRARY_PATH` is co
|
|
|
1290
1291
|
|
|
1291
1292
|
See [Performance checks](benchmarks/README.md) for reproducible CPU benchmarks and the performance audit results.
|
|
1292
1293
|
|
|
1293
|
-
GitLab runs `sh tests/doctor/test_lint.sh` in a pinned Debian-based uv image.
|
|
1294
|
-
|
|
1295
|
-
|
|
1294
|
+
GitLab runs `sh tests/doctor/test_lint.sh` in a pinned Debian-based uv image. It checks Python with Ruff and Markdown
|
|
1295
|
+
with mdformat using the GFM, footnote, and MyST plugins from the `lint` dependency group alone, then type checks `src/`
|
|
1296
|
+
with ty against the locked development environment so that third-party imports resolve. TensorFlow attaches `Tensor`
|
|
1297
|
+
operators at runtime, so ty treats `tensorflow` imports as `Any` (see `[tool.ty.analysis]` in `pyproject.toml`). The
|
|
1298
|
+
image provides the standard Linux dynamic loader required by the Ruff and ty PyPI executables, which the Nix image
|
|
1299
|
+
lacks.
|
|
1296
1300
|
|
|
1297
1301
|
### Releases
|
|
1298
1302
|
|
|
@@ -1302,7 +1306,7 @@ package-support jobs pass. Build artifacts are retained for one week. Tag pipeli
|
|
|
1302
1306
|
to complete a release.
|
|
1303
1307
|
|
|
1304
1308
|
Before tagging, update `pyproject.toml`, `src/justdata/__init__.py`, and `CHANGELOG.md`, then run `uv lock`. Push a tag
|
|
1305
|
-
matching the package version, with an optional `v` prefix (for example, `v1.2.
|
|
1309
|
+
matching the package version, with an optional `v` prefix (for example, `v1.2.1` or `1.2.1`). Publishing rejects tags
|
|
1306
1310
|
that disagree with either version declaration. Branch pipelines build artifacts without publishing.
|
|
1307
1311
|
|
|
1308
1312
|
Configure a [PyPI GitLab trusted publisher](https://docs.pypi.org/trusted-publishers/adding-a-publisher/) for the
|
|
@@ -1239,6 +1239,7 @@ uv sync # install/update dependencies
|
|
|
1239
1239
|
uv run pytest # run all tests
|
|
1240
1240
|
uv run ruff check . # run lint checks
|
|
1241
1241
|
uv run ruff format --check . # check formatting
|
|
1242
|
+
uv run ty check src/ # type check the package
|
|
1242
1243
|
uv run mdformat --check --wrap 120 ./*.md benchmarks docs examples src tests # check Markdown formatting
|
|
1243
1244
|
uv run pytest tests/path/to/test_file.py::test_name # run a single test
|
|
1244
1245
|
```
|
|
@@ -1247,9 +1248,12 @@ The project targets Python 3.12 (see `.python-version`). `LD_LIBRARY_PATH` is co
|
|
|
1247
1248
|
|
|
1248
1249
|
See [Performance checks](benchmarks/README.md) for reproducible CPU benchmarks and the performance audit results.
|
|
1249
1250
|
|
|
1250
|
-
GitLab runs `sh tests/doctor/test_lint.sh` in a pinned Debian-based uv image.
|
|
1251
|
-
|
|
1252
|
-
|
|
1251
|
+
GitLab runs `sh tests/doctor/test_lint.sh` in a pinned Debian-based uv image. It checks Python with Ruff and Markdown
|
|
1252
|
+
with mdformat using the GFM, footnote, and MyST plugins from the `lint` dependency group alone, then type checks `src/`
|
|
1253
|
+
with ty against the locked development environment so that third-party imports resolve. TensorFlow attaches `Tensor`
|
|
1254
|
+
operators at runtime, so ty treats `tensorflow` imports as `Any` (see `[tool.ty.analysis]` in `pyproject.toml`). The
|
|
1255
|
+
image provides the standard Linux dynamic loader required by the Ruff and ty PyPI executables, which the Nix image
|
|
1256
|
+
lacks.
|
|
1253
1257
|
|
|
1254
1258
|
### Releases
|
|
1255
1259
|
|
|
@@ -1259,7 +1263,7 @@ package-support jobs pass. Build artifacts are retained for one week. Tag pipeli
|
|
|
1259
1263
|
to complete a release.
|
|
1260
1264
|
|
|
1261
1265
|
Before tagging, update `pyproject.toml`, `src/justdata/__init__.py`, and `CHANGELOG.md`, then run `uv lock`. Push a tag
|
|
1262
|
-
matching the package version, with an optional `v` prefix (for example, `v1.2.
|
|
1266
|
+
matching the package version, with an optional `v` prefix (for example, `v1.2.1` or `1.2.1`). Publishing rejects tags
|
|
1263
1267
|
that disagree with either version declaration. Branch pipelines build artifacts without publishing.
|
|
1264
1268
|
|
|
1265
1269
|
Configure a [PyPI GitLab trusted publisher](https://docs.pypi.org/trusted-publishers/adding-a-publisher/) for the
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "justdata"
|
|
3
|
-
version = "1.2.
|
|
3
|
+
version = "1.2.1"
|
|
4
4
|
description = "A TensorFlow-native data pipeline library with a modality-neutral core and first-class computer vision and acoustic recipes"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "MIT"
|
|
@@ -62,6 +62,7 @@ lint = [
|
|
|
62
62
|
"mdformat-gfm==1.0.0",
|
|
63
63
|
"mdformat-myst==0.3.0",
|
|
64
64
|
"ruff==0.15.20",
|
|
65
|
+
"ty==0.0.83",
|
|
65
66
|
]
|
|
66
67
|
dev = [
|
|
67
68
|
{ include-group = "lint" },
|
|
@@ -100,3 +101,9 @@ target-version = "py311"
|
|
|
100
101
|
|
|
101
102
|
[tool.ruff.lint]
|
|
102
103
|
select = ["E4", "E7", "E9", "F"]
|
|
104
|
+
|
|
105
|
+
[tool.ty.analysis]
|
|
106
|
+
# TensorFlow attaches Tensor operators and indexing at runtime, so its inferred types
|
|
107
|
+
# are unusable; the optional `wilds` extra is absent from the development environment.
|
|
108
|
+
replace-imports-with-any = ["tensorflow.**"]
|
|
109
|
+
allowed-unresolved-imports = ["wilds"]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
from typing import Any, Mapping
|
|
3
|
+
from typing import Any, Mapping, TypedDict
|
|
4
4
|
|
|
5
5
|
import tensorflow as tf
|
|
6
6
|
|
|
@@ -16,7 +16,7 @@ from justdata.acoustic.configs import (
|
|
|
16
16
|
SegmentStrategyConfig,
|
|
17
17
|
)
|
|
18
18
|
from justdata.acoustic.labels import transform_label
|
|
19
|
-
from justdata.acoustic.layouts import convert_audio_layout
|
|
19
|
+
from justdata.acoustic.layouts import AudioLayout, convert_audio_layout
|
|
20
20
|
from justdata.acoustic.preprocessing import make_preprocessing
|
|
21
21
|
from justdata.acoustic.schema import FEATURES, LABEL, METADATA, SAMPLE_RATE, WAVEFORM
|
|
22
22
|
from justdata.acoustic.segment import segment_waveform
|
|
@@ -28,7 +28,20 @@ AST_MIX_WAVEFORM = "ast_mix_waveform"
|
|
|
28
28
|
AST_MIX_LABEL = "ast_mix_label"
|
|
29
29
|
AST_MIX_LAMBDA = "ast_mix_lambda"
|
|
30
30
|
|
|
31
|
-
|
|
31
|
+
|
|
32
|
+
class _ASTRecipe(TypedDict):
|
|
33
|
+
target_length: int
|
|
34
|
+
freqm: int
|
|
35
|
+
timem: int
|
|
36
|
+
mixup: float
|
|
37
|
+
mean: float
|
|
38
|
+
std: float
|
|
39
|
+
noise: bool
|
|
40
|
+
input_duration: float
|
|
41
|
+
num_classes: int
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
AST_RECIPE_CONFIGS: dict[str, _ASTRecipe] = {
|
|
32
45
|
"audioset": {
|
|
33
46
|
"target_length": 1024,
|
|
34
47
|
"freqm": 48,
|
|
@@ -351,7 +364,9 @@ def _ast_config(
|
|
|
351
364
|
|
|
352
365
|
|
|
353
366
|
def _target_length(
|
|
354
|
-
layout:
|
|
367
|
+
layout: AudioLayout,
|
|
368
|
+
static_shape: tuple[int | None, ...] | None,
|
|
369
|
+
ast: Mapping[str, Any],
|
|
355
370
|
) -> int:
|
|
356
371
|
if "target_length" in ast:
|
|
357
372
|
return int(ast["target_length"])
|
|
@@ -359,15 +374,19 @@ def _target_length(
|
|
|
359
374
|
raise ValueError(
|
|
360
375
|
"AST pipeline requires static_shape or metadata['ast']['target_length']"
|
|
361
376
|
)
|
|
362
|
-
if layout
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
377
|
+
if layout in ("btf", "btfc"):
|
|
378
|
+
target_length = static_shape[0]
|
|
379
|
+
elif layout == "bft":
|
|
380
|
+
target_length = static_shape[1]
|
|
381
|
+
elif layout == "bcft":
|
|
382
|
+
target_length = static_shape[2]
|
|
383
|
+
else:
|
|
384
|
+
raise ValueError(f"Unsupported AST layout: {layout!r}")
|
|
385
|
+
if target_length is None:
|
|
386
|
+
raise ValueError(
|
|
387
|
+
"AST pipeline requires a static time dimension in static_shape"
|
|
388
|
+
)
|
|
389
|
+
return int(target_length)
|
|
371
390
|
|
|
372
391
|
|
|
373
392
|
def _transform_sample_label(
|
|
@@ -454,7 +473,7 @@ def _maybe_mix_waveform_and_label(
|
|
|
454
473
|
def _make_ast_feature_stage(
|
|
455
474
|
*,
|
|
456
475
|
frontend: FrontendConfig,
|
|
457
|
-
layout:
|
|
476
|
+
layout: AudioLayout,
|
|
458
477
|
dtype: str,
|
|
459
478
|
output_key: str,
|
|
460
479
|
static_shape: tuple[int | None, ...] | None,
|
|
@@ -529,7 +548,7 @@ def ast_pipeline(
|
|
|
529
548
|
preprocess: dict | None = None,
|
|
530
549
|
segment: dict | None = None,
|
|
531
550
|
frontend: dict | FrontendConfig | None = None,
|
|
532
|
-
layout:
|
|
551
|
+
layout: AudioLayout = "btf",
|
|
533
552
|
dtype: str = "float32",
|
|
534
553
|
output_key: str | None = None,
|
|
535
554
|
static_shape: tuple[int | None, ...] | None = None,
|
|
@@ -7,12 +7,14 @@ from justdata.acoustic._dcase import _dcase_label_transform
|
|
|
7
7
|
from justdata.acoustic.configs import (
|
|
8
8
|
AudioPreprocessConfig,
|
|
9
9
|
AudioPreset,
|
|
10
|
+
DurationPolicy,
|
|
10
11
|
FeatureNormConfig,
|
|
11
12
|
FrontendConfig,
|
|
12
13
|
LabelTransformConfig,
|
|
13
14
|
LogCompressionConfig,
|
|
14
15
|
MelConfig,
|
|
15
16
|
STFTConfig,
|
|
17
|
+
SegmentPadMode,
|
|
16
18
|
SegmentStrategyConfig,
|
|
17
19
|
)
|
|
18
20
|
from justdata.acoustic.frontends.log import compress_log
|
|
@@ -174,7 +176,9 @@ def dcase2025_task1_dymn_32k_1s() -> AudioPreset:
|
|
|
174
176
|
)
|
|
175
177
|
|
|
176
178
|
|
|
177
|
-
def _dcase_10s(
|
|
179
|
+
def _dcase_10s(
|
|
180
|
+
name: str, *, duration_policy: DurationPolicy, pad_mode: SegmentPadMode
|
|
181
|
+
) -> AudioPreset:
|
|
178
182
|
metadata = _metadata("efficientat", source_duration=1.0)
|
|
179
183
|
metadata["model_input_duration"] = 10.0
|
|
180
184
|
return _preset(
|
|
@@ -5,12 +5,14 @@ from justdata.acoustic._dcase import _dcase_label_transform
|
|
|
5
5
|
from justdata.acoustic.configs import (
|
|
6
6
|
AudioPreprocessConfig,
|
|
7
7
|
AudioPreset,
|
|
8
|
+
DurationPolicy,
|
|
8
9
|
FeatureNormConfig,
|
|
9
10
|
FrontendConfig,
|
|
10
11
|
LabelTransformConfig,
|
|
11
12
|
LogCompressionConfig,
|
|
12
13
|
MelConfig,
|
|
13
14
|
STFTConfig,
|
|
15
|
+
SegmentPadMode,
|
|
14
16
|
SegmentStrategyConfig,
|
|
15
17
|
)
|
|
16
18
|
|
|
@@ -137,7 +139,9 @@ def dcase2025_task1_passt_32k_1s() -> AudioPreset:
|
|
|
137
139
|
)
|
|
138
140
|
|
|
139
141
|
|
|
140
|
-
def _dcase_10s(
|
|
142
|
+
def _dcase_10s(
|
|
143
|
+
name: str, *, duration_policy: DurationPolicy, pad_mode: SegmentPadMode
|
|
144
|
+
) -> AudioPreset:
|
|
141
145
|
metadata = _metadata(source_duration=1.0)
|
|
142
146
|
metadata["model_input_duration"] = 10.0
|
|
143
147
|
return _preset(
|
|
@@ -5,10 +5,19 @@ import hashlib
|
|
|
5
5
|
import json
|
|
6
6
|
from collections.abc import Mapping
|
|
7
7
|
from dataclasses import dataclass, field
|
|
8
|
-
from typing import Any, Literal,
|
|
8
|
+
from typing import Any, ClassVar, Literal, Self
|
|
9
9
|
|
|
10
10
|
|
|
11
|
-
|
|
11
|
+
SegmentPadMode = Literal["zero", "repeat", "reflect"]
|
|
12
|
+
DurationPolicy = Literal[
|
|
13
|
+
"keep_1s",
|
|
14
|
+
"pad_to_model_duration",
|
|
15
|
+
"tile_to_model_duration",
|
|
16
|
+
"repeat_pad_to_model_duration",
|
|
17
|
+
"sliding_windows",
|
|
18
|
+
"none",
|
|
19
|
+
]
|
|
20
|
+
LabelTransformMode = Literal["index", "one_hot", "multi_hot", "text", "event_frames"]
|
|
12
21
|
|
|
13
22
|
|
|
14
23
|
def _canonical_json(config: Any) -> str:
|
|
@@ -48,6 +57,8 @@ def _tuple_or_scalar_value(value: Any) -> Any:
|
|
|
48
57
|
|
|
49
58
|
|
|
50
59
|
class _SerializableConfig:
|
|
60
|
+
__dataclass_fields__: ClassVar[dict[str, dataclasses.Field[Any]]]
|
|
61
|
+
|
|
51
62
|
def __post_init__(self) -> None:
|
|
52
63
|
self.validate()
|
|
53
64
|
|
|
@@ -58,13 +69,15 @@ class _SerializableConfig:
|
|
|
58
69
|
return _canonical_json(self)
|
|
59
70
|
|
|
60
71
|
@classmethod
|
|
61
|
-
def from_dict(cls
|
|
72
|
+
def from_dict(cls, data: Mapping[str, Any] | _SerializableConfig) -> Self:
|
|
62
73
|
if isinstance(data, cls):
|
|
63
74
|
return data
|
|
64
|
-
|
|
75
|
+
if not isinstance(data, Mapping):
|
|
76
|
+
raise TypeError(f"{cls.__name__} requires a mapping or {cls.__name__}")
|
|
77
|
+
return cls(**data)
|
|
65
78
|
|
|
66
79
|
@classmethod
|
|
67
|
-
def from_json(cls
|
|
80
|
+
def from_json(cls, data: str) -> Self:
|
|
68
81
|
return cls.from_dict(json.loads(data))
|
|
69
82
|
|
|
70
83
|
def hash(self) -> str:
|
|
@@ -119,21 +132,14 @@ class SegmentStrategyConfig(_SerializableConfig):
|
|
|
119
132
|
clip_duration: float
|
|
120
133
|
train_mode: Literal["random_crop", "center_crop", "full", "sliding", "pad_or_crop"]
|
|
121
134
|
eval_mode: Literal["center_crop", "full", "sliding", "multi_crop"]
|
|
122
|
-
pad_mode:
|
|
135
|
+
pad_mode: SegmentPadMode
|
|
123
136
|
pad_position: Literal["right", "center", "random"]
|
|
124
137
|
num_views: int = 1
|
|
125
138
|
sliding_hop_duration: float | None = None
|
|
126
139
|
allow_train_sliding: bool = False
|
|
127
140
|
drop_short: bool = False
|
|
128
141
|
min_duration: float | None = None
|
|
129
|
-
duration_policy:
|
|
130
|
-
"keep_1s",
|
|
131
|
-
"pad_to_model_duration",
|
|
132
|
-
"tile_to_model_duration",
|
|
133
|
-
"repeat_pad_to_model_duration",
|
|
134
|
-
"sliding_windows",
|
|
135
|
-
"none",
|
|
136
|
-
] = "none"
|
|
142
|
+
duration_policy: DurationPolicy = "none"
|
|
137
143
|
|
|
138
144
|
def validate(self) -> SegmentStrategyConfig:
|
|
139
145
|
_ensure_positive("clip_duration", self.clip_duration)
|
|
@@ -401,7 +407,7 @@ class FrontendConfig(_SerializableConfig):
|
|
|
401
407
|
|
|
402
408
|
@dataclass(frozen=True)
|
|
403
409
|
class LabelTransformConfig(_SerializableConfig):
|
|
404
|
-
mode:
|
|
410
|
+
mode: LabelTransformMode
|
|
405
411
|
num_classes: int | None = None
|
|
406
412
|
class_names: tuple[str, ...] | None = None
|
|
407
413
|
smoothing: float = 0.0
|
|
@@ -480,12 +486,10 @@ class AudioPreset(_SerializableConfig):
|
|
|
480
486
|
self.validate()
|
|
481
487
|
|
|
482
488
|
@classmethod
|
|
483
|
-
def from_dict(cls, data: Mapping[str, Any] |
|
|
484
|
-
if isinstance(data,
|
|
485
|
-
return data
|
|
486
|
-
if "target_sample_rate" not in data:
|
|
489
|
+
def from_dict(cls, data: Mapping[str, Any] | _SerializableConfig) -> Self:
|
|
490
|
+
if isinstance(data, Mapping) and "target_sample_rate" not in data:
|
|
487
491
|
raise ValueError("AudioPreset requires target_sample_rate")
|
|
488
|
-
return
|
|
492
|
+
return super().from_dict(data)
|
|
489
493
|
|
|
490
494
|
def validate(self) -> AudioPreset:
|
|
491
495
|
if not self.name:
|
|
@@ -37,10 +37,10 @@ def macro_classwise_accuracy(y_true, y_pred, num_classes: int = 10) -> float:
|
|
|
37
37
|
def _group_accuracy(y_true, y_pred, groups: Sequence[Any]) -> dict[Any, float]:
|
|
38
38
|
true = _labels(y_true)
|
|
39
39
|
pred = _predictions(y_pred)
|
|
40
|
-
|
|
40
|
+
group_values = np.asarray(groups)
|
|
41
41
|
result = {}
|
|
42
|
-
for group in sorted(set(
|
|
43
|
-
mask =
|
|
42
|
+
for group in sorted(set(group_values.tolist()), key=str):
|
|
43
|
+
mask = group_values == group
|
|
44
44
|
result[group] = float(np.mean(pred[mask] == true[mask]))
|
|
45
45
|
return result
|
|
46
46
|
|
|
@@ -32,12 +32,12 @@ class MetadataEncoder:
|
|
|
32
32
|
examples_or_vocab
|
|
33
33
|
):
|
|
34
34
|
self.vocab = {
|
|
35
|
-
field: {
|
|
36
|
-
|
|
37
|
-
|
|
35
|
+
field: {
|
|
36
|
+
str(value): i
|
|
37
|
+
for i, value in enumerate(examples_or_vocab.get(field, ()))
|
|
38
|
+
}
|
|
39
|
+
for field in self._FIELD_KEYS
|
|
38
40
|
}
|
|
39
|
-
for field in self._FIELD_KEYS:
|
|
40
|
-
self.vocab.setdefault(field, {})
|
|
41
41
|
return self
|
|
42
42
|
|
|
43
43
|
examples = (
|
|
@@ -99,12 +99,12 @@ class MetadataEncoder:
|
|
|
99
99
|
return decoded
|
|
100
100
|
|
|
101
101
|
@staticmethod
|
|
102
|
-
def _looks_like_vocab(value: Mapping[
|
|
102
|
+
def _looks_like_vocab(value: Mapping[Any, Any]) -> bool:
|
|
103
103
|
return all(isinstance(v, (list, tuple, set)) for v in value.values())
|
|
104
104
|
|
|
105
105
|
@staticmethod
|
|
106
106
|
def _first_present(
|
|
107
|
-
metadata: Mapping[
|
|
107
|
+
metadata: Mapping[Any, Any], keys: tuple[str, ...]
|
|
108
108
|
) -> Any | None:
|
|
109
109
|
for key in keys:
|
|
110
110
|
if key in metadata:
|
|
@@ -460,25 +460,25 @@ def _resolve_ast_config(config, is_training):
|
|
|
460
460
|
|
|
461
461
|
|
|
462
462
|
def default_pipeline(
|
|
463
|
-
preproc_kwargs: dict = None,
|
|
464
|
-
aug_kwargs: dict = None,
|
|
465
|
-
laug_kwargs: dict = None,
|
|
466
|
-
postproc_kwargs: dict = None,
|
|
467
|
-
preprocess: dict = None,
|
|
468
|
-
segment: dict = None,
|
|
469
|
-
frontend: dict = None,
|
|
470
|
-
layout: str = None,
|
|
463
|
+
preproc_kwargs: dict | None = None,
|
|
464
|
+
aug_kwargs: dict | None = None,
|
|
465
|
+
laug_kwargs: dict | None = None,
|
|
466
|
+
postproc_kwargs: dict | None = None,
|
|
467
|
+
preprocess: dict | None = None,
|
|
468
|
+
segment: dict | None = None,
|
|
469
|
+
frontend: dict | None = None,
|
|
470
|
+
layout: str | None = None,
|
|
471
471
|
dtype: str = "float32",
|
|
472
|
-
output_key: str = None,
|
|
473
|
-
static_shape: tuple[int | None, ...] = None,
|
|
474
|
-
input_duration: float = None,
|
|
475
|
-
target_sample_rate: int = None,
|
|
476
|
-
label_transform: dict = None,
|
|
477
|
-
train_augment: dict = None,
|
|
472
|
+
output_key: str | None = None,
|
|
473
|
+
static_shape: tuple[int | None, ...] | None = None,
|
|
474
|
+
input_duration: float | None = None,
|
|
475
|
+
target_sample_rate: int | None = None,
|
|
476
|
+
label_transform: dict | None = None,
|
|
477
|
+
train_augment: dict | None = None,
|
|
478
478
|
waveform_augmentations=None,
|
|
479
479
|
spectrogram_augmentations=None,
|
|
480
480
|
batch_augmentations=None,
|
|
481
|
-
spectrogram_layout: str = None,
|
|
481
|
+
spectrogram_layout: str | None = None,
|
|
482
482
|
augment_eval: bool = False,
|
|
483
483
|
**kwargs,
|
|
484
484
|
) -> PipelineFuncs:
|
|
@@ -36,6 +36,7 @@ from justdata.acoustic.configs import (
|
|
|
36
36
|
AudioPreset,
|
|
37
37
|
FrontendConfig,
|
|
38
38
|
LabelTransformConfig,
|
|
39
|
+
LabelTransformMode,
|
|
39
40
|
LogCompressionConfig,
|
|
40
41
|
MelConfig,
|
|
41
42
|
STFTConfig,
|
|
@@ -93,7 +94,9 @@ def _preprocess(sample_rate: int) -> AudioPreprocessConfig:
|
|
|
93
94
|
)
|
|
94
95
|
|
|
95
96
|
|
|
96
|
-
def _label(
|
|
97
|
+
def _label(
|
|
98
|
+
mode: LabelTransformMode, num_classes: int | None = None
|
|
99
|
+
) -> LabelTransformConfig:
|
|
97
100
|
return LabelTransformConfig(mode=mode, num_classes=num_classes)
|
|
98
101
|
|
|
99
102
|
|
|
@@ -7,6 +7,7 @@ from justdata.core.augmentations import (
|
|
|
7
7
|
AugmentationMetadata,
|
|
8
8
|
attach_augmentation_metadata,
|
|
9
9
|
)
|
|
10
|
+
from justdata.core.registry import qualified_name
|
|
10
11
|
|
|
11
12
|
|
|
12
13
|
RegistryFn = TypeVar("RegistryFn", bound=Callable)
|
|
@@ -37,8 +38,7 @@ def _register(registry: dict[str, Callable], label: str, name: str):
|
|
|
37
38
|
if name in registry:
|
|
38
39
|
existing = registry[name]
|
|
39
40
|
raise ValueError(
|
|
40
|
-
f"{label} '{name}' already registered by "
|
|
41
|
-
f"{existing.__module__}.{existing.__qualname__}"
|
|
41
|
+
f"{label} '{name}' already registered by {qualified_name(existing)}"
|
|
42
42
|
)
|
|
43
43
|
registry[name] = fn
|
|
44
44
|
return fn
|
|
@@ -68,8 +68,7 @@ def _register_augment(
|
|
|
68
68
|
if name in registry:
|
|
69
69
|
existing = registry[name]
|
|
70
70
|
raise ValueError(
|
|
71
|
-
f"{label} '{name}' already registered by "
|
|
72
|
-
f"{existing.__module__}.{existing.__qualname__}"
|
|
71
|
+
f"{label} '{name}' already registered by {qualified_name(existing)}"
|
|
73
72
|
)
|
|
74
73
|
registry[name] = fn
|
|
75
74
|
metadata_registry[name] = metadata
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
from collections.abc import Callable
|
|
3
|
+
from collections.abc import Callable, Mapping
|
|
4
|
+
from typing import Any, Protocol
|
|
4
5
|
|
|
5
6
|
import tensorflow as tf
|
|
6
7
|
|
|
@@ -10,12 +11,16 @@ from justdata.acoustic.segment import segment_waveform
|
|
|
10
11
|
from justdata.acoustic.schema import DURATION, METADATA, SAMPLE_RATE, WAVEFORM
|
|
11
12
|
|
|
12
13
|
|
|
14
|
+
class SeededSampleStage(Protocol):
|
|
15
|
+
def __call__(self, sample: dict, seed: tf.Tensor | int | None = None) -> dict: ...
|
|
16
|
+
|
|
17
|
+
|
|
13
18
|
def make_segment_stage(
|
|
14
|
-
config: SegmentStrategyConfig,
|
|
19
|
+
config: SegmentStrategyConfig | Mapping[str, Any],
|
|
15
20
|
*,
|
|
16
21
|
is_training: bool,
|
|
17
22
|
audio_key: str = WAVEFORM,
|
|
18
|
-
) ->
|
|
23
|
+
) -> SeededSampleStage:
|
|
19
24
|
config = SegmentStrategyConfig.from_dict(config)
|
|
20
25
|
|
|
21
26
|
def stage(sample: dict, seed: tf.Tensor | int | None = None) -> dict:
|
|
@@ -20,6 +20,7 @@ from justdata.acoustic.configs import (
|
|
|
20
20
|
SegmentStrategyConfig,
|
|
21
21
|
)
|
|
22
22
|
from justdata.acoustic.labels import transform_label
|
|
23
|
+
from justdata.acoustic.layouts import AudioLayout
|
|
23
24
|
from justdata.acoustic.postprocessing import make_model_input_stage, preset_info
|
|
24
25
|
from justdata.acoustic.preprocessing import make_preprocessing as _make_preprocessing
|
|
25
26
|
from justdata.acoustic.schema import FEATURES, LABEL, METADATA
|
|
@@ -124,6 +125,7 @@ def make_late_augmentations(**kwargs):
|
|
|
124
125
|
make_spectrogram_augmentation_stage,
|
|
125
126
|
normalize_spectrogram_augment_specs,
|
|
126
127
|
)
|
|
128
|
+
from justdata.acoustic.augment.spectrogram import SpectrogramLayout
|
|
127
129
|
|
|
128
130
|
known_batch_augments = {
|
|
129
131
|
"batch_mixstyle",
|
|
@@ -183,7 +185,11 @@ def make_late_augmentations(**kwargs):
|
|
|
183
185
|
return _identity_batch
|
|
184
186
|
|
|
185
187
|
sample_layouts = {"tf", "tfc", "cft"}
|
|
186
|
-
batch_to_sample_layout
|
|
188
|
+
batch_to_sample_layout: dict[str, SpectrogramLayout] = {
|
|
189
|
+
"btf": "tf",
|
|
190
|
+
"btfc": "tfc",
|
|
191
|
+
"bcft": "cft",
|
|
192
|
+
}
|
|
187
193
|
layout_source = spectrogram_layout or model_layout
|
|
188
194
|
if not spectrogram_specs:
|
|
189
195
|
sample_spectrogram_layout = None
|
|
@@ -327,7 +333,7 @@ def make_postprocessing(
|
|
|
327
333
|
segment_config: SegmentStrategyConfig | dict | None = None,
|
|
328
334
|
is_training: bool = False,
|
|
329
335
|
frontend: dict | None = None,
|
|
330
|
-
layout:
|
|
336
|
+
layout: AudioLayout | None = None,
|
|
331
337
|
dtype: str = "float32",
|
|
332
338
|
output_key: str | None = None,
|
|
333
339
|
static_shape: tuple[int | None, ...] | None = None,
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import threading
|
|
2
2
|
from typing import Any, Dict, Protocol
|
|
3
3
|
|
|
4
|
+
from justdata.core.registry import qualified_name
|
|
5
|
+
|
|
4
6
|
|
|
5
7
|
class DatasetAdapter(Protocol):
|
|
6
8
|
"""Maps raw dataset samples to a canonical dict schema."""
|
|
@@ -18,7 +20,7 @@ def register_adapter(dataset_name: str):
|
|
|
18
20
|
if dataset_name in _ADAPTERS:
|
|
19
21
|
raise ValueError(
|
|
20
22
|
f"Adapter for '{dataset_name}' already registered by "
|
|
21
|
-
f"{_ADAPTERS[dataset_name]
|
|
23
|
+
f"{qualified_name(_ADAPTERS[dataset_name])}"
|
|
22
24
|
)
|
|
23
25
|
_ADAPTERS[dataset_name] = fn
|
|
24
26
|
return fn
|
|
@@ -2,7 +2,7 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import copy
|
|
4
4
|
import inspect
|
|
5
|
-
from collections.abc import Mapping
|
|
5
|
+
from collections.abc import Collection, Mapping
|
|
6
6
|
from typing import Any, Callable
|
|
7
7
|
|
|
8
8
|
|
|
@@ -25,7 +25,7 @@ def resolve_callable_config(
|
|
|
25
25
|
supplied: Mapping[str, Any] | None,
|
|
26
26
|
*,
|
|
27
27
|
path: str,
|
|
28
|
-
omit:
|
|
28
|
+
omit: Collection[str] | None = None,
|
|
29
29
|
) -> dict[str, Any]:
|
|
30
30
|
"""Validate keyword configuration and expand a callable's declared defaults."""
|
|
31
31
|
if supplied is None:
|
|
@@ -11,7 +11,7 @@ import shutil
|
|
|
11
11
|
from collections.abc import Callable, Mapping, Sequence
|
|
12
12
|
from dataclasses import dataclass, field
|
|
13
13
|
from pathlib import Path
|
|
14
|
-
from typing import Any
|
|
14
|
+
from typing import Any, NoReturn
|
|
15
15
|
|
|
16
16
|
import tensorflow as tf
|
|
17
17
|
|
|
@@ -97,7 +97,7 @@ class AdmittedInventory:
|
|
|
97
97
|
path: Path
|
|
98
98
|
|
|
99
99
|
|
|
100
|
-
def _fail(report, code, message, *, cause=None, **context):
|
|
100
|
+
def _fail(report, code, message, *, cause=None, **context) -> NoReturn:
|
|
101
101
|
issue = {"code": code, "message": message, **context}
|
|
102
102
|
report["complete"] = False
|
|
103
103
|
failures = report.get("failures", [])
|
|
@@ -384,7 +384,7 @@ def admit_inventory(
|
|
|
384
384
|
copy of InventoryRecord.metadata; identity is supplied by the inventory.
|
|
385
385
|
The parent of ``snapshot_dir`` must exist and the destination must not exist.
|
|
386
386
|
"""
|
|
387
|
-
report = {
|
|
387
|
+
report: dict[str, Any] = {
|
|
388
388
|
"schema": _VERSION,
|
|
389
389
|
"inventory_id": inventory_id if isinstance(inventory_id, str) else None,
|
|
390
390
|
"complete": False,
|