osekit 1.0.0__tar.gz → 1.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {osekit-1.0.0 → osekit-1.0.1}/PKG-INFO +2 -2
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/jobs.rst +26 -19
- {osekit-1.0.0 → osekit-1.0.1}/pyproject.toml +2 -2
- {osekit-1.0.0 → osekit-1.0.1}/src/osekit/core/audio_dataset.py +6 -1
- {osekit-1.0.0 → osekit-1.0.1}/src/osekit/core/base_data.py +5 -6
- {osekit-1.0.0 → osekit-1.0.1}/src/osekit/core/base_dataset.py +9 -0
- {osekit-1.0.0 → osekit-1.0.1}/src/osekit/core/base_file.py +4 -1
- {osekit-1.0.0 → osekit-1.0.1}/src/osekit/core/spectro_data.py +7 -12
- {osekit-1.0.0 → osekit-1.0.1}/src/osekit/core/spectro_dataset.py +6 -5
- {osekit-1.0.0 → osekit-1.0.1}/src/osekit/utils/job.py +25 -2
- {osekit-1.0.0 → osekit-1.0.1}/tests/test_job.py +25 -1
- {osekit-1.0.0 → osekit-1.0.1}/tests/test_public_api.py +18 -6
- {osekit-1.0.0 → osekit-1.0.1}/tests/test_serialization.py +9 -4
- {osekit-1.0.0 → osekit-1.0.1}/uv.lock +1 -1
- {osekit-1.0.0 → osekit-1.0.1}/.github/workflows/documentation.yml +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/.github/workflows/github_ci.yml +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/.github/workflows/publish.yml +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/.gitignore +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/.pre-commit-config.yaml +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/CHANGELOG.md +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/CITATION.cff +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/CONTRIBUTING.md +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/LICENSE +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/README.md +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/Makefile +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/logo/osekit.png +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/logo/osekit_mid.png +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/logo/osekit_small.png +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/logo/osmose.png +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/logo/osmose_texte_sombre.png +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/logo/osmose_texte_sombre_small.png +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/make.bat +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/_static/audio_data/ex1.svg +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/_static/audio_data/ex2.svg +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/_static/data_structure/file_item_data_0.svg +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/_static/data_structure/file_item_data_1.svg +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/_static/data_structure/file_item_data_2.svg +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/_static/favicon.ico +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/_static/ltas/ltas.gif +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/_static/sample_audio/id/1829.wav +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/_static/sample_audio/id/1830.wav +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/_static/sample_audio/id/1832.wav +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/_static/sample_audio/id/1833.wav +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/_static/sample_audio/id/1834.wav +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/_static/sample_audio/id/1835.wav +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/_static/sample_audio/id/1836.wav +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/_static/sample_audio/id/1837.wav +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/_static/sample_audio/id/1838.wav +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/_static/sample_audio/timestamped/sample_220925_223450.wav +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/_static/sample_audio/timestamped/sample_220925_223500.wav +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/_static/sample_audio/timestamped/sample_220925_223510.wav +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/_static/sample_audio/timestamped/sample_220925_223520.wav +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/_static/sample_audio/timestamped/sample_220925_223530.wav +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/_static/sample_audio/timestamped/sample_220925_223600.wav +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/_static/sample_audio/timestamped/sample_220925_223610.wav +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/_static/sample_audio/timestamped/sample_220925_223620.wav +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/_static/sample_audio/timestamped/sample_220925_223630.wav +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/_static/sample_audio/timestamped/sample_220925_223640.wav +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/_static/spectrograms/frequency_scale.png +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/api.rst +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/audiodata.rst +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/audiodataset.rst +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/audiofile.rst +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/audiofilemanager.rst +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/audioitem.rst +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/basedata.rst +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/basedataset.rst +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/basefile.rst +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/baseitem.rst +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/conf.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/contributing.rst +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/coreapi.rst +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/coreapi_home.rst +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/coreapi_introduction.rst +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/coreapi_usage.rst +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/event.rst +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/example_ltas.rst +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/example_ltas_core.ipynb +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/example_ltas_public.ipynb +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/example_multiple_spectrograms.rst +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/example_multiple_spectrograms_core.ipynb +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/example_multiple_spectrograms_id.rst +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/example_multiple_spectrograms_id_core.ipynb +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/example_multiple_spectrograms_id_public.ipynb +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/example_multiple_spectrograms_public.ipynb +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/example_reshaping_multiple_files.rst +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/example_reshaping_multiple_files_core.ipynb +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/example_reshaping_multiple_files_public.ipynb +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/example_reshaping_one_file.ipynb +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/example_spectrogram.ipynb +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/examples.rst +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/export_transform.rst +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/frequencyscale.rst +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/index.rst +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/installation.rst +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/instrument.rst +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/job.rst +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/ltasdata.rst +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/multiprocessing.rst +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/normalization.rst +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/project.rst +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/publicapi.rst +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/publicapi_usage.rst +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/spectrodata.rst +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/spectrodataset.rst +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/spectrofile.rst +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/spectroitem.rst +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/transform.rst +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/usage.rst +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/docs/source/utils.rst +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/src/osekit/__init__.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/src/osekit/audio_backend/audio_backend.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/src/osekit/audio_backend/audio_file_manager.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/src/osekit/audio_backend/mseed_backend.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/src/osekit/audio_backend/soundfile_backend.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/src/osekit/config.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/src/osekit/core/__init__.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/src/osekit/core/audio_data.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/src/osekit/core/audio_file.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/src/osekit/core/audio_item.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/src/osekit/core/base_item.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/src/osekit/core/event.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/src/osekit/core/frequency_scale.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/src/osekit/core/instrument.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/src/osekit/core/json_serializer.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/src/osekit/core/ltas_data.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/src/osekit/core/ltas_dataset.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/src/osekit/core/spectro_file.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/src/osekit/core/spectro_item.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/src/osekit/core_api/__init__.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/src/osekit/logging_config.yaml +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/src/osekit/logging_context.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/src/osekit/public/__init__.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/src/osekit/public/export_transform.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/src/osekit/public/project.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/src/osekit/public/transform.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/src/osekit/public_api/__init__.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/src/osekit/utils/__init__.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/src/osekit/utils/audio.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/src/osekit/utils/core.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/src/osekit/utils/deserialization.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/src/osekit/utils/formatting.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/src/osekit/utils/multiprocess.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/src/osekit/utils/path.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/src/osekit/utils/timestamp.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/tests/__init__.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/tests/conftest.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/tests/helpers/__init__.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/tests/helpers/dummy.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/tests/sample_dataset/sample1_180701_204453.wav +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/tests/sample_dataset/sample1_190222_010000.wav +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/tests/sample_dataset/sample1_220998_185532.wav +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/tests/test_audio.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/tests/test_audio_file_manager.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/tests/test_core_api_base.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/tests/test_event.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/tests/test_export_transform.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/tests/test_files.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/tests/test_frequency_scales.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/tests/test_instrument.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/tests/test_logging.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/tests/test_mseed.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/tests/test_multiprocessing.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/tests/test_permissions.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/tests/test_spectro.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/tests/test_timestamp_utils.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/tests/test_utils.py +0 -0
- {osekit-1.0.0 → osekit-1.0.1}/tests/test_welch.py +0 -0
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: osekit
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.1
|
|
4
4
|
Summary: OSEkit
|
|
5
5
|
Author-email: OSmOSE team <osmose@ensta-bretagne.fr>
|
|
6
6
|
License-Expression: MIT
|
|
7
7
|
License-File: LICENSE
|
|
8
|
-
Classifier: Development Status ::
|
|
8
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
9
9
|
Classifier: Programming Language :: Python :: 3.12
|
|
10
10
|
Classifier: Programming Language :: Python :: 3.13
|
|
11
11
|
Requires-Python: >=3.12
|
|
@@ -25,6 +25,7 @@ instance to the :attr:`osekit.public.project.Project.job_builder` attribute:
|
|
|
25
25
|
job_config = JobConfig(
|
|
26
26
|
nb_nodes=1, # Number of nodes on which the job runs
|
|
27
27
|
ncpus=28, # Number of total cores used per node
|
|
28
|
+
ngpus=1, # Number of total GPU used per node
|
|
28
29
|
mem="60gb", # Maximum amount of physical memory used by the job
|
|
29
30
|
walltime=Timedelta(hours=5), # Maximum amount of real itime during which the job can be running
|
|
30
31
|
venv_name=os.environ["CONDA_DEFAULT_ENV"], # Works only for conda venvs
|
|
@@ -53,17 +54,20 @@ and follow the console arguments of the :mod:`osekit.public.export` script.
|
|
|
53
54
|
.. code-block:: python
|
|
54
55
|
|
|
55
56
|
import os
|
|
57
|
+
from pathlib import Path
|
|
58
|
+
|
|
59
|
+
from pandas import Timedelta
|
|
56
60
|
|
|
57
|
-
from osekit.core.spectro_dataset import SpectroDataset
|
|
58
61
|
from osekit.core.audio_dataset import AudioDataset
|
|
59
|
-
from osekit.
|
|
62
|
+
from osekit.core.spectro_dataset import SpectroDataset
|
|
63
|
+
from osekit.public import export_transform
|
|
60
64
|
|
|
61
65
|
# Some Public API imports are required
|
|
62
66
|
from osekit.public.transform import OutputType
|
|
63
|
-
from osekit.
|
|
67
|
+
from osekit.utils.job import Job, JobConfig
|
|
64
68
|
|
|
65
|
-
ads = AudioDataset(...)
|
|
66
|
-
sds = SpectroDataset(...)
|
|
69
|
+
ads = AudioDataset(...) # See the AudioDataset doc
|
|
70
|
+
sds = SpectroDataset(...) # See the SpectroDataset doc
|
|
67
71
|
|
|
68
72
|
# We must specify the folder in which the files will be exported
|
|
69
73
|
# This is an example with both audio and spectro exports.
|
|
@@ -71,28 +75,31 @@ and follow the console arguments of the :mod:`osekit.public.export` script.
|
|
|
71
75
|
sds.folder = Path(...)
|
|
72
76
|
|
|
73
77
|
# Datasets must be serialized
|
|
74
|
-
ads.write_json(ads.folder/"output")
|
|
75
|
-
sds.write_json(sds.folder/"output")
|
|
78
|
+
ads.write_json(ads.folder / "output")
|
|
79
|
+
sds.write_json(sds.folder / "output")
|
|
76
80
|
|
|
77
81
|
# Export specifications
|
|
78
82
|
# All parameters are listed in this example, but all parameters other than transform have default values
|
|
79
83
|
args = {
|
|
80
|
-
"output_type": (OutputType.AUDIO|OutputType.SPECTROGRAM).value,
|
|
81
|
-
"ads-json": ads.foler/"output"/f"{ads.name}.json",
|
|
82
|
-
"sds-json": sds.foler/"output"/f"{sds.name}.json",
|
|
84
|
+
"output_type": (OutputType.AUDIO | OutputType.SPECTROGRAM).value,
|
|
85
|
+
"ads-json": ads.foler / "output" / f"{ads.name}.json",
|
|
86
|
+
"sds-json": sds.foler / "output" / f"{sds.name}.json",
|
|
83
87
|
"subtype": "FLOAT",
|
|
84
|
-
"spectrum-folder-path": "None",
|
|
85
|
-
"spectrogram-folder-path": sds.folder
|
|
88
|
+
"spectrum-folder-path": "None", # Folder in which npz matrices are exported
|
|
89
|
+
"spectrogram-folder-path": sds.folder
|
|
90
|
+
/ "output", # Folder in which png spectrograms are exported
|
|
86
91
|
"welch-folder-path": "None", # Folder in which npz welch matrices are exported
|
|
87
|
-
"first": 0,
|
|
88
|
-
"last": len(
|
|
92
|
+
"first": 0, # First data of the dataset to be exported
|
|
93
|
+
"last": len(
|
|
94
|
+
ads.data,
|
|
95
|
+
), # Last data of the dataset to be exported, up to the last one if not included
|
|
89
96
|
"downsampling-quality": "HQ",
|
|
90
97
|
"upsampling-quality": "VHQ",
|
|
91
98
|
"umask": 0o022,
|
|
92
|
-
"tqdm-disable": "False",
|
|
99
|
+
"tqdm-disable": "False", # Disable TQDM progress bars
|
|
93
100
|
"multiprocessing": "True",
|
|
94
101
|
"nb-processes": "None", # Should be a string. "None" uses the max number of processes, otherwise e.g. "3" will use 3.
|
|
95
|
-
"use-logging-setup": "True",
|
|
102
|
+
"use-logging-setup": "True", # Call osekit.setup_logging() before exporting the dataset.
|
|
96
103
|
}
|
|
97
104
|
|
|
98
105
|
# Job and server configuration
|
|
@@ -102,15 +109,15 @@ and follow the console arguments of the :mod:`osekit.public.export` script.
|
|
|
102
109
|
mem="60gb",
|
|
103
110
|
walltime=Timedelta(hours=1),
|
|
104
111
|
venv_name=os.environ["CONDA_DEFAULT_ENV"],
|
|
105
|
-
queue="omp"
|
|
112
|
+
queue="omp",
|
|
106
113
|
)
|
|
107
114
|
|
|
108
115
|
job = Job(
|
|
109
|
-
script_path
|
|
116
|
+
script_path=Path(export_transform.__file__),
|
|
110
117
|
script_args=args,
|
|
111
118
|
config=job_config,
|
|
112
119
|
name="test_job_core",
|
|
113
|
-
output_folder=Path(...),
|
|
120
|
+
output_folder=Path(...), # Path in which the .out and .err files are written
|
|
114
121
|
)
|
|
115
122
|
|
|
116
123
|
# Write the PBS file and submit the job
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "osekit"
|
|
3
|
-
version = "1.0.
|
|
3
|
+
version = "1.0.1"
|
|
4
4
|
description = "OSEkit"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "MIT"
|
|
@@ -22,7 +22,7 @@ dependencies = [
|
|
|
22
22
|
"tqdm>=4.67.1",
|
|
23
23
|
]
|
|
24
24
|
classifiers = [
|
|
25
|
-
"Development Status ::
|
|
25
|
+
"Development Status :: 5 - Production/Stable",
|
|
26
26
|
"Programming Language :: Python :: 3.12",
|
|
27
27
|
"Programming Language :: Python :: 3.13",
|
|
28
28
|
]
|
|
@@ -165,7 +165,12 @@ class AudioDataset(BaseDataset[AudioData, AudioFile]):
|
|
|
165
165
|
The list of deserialized ``AudioData`` objects.
|
|
166
166
|
|
|
167
167
|
"""
|
|
168
|
-
|
|
168
|
+
ads = []
|
|
169
|
+
for name, value in dictionary.items():
|
|
170
|
+
ad = AudioData.from_dict(value)
|
|
171
|
+
ad.name = name
|
|
172
|
+
ads.append(ad)
|
|
173
|
+
return ads
|
|
169
174
|
|
|
170
175
|
@classmethod
|
|
171
176
|
def from_folder( # noqa: PLR0913
|
|
@@ -161,18 +161,17 @@ class BaseData[TItem: BaseItem, TFile: BaseFile](Event, ABC):
|
|
|
161
161
|
"""Abstract method for writing data to file."""
|
|
162
162
|
|
|
163
163
|
@abstractmethod
|
|
164
|
-
def link(self,
|
|
165
|
-
"""Abstract method for linking data to a file in a given
|
|
164
|
+
def link(self, file: Path) -> None:
|
|
165
|
+
"""Abstract method for linking data to a file in a given file.
|
|
166
166
|
|
|
167
167
|
Linking is intended for data objects that have been written to disk.
|
|
168
168
|
After linking the data to the written file, it will have a single
|
|
169
|
-
item that matches the File properties.
|
|
170
|
-
The folder should contain a file named as ``str(self).extension``.
|
|
169
|
+
item that matches the ``File`` properties.
|
|
171
170
|
|
|
172
171
|
Parameters
|
|
173
172
|
----------
|
|
174
|
-
|
|
175
|
-
|
|
173
|
+
file: Path
|
|
174
|
+
File to which the ``BaseData`` instance should be linked.
|
|
176
175
|
|
|
177
176
|
"""
|
|
178
177
|
|
|
@@ -130,6 +130,15 @@ class BaseDataset[TData: BaseData, TFile: BaseFile](Event, ABC):
|
|
|
130
130
|
else next(iter(file.path.parent for file in self.files), None)
|
|
131
131
|
)
|
|
132
132
|
|
|
133
|
+
@property
|
|
134
|
+
def data(self) -> list[TData]:
|
|
135
|
+
"""List of Data contained in this Dataset."""
|
|
136
|
+
return sorted(self._data, key=lambda d: d.begin)
|
|
137
|
+
|
|
138
|
+
@data.setter
|
|
139
|
+
def data(self, data: list[TData]) -> None:
|
|
140
|
+
self._data = data
|
|
141
|
+
|
|
133
142
|
@folder.setter
|
|
134
143
|
def folder(self, folder: Path) -> None:
|
|
135
144
|
"""Set the folder in which the dataset files might be written.
|
|
@@ -148,7 +148,10 @@ class BaseFile(Event, ABC):
|
|
|
148
148
|
path = serialized["path"]
|
|
149
149
|
return cls(
|
|
150
150
|
path=path,
|
|
151
|
-
|
|
151
|
+
begin=strptime_from_text(
|
|
152
|
+
text=serialized["begin"],
|
|
153
|
+
datetime_template=TIMESTAMP_FORMATS_EXPORTED_FILES,
|
|
154
|
+
),
|
|
152
155
|
)
|
|
153
156
|
|
|
154
157
|
def __hash__(self) -> int:
|
|
@@ -21,9 +21,6 @@ from matplotlib.dates import date2num
|
|
|
21
21
|
from pandas import Timedelta
|
|
22
22
|
from scipy.signal import ShortTimeFFT, welch
|
|
23
23
|
|
|
24
|
-
from osekit.config import (
|
|
25
|
-
TIMESTAMP_FORMATS_EXPORTED_FILES,
|
|
26
|
-
)
|
|
27
24
|
from osekit.core.audio_data import AudioData
|
|
28
25
|
from osekit.core.base_data import BaseData, TFile
|
|
29
26
|
from osekit.core.spectro_file import SpectroFile
|
|
@@ -602,12 +599,11 @@ class SpectroData(BaseData[SpectroItem, SpectroFile]):
|
|
|
602
599
|
timestamps="_".join(timestamps),
|
|
603
600
|
)
|
|
604
601
|
if link:
|
|
605
|
-
self.link(
|
|
602
|
+
self.link(file=folder / f"{self}.npz")
|
|
606
603
|
|
|
607
|
-
def link(self,
|
|
608
|
-
"""Link the ``SpectroData`` to a ``SpectroFile
|
|
604
|
+
def link(self, file: Path) -> None:
|
|
605
|
+
"""Link the ``SpectroData`` to a ``SpectroFile``.
|
|
609
606
|
|
|
610
|
-
The given folder should contain a file named ``"str(self).npz"``.
|
|
611
607
|
Linking is intended for ``SpectroData`` objects that have already been
|
|
612
608
|
written to disk.
|
|
613
609
|
After linking, the ``SpectroData`` will have a single item with the same
|
|
@@ -615,14 +611,13 @@ class SpectroData(BaseData[SpectroItem, SpectroFile]):
|
|
|
615
611
|
|
|
616
612
|
Parameters
|
|
617
613
|
----------
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
instance should be linked.
|
|
614
|
+
file: Path
|
|
615
|
+
File to which the ``SpectroData`` instance should be linked.
|
|
621
616
|
|
|
622
617
|
"""
|
|
623
618
|
file = SpectroFile(
|
|
624
|
-
path=
|
|
625
|
-
|
|
619
|
+
path=file,
|
|
620
|
+
begin=self.begin,
|
|
626
621
|
)
|
|
627
622
|
self.items = SpectroData.from_files([file]).items
|
|
628
623
|
|
|
@@ -504,18 +504,19 @@ class SpectroDataset(BaseDataset[SpectroData, SpectroFile]):
|
|
|
504
504
|
)
|
|
505
505
|
for name, sft in dictionary["sft"].items()
|
|
506
506
|
]
|
|
507
|
-
|
|
508
|
-
|
|
507
|
+
sds = []
|
|
508
|
+
for name, params in dictionary["data"].items():
|
|
509
|
+
sd = cls.data_cls.from_dict(
|
|
509
510
|
params,
|
|
510
511
|
sft=next(sft for sft, linked_data in sfts if name in linked_data),
|
|
511
512
|
)
|
|
512
|
-
|
|
513
|
-
|
|
513
|
+
sd.name = name
|
|
514
|
+
sds.append(sd)
|
|
514
515
|
scale = dictionary["scale"]
|
|
515
516
|
if dictionary["scale"] is not None:
|
|
516
517
|
scale = Scale.from_dict_value(scale)
|
|
517
518
|
return cls(
|
|
518
|
-
data=
|
|
519
|
+
data=sds,
|
|
519
520
|
name=dictionary["name"],
|
|
520
521
|
suffix=dictionary["suffix"],
|
|
521
522
|
folder=Path(dictionary["folder"]),
|
|
@@ -46,6 +46,8 @@ class JobConfig:
|
|
|
46
46
|
Number of nodes on which the job runs.
|
|
47
47
|
ncpus: int
|
|
48
48
|
Number of total cores used per node.
|
|
49
|
+
ngpus: int | None
|
|
50
|
+
Number of total GPU used per node.
|
|
49
51
|
mem: str
|
|
50
52
|
Maximum amount of physical memory used by the job.
|
|
51
53
|
walltime: str | Timedelta
|
|
@@ -59,9 +61,10 @@ class JobConfig:
|
|
|
59
61
|
|
|
60
62
|
nb_nodes: int = 1
|
|
61
63
|
ncpus: int = 2
|
|
64
|
+
ngpus: int | None = None
|
|
62
65
|
mem: str = "8gb"
|
|
63
66
|
walltime: str | Timedelta = "01:00:00"
|
|
64
|
-
venv_name: str = "
|
|
67
|
+
venv_name: str = "osekit"
|
|
65
68
|
queue: Literal["omp", "mpi"] = "omp"
|
|
66
69
|
|
|
67
70
|
|
|
@@ -97,6 +100,7 @@ class Job:
|
|
|
97
100
|
self.script_args = script_args if script_args else {}
|
|
98
101
|
self.nb_nodes = config.nb_nodes
|
|
99
102
|
self.ncpus = config.ncpus
|
|
103
|
+
self.ngpus = config.ngpus
|
|
100
104
|
self.mem = config.mem
|
|
101
105
|
self.walltime = config.walltime
|
|
102
106
|
self.venv_name = config.venv_name
|
|
@@ -144,6 +148,15 @@ class Job:
|
|
|
144
148
|
def ncpus(self, ncpus: int) -> None:
|
|
145
149
|
self._ncpus = ncpus
|
|
146
150
|
|
|
151
|
+
@property
|
|
152
|
+
def ngpus(self) -> int:
|
|
153
|
+
"""Number of total GPU used per node."""
|
|
154
|
+
return self._ngpus
|
|
155
|
+
|
|
156
|
+
@ngpus.setter
|
|
157
|
+
def ngpus(self, ngpus: int) -> None:
|
|
158
|
+
self._ngpus = ngpus
|
|
159
|
+
|
|
147
160
|
@property
|
|
148
161
|
def mem(self) -> str:
|
|
149
162
|
"""Maximum amount of physical memory used by the job."""
|
|
@@ -283,11 +296,21 @@ class Job:
|
|
|
283
296
|
|
|
284
297
|
"""
|
|
285
298
|
preamble = "#!/bin/bash"
|
|
299
|
+
|
|
300
|
+
select_parts = {
|
|
301
|
+
"select": self.nb_nodes,
|
|
302
|
+
"ncpus": self.ncpus,
|
|
303
|
+
"mem": self.mem,
|
|
304
|
+
}
|
|
305
|
+
if self.ngpus is not None:
|
|
306
|
+
select_parts["ngpus"] = self.ngpus
|
|
307
|
+
select_str = ":".join(f"{k}={v}" for k, v in select_parts.items())
|
|
308
|
+
|
|
286
309
|
request = {
|
|
287
310
|
"-N": self.name,
|
|
288
311
|
"-q": self.queue,
|
|
289
312
|
"-l": [
|
|
290
|
-
|
|
313
|
+
select_str,
|
|
291
314
|
f"walltime={self.walltime_str}",
|
|
292
315
|
],
|
|
293
316
|
"-o": f"{self.output_folder}/{self.name}.out"
|
|
@@ -70,6 +70,7 @@ def test_properties_and_venv_activation() -> None:
|
|
|
70
70
|
assert job.script_args == {"purple": "bottle"}
|
|
71
71
|
assert job.nb_nodes == nb_nodes
|
|
72
72
|
assert job.ncpus == ncpus
|
|
73
|
+
assert job.ngpus is None
|
|
73
74
|
assert job.mem == "16gb"
|
|
74
75
|
assert job.walltime == Timedelta(hours=2)
|
|
75
76
|
assert job.venv_name == "merriweather"
|
|
@@ -138,7 +139,7 @@ def test_write_pbs(tmp_path: Path) -> None:
|
|
|
138
139
|
)
|
|
139
140
|
|
|
140
141
|
assert (
|
|
141
|
-
". /appli/anaconda/latest/etc/profile.d/conda.sh; conda activate
|
|
142
|
+
". /appli/anaconda/latest/etc/profile.d/conda.sh; conda activate osekit"
|
|
142
143
|
in content
|
|
143
144
|
)
|
|
144
145
|
last = content[-1]
|
|
@@ -152,6 +153,29 @@ def test_write_pbs(tmp_path: Path) -> None:
|
|
|
152
153
|
assert job.status == JobStatus.PREPARED
|
|
153
154
|
|
|
154
155
|
|
|
156
|
+
def test_write_pbs_job_with_gpu(tmp_path: Path) -> None:
|
|
157
|
+
script = tmp_path / "deville.py"
|
|
158
|
+
script.write_text("print('cruella')")
|
|
159
|
+
output_dir = tmp_path / "output"
|
|
160
|
+
output_dir.mkdir()
|
|
161
|
+
|
|
162
|
+
job = Job(
|
|
163
|
+
script_path=script,
|
|
164
|
+
script_args={"cruelle": "diablesse"},
|
|
165
|
+
name="penny",
|
|
166
|
+
config=JobConfig(ngpus=2),
|
|
167
|
+
output_folder=output_dir,
|
|
168
|
+
)
|
|
169
|
+
pbs_path = tmp_path / "patch.pbs"
|
|
170
|
+
job.write_pbs(pbs_path)
|
|
171
|
+
|
|
172
|
+
content = pbs_path.read_text().splitlines()
|
|
173
|
+
assert any("select=1:ncpus=2:mem=8gb:ngpus=2" in line for line in content)
|
|
174
|
+
last = content[-1]
|
|
175
|
+
assert last.startswith(f"python {script}")
|
|
176
|
+
assert "--cruelle diablesse" in last
|
|
177
|
+
|
|
178
|
+
|
|
155
179
|
def test_submit_pbs_without_write_raises() -> None:
|
|
156
180
|
job = Job(Path("script.py"))
|
|
157
181
|
with pytest.raises(
|
|
@@ -618,11 +618,11 @@ def test_serialization(
|
|
|
618
618
|
id="spectro_only",
|
|
619
619
|
),
|
|
620
620
|
pytest.param(
|
|
621
|
-
OutputType.
|
|
621
|
+
OutputType.SPECTRUM,
|
|
622
622
|
id="spectrum_only",
|
|
623
623
|
),
|
|
624
624
|
pytest.param(
|
|
625
|
-
OutputType.SPECTROGRAM,
|
|
625
|
+
OutputType.SPECTRUM | OutputType.SPECTROGRAM,
|
|
626
626
|
id="both_spectral_flags",
|
|
627
627
|
),
|
|
628
628
|
],
|
|
@@ -1169,8 +1169,8 @@ def test_prepare_spectro(
|
|
|
1169
1169
|
|
|
1170
1170
|
|
|
1171
1171
|
def test_edit_transform_before_run(
|
|
1172
|
-
tmp_path:
|
|
1173
|
-
audio_files:
|
|
1172
|
+
tmp_path: Path,
|
|
1173
|
+
audio_files: None,
|
|
1174
1174
|
) -> None:
|
|
1175
1175
|
project = Project(
|
|
1176
1176
|
folder=tmp_path,
|
|
@@ -1181,10 +1181,11 @@ def test_edit_transform_before_run(
|
|
|
1181
1181
|
project.build()
|
|
1182
1182
|
|
|
1183
1183
|
transform = Transform(
|
|
1184
|
-
output_type=OutputType.AUDIO | OutputType.SPECTROGRAM,
|
|
1184
|
+
output_type=OutputType.AUDIO | OutputType.SPECTRUM | OutputType.SPECTROGRAM,
|
|
1185
1185
|
data_duration=project.origin_dataset.duration / 2,
|
|
1186
1186
|
name="original_transform",
|
|
1187
1187
|
sample_rate=24_000,
|
|
1188
|
+
v_lim=(0.0, 120.0),
|
|
1188
1189
|
fft=ShortTimeFFT(win=hamming(1024), hop=1024, fs=24_000),
|
|
1189
1190
|
)
|
|
1190
1191
|
|
|
@@ -1205,7 +1206,14 @@ def test_edit_transform_before_run(
|
|
|
1205
1206
|
ads.data = new_data
|
|
1206
1207
|
ads.normalization = new_normalization
|
|
1207
1208
|
|
|
1208
|
-
|
|
1209
|
+
# Spectro edits
|
|
1210
|
+
new_v_lim = (50.0, 100.0)
|
|
1211
|
+
sds = project.prepare_spectro(transform=transform, audio_dataset=ads)
|
|
1212
|
+
sds.v_lim = new_v_lim
|
|
1213
|
+
for idx, sd in enumerate(sds.data):
|
|
1214
|
+
sd.name = str(idx)
|
|
1215
|
+
|
|
1216
|
+
project.run(transform, audio_dataset=ads, spectro_dataset=sds)
|
|
1209
1217
|
|
|
1210
1218
|
# New ads name
|
|
1211
1219
|
assert (project.folder / "data" / "audio" / ads.name).exists()
|
|
@@ -1234,6 +1242,10 @@ def test_edit_transform_before_run(
|
|
|
1234
1242
|
# Instrument has been edited
|
|
1235
1243
|
assert output_ads.instrument.end_to_end_db == new_instrument.end_to_end_db
|
|
1236
1244
|
|
|
1245
|
+
# Spectro data have been edited
|
|
1246
|
+
assert output_sds.v_lim == new_v_lim
|
|
1247
|
+
assert all(sd.name == str(i) for i, sd in enumerate(output_sds.data))
|
|
1248
|
+
|
|
1237
1249
|
|
|
1238
1250
|
def test_delete_output_dataset(
|
|
1239
1251
|
tmp_path: pytest.fixture,
|
|
@@ -280,6 +280,10 @@ def test_audio_dataset_serialization(
|
|
|
280
280
|
name=name,
|
|
281
281
|
)
|
|
282
282
|
|
|
283
|
+
# NO TIMESTAMP IN THE NAME SHOULD STILL BE DESERIALIZED PROPERLY
|
|
284
|
+
for idx, ad in list(enumerate(ads.data))[::2]:
|
|
285
|
+
ad.name = str(idx)
|
|
286
|
+
|
|
283
287
|
assert ads.begin == begin
|
|
284
288
|
|
|
285
289
|
if type(sample_rate) is list:
|
|
@@ -308,10 +312,11 @@ def test_audio_dataset_serialization(
|
|
|
308
312
|
assert ads.begin == ads2.begin
|
|
309
313
|
assert ads.normalization == ads2.normalization
|
|
310
314
|
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
+
zipped = zip(ads.data, ads2.data, strict=True)
|
|
316
|
+
|
|
317
|
+
assert all(ad1.name == ad2.name for ad1, ad2 in zipped)
|
|
318
|
+
|
|
319
|
+
assert all(np.array_equal(ad.get_value(), ad2.get_value()) for ad, ad2 in zipped)
|
|
315
320
|
|
|
316
321
|
|
|
317
322
|
@pytest.mark.parametrize(
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{osekit-1.0.0 → osekit-1.0.1}/docs/source/_static/sample_audio/timestamped/sample_220925_223450.wav
RENAMED
|
File without changes
|
{osekit-1.0.0 → osekit-1.0.1}/docs/source/_static/sample_audio/timestamped/sample_220925_223500.wav
RENAMED
|
File without changes
|
{osekit-1.0.0 → osekit-1.0.1}/docs/source/_static/sample_audio/timestamped/sample_220925_223510.wav
RENAMED
|
File without changes
|
{osekit-1.0.0 → osekit-1.0.1}/docs/source/_static/sample_audio/timestamped/sample_220925_223520.wav
RENAMED
|
File without changes
|
{osekit-1.0.0 → osekit-1.0.1}/docs/source/_static/sample_audio/timestamped/sample_220925_223530.wav
RENAMED
|
File without changes
|
{osekit-1.0.0 → osekit-1.0.1}/docs/source/_static/sample_audio/timestamped/sample_220925_223600.wav
RENAMED
|
File without changes
|
{osekit-1.0.0 → osekit-1.0.1}/docs/source/_static/sample_audio/timestamped/sample_220925_223610.wav
RENAMED
|
File without changes
|
{osekit-1.0.0 → osekit-1.0.1}/docs/source/_static/sample_audio/timestamped/sample_220925_223620.wav
RENAMED
|
File without changes
|
{osekit-1.0.0 → osekit-1.0.1}/docs/source/_static/sample_audio/timestamped/sample_220925_223630.wav
RENAMED
|
File without changes
|
{osekit-1.0.0 → osekit-1.0.1}/docs/source/_static/sample_audio/timestamped/sample_220925_223640.wav
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|