osekit 1.4.0__tar.gz → 1.5.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {osekit-1.4.0 → osekit-1.5.1}/CITATION.cff +1 -1
- {osekit-1.4.0 → osekit-1.5.1}/PKG-INFO +1 -1
- osekit-1.5.1/docs/source/job.rst +27 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/jobs.rst +41 -23
- {osekit-1.4.0 → osekit-1.5.1}/pyproject.toml +1 -1
- {osekit-1.4.0 → osekit-1.5.1}/src/osekit/core/event.py +14 -19
- osekit-1.5.1/src/osekit/job/builder.py +153 -0
- osekit-1.5.1/src/osekit/job/config.py +32 -0
- osekit-1.5.1/src/osekit/job/job.py +237 -0
- osekit-1.5.1/src/osekit/job/scheduler/pbs.py +209 -0
- osekit-1.5.1/src/osekit/job/scheduler/scheduler.py +301 -0
- osekit-1.5.1/src/osekit/job/scheduler/slurm.py +267 -0
- {osekit-1.4.0 → osekit-1.5.1}/src/osekit/public/project.py +2 -2
- osekit-1.4.0/tests/sample_dataset/sample1_180701_204453.wav → osekit-1.5.1/tests/__init__.py +0 -0
- osekit-1.5.1/tests/_static/job_status_request_results/pbs.txt +39 -0
- osekit-1.5.1/tests/_static/job_status_request_results/slurm_sacct.txt +1 -0
- osekit-1.5.1/tests/_static/job_status_request_results/slurm_squeue.txt +1 -0
- osekit-1.4.0/tests/sample_dataset/sample1_190222_010000.wav → osekit-1.5.1/tests/helpers/__init__.py +0 -0
- osekit-1.5.1/tests/sample_dataset/sample1_190222_010000.wav +0 -0
- osekit-1.5.1/tests/sample_dataset/sample1_220998_185532.wav +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/tests/test_event.py +31 -0
- {osekit-1.4.0 → osekit-1.5.1}/tests/test_export_transform.py +2 -2
- osekit-1.5.1/tests/test_job.py +973 -0
- {osekit-1.4.0 → osekit-1.5.1}/uv.lock +248 -215
- osekit-1.4.0/docs/source/job.rst +0 -7
- osekit-1.4.0/src/osekit/core_api/__init__.py +0 -12
- osekit-1.4.0/src/osekit/public_api/__init__.py +0 -12
- osekit-1.4.0/src/osekit/utils/job.py +0 -635
- osekit-1.4.0/tests/test_job.py +0 -799
- {osekit-1.4.0 → osekit-1.5.1}/.github/dependabot.yml +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/.github/workflows/dependabot-auto-merge.yml +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/.github/workflows/documentation.yml +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/.github/workflows/github_ci.yml +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/.github/workflows/publish.yml +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/.gitignore +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/.pre-commit-config.yaml +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/CHANGELOG.md +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/CONTRIBUTING.md +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/LICENSE +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/README.md +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/Makefile +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/logo/osekit.png +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/logo/osekit_mid.png +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/logo/osekit_small.png +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/logo/osmose.png +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/logo/osmose_texte_sombre.png +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/logo/osmose_texte_sombre_small.png +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/make.bat +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/audio_data/ex2.svg +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/audio_data/ex_1.svg +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/audio_data/ex_2.svg +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/custom.css +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/data_structure/data_structure.svg +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/data_structure/optimized_IO.svg +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/detections/aplose_results.csv +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/detections/detection_labelled.png +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/detections/detection_unlabelled.png +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/detections/label_anchors.png +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/favicon.ico +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/ltas/ltas.svg +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/sample_audio/id/1829.wav +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/sample_audio/id/1830.wav +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/sample_audio/id/1832.wav +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/sample_audio/id/1833.wav +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/sample_audio/id/1834.wav +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/sample_audio/id/1835.wav +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/sample_audio/id/1836.wav +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/sample_audio/id/1837.wav +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/sample_audio/id/1838.wav +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/sample_audio/multichannel/multichannel_220925_223450.wav +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/sample_audio/timestamped/sample_220925_223450.wav +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/sample_audio/timestamped/sample_220925_223500.wav +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/sample_audio/timestamped/sample_220925_223510.wav +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/sample_audio/timestamped/sample_220925_223520.wav +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/sample_audio/timestamped/sample_220925_223530.wav +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/sample_audio/timestamped/sample_220925_223600.wav +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/sample_audio/timestamped/sample_220925_223610.wav +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/sample_audio/timestamped/sample_220925_223620.wav +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/sample_audio/timestamped/sample_220925_223630.wav +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/sample_audio/timestamped/sample_220925_223640.wav +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/spectrograms/frequency_scale.svg +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/api.rst +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/aplose.rst +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/audiodata.rst +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/audiodataset.rst +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/audiofile.rst +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/audiofilemanager.rst +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/audioitem.rst +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/basedata.rst +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/basedataset.rst +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/basefile.rst +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/baseitem.rst +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/conf.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/contributing.rst +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/coreapi.rst +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/coreapi_home.rst +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/coreapi_introduction.rst +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/coreapi_usage.rst +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/detection.rst +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/event.rst +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/example_aplose_result.ipynb +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/example_ltas.rst +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/example_ltas_core.ipynb +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/example_ltas_public.ipynb +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/example_multichannel.ipynb +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/example_multiple_spectrograms.rst +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/example_multiple_spectrograms_core.ipynb +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/example_multiple_spectrograms_id.rst +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/example_multiple_spectrograms_id_core.ipynb +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/example_multiple_spectrograms_id_public.ipynb +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/example_multiple_spectrograms_public.ipynb +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/example_reshaping_multiple_files.rst +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/example_reshaping_multiple_files_core.ipynb +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/example_reshaping_multiple_files_public.ipynb +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/example_reshaping_one_file.ipynb +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/example_spectrogram.ipynb +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/examples.rst +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/export_transform.rst +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/frequencyscale.rst +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/index.rst +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/installation.rst +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/instrument.rst +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/ltasdata.rst +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/multichannel.rst +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/multiprocessing.rst +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/normalization.rst +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/project.rst +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/publicapi.rst +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/publicapi_usage.rst +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/spectrodata.rst +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/spectrodataset.rst +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/spectrofile.rst +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/spectroitem.rst +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/transform.rst +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/usage.rst +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/docs/source/utils.rst +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/src/osekit/__init__.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/src/osekit/audio_backend/audio_backend.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/src/osekit/audio_backend/audio_file_manager.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/src/osekit/audio_backend/mseed_backend.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/src/osekit/audio_backend/soundfile_backend.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/src/osekit/config.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/src/osekit/core/__init__.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/src/osekit/core/audio_data.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/src/osekit/core/audio_dataset.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/src/osekit/core/audio_file.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/src/osekit/core/audio_item.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/src/osekit/core/base_data.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/src/osekit/core/base_dataset.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/src/osekit/core/base_file.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/src/osekit/core/base_item.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/src/osekit/core/detection.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/src/osekit/core/frequency_scale.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/src/osekit/core/instrument.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/src/osekit/core/json_serializer.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/src/osekit/core/ltas_data.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/src/osekit/core/ltas_dataset.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/src/osekit/core/spectro_data.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/src/osekit/core/spectro_dataset.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/src/osekit/core/spectro_file.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/src/osekit/core/spectro_item.py +0 -0
- {osekit-1.4.0/src/osekit/public → osekit-1.5.1/src/osekit/job}/__init__.py +0 -0
- {osekit-1.4.0/src/osekit/utils → osekit-1.5.1/src/osekit/job/scheduler}/__init__.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/src/osekit/logging_config.yaml +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/src/osekit/logging_context.py +0 -0
- {osekit-1.4.0/tests → osekit-1.5.1/src/osekit/public}/__init__.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/src/osekit/public/export_transform.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/src/osekit/public/transform.py +0 -0
- {osekit-1.4.0/tests/helpers → osekit-1.5.1/src/osekit/utils}/__init__.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/src/osekit/utils/aplose.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/src/osekit/utils/audio.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/src/osekit/utils/core.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/src/osekit/utils/deserialization.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/src/osekit/utils/multiprocess.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/src/osekit/utils/path.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/src/osekit/utils/plot.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/src/osekit/utils/timestamp.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/tests/_static/aplose_result.csv +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/tests/_static/minimal_detector_result.csv +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/tests/_static/retro_compatibility_result.csv +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/tests/conftest.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/tests/helpers/audio.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/tests/helpers/dummy.py +0 -0
- /osekit-1.4.0/tests/sample_dataset/sample1_220998_185532.wav → /osekit-1.5.1/tests/sample_dataset/sample1_180701_204453.wav +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/tests/test_audio.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/tests/test_audio_file_manager.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/tests/test_core_api_base.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/tests/test_detection.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/tests/test_files.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/tests/test_frequency_scales.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/tests/test_instrument.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/tests/test_logging.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/tests/test_mseed.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/tests/test_multiprocessing.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/tests/test_permissions.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/tests/test_public_api.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/tests/test_serialization.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/tests/test_spectro.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/tests/test_timestamp_utils.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/tests/test_utils.py +0 -0
- {osekit-1.4.0 → osekit-1.5.1}/tests/test_welch.py +0 -0
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
.. _job:
|
|
2
|
+
|
|
3
|
+
Job
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
.. automodule:: osekit.job.job
|
|
7
|
+
:members: Job
|
|
8
|
+
|
|
9
|
+
Job Config
|
|
10
|
+
----------
|
|
11
|
+
|
|
12
|
+
.. automodule:: osekit.job.config
|
|
13
|
+
:members: JobConfig
|
|
14
|
+
|
|
15
|
+
Job Scheduler
|
|
16
|
+
-------------
|
|
17
|
+
|
|
18
|
+
.. automodule:: osekit.job.scheduler.pbs
|
|
19
|
+
:members: Pbs
|
|
20
|
+
.. automodule:: osekit.job.scheduler.slurm
|
|
21
|
+
:members: Slurm
|
|
22
|
+
|
|
23
|
+
Job Builder
|
|
24
|
+
-----------
|
|
25
|
+
|
|
26
|
+
.. automodule:: osekit.job.builder
|
|
27
|
+
:members: JobBuilder
|
|
@@ -2,51 +2,65 @@ Working with jobs
|
|
|
2
2
|
-----------------
|
|
3
3
|
|
|
4
4
|
**OSEkit** can be set to send transform instructions to be computed on a remote server
|
|
5
|
-
through
|
|
5
|
+
through queuing systems.
|
|
6
6
|
|
|
7
7
|
This feature has mainly be thought for the Public API, but it can nonetheless be used for
|
|
8
8
|
any Core API operation.
|
|
9
9
|
|
|
10
|
-
|
|
10
|
+
This is done thanks to the job package, located in :mod:`osekit.job`.
|
|
11
11
|
|
|
12
12
|
Public API
|
|
13
13
|
^^^^^^^^^^
|
|
14
14
|
|
|
15
|
-
Running Public API Analyses through
|
|
16
|
-
instance to the :attr:`osekit.public.project.Project.job_builder` attribute
|
|
15
|
+
Running Public API Analyses through jobs only requires adding a :class:`osekit.job.builder.JobBuilder`
|
|
16
|
+
instance to the :attr:`osekit.public.project.Project.job_builder` attribute.
|
|
17
|
+
|
|
18
|
+
The :class:`osekit.job.builder.JobBuilder`
|
|
19
|
+
|
|
20
|
+
Here is an example for running a transform on a PBS queue:
|
|
17
21
|
|
|
18
22
|
.. code-block:: python
|
|
19
23
|
|
|
20
|
-
|
|
24
|
+
import os
|
|
25
|
+
|
|
26
|
+
from pandas import Timedelta
|
|
27
|
+
|
|
28
|
+
from osekit.job.builder import JobBuilder
|
|
29
|
+
from osekit.job.config import JobConfig
|
|
30
|
+
from osekit.job.scheduler.pbs import Pbs
|
|
21
31
|
from osekit.public.project import Project
|
|
22
32
|
|
|
23
|
-
project = Project(...)
|
|
33
|
+
project = Project(...) # See the Project documentation
|
|
24
34
|
|
|
25
35
|
job_config = JobConfig(
|
|
26
|
-
nb_nodes=1,
|
|
27
|
-
ncpus=28,
|
|
28
|
-
ngpus=1,
|
|
29
|
-
mem="60gb",
|
|
30
|
-
walltime=Timedelta(
|
|
31
|
-
|
|
32
|
-
|
|
36
|
+
nb_nodes=1, # Number of nodes on which the job runs
|
|
37
|
+
ncpus=28, # Number of total cores used per node
|
|
38
|
+
ngpus=1, # Number of total GPU used per node
|
|
39
|
+
mem="60gb", # Maximum amount of physical memory used by the job
|
|
40
|
+
walltime=Timedelta(
|
|
41
|
+
hours=5
|
|
42
|
+
), # Maximum amount of real itime during which the job can be running
|
|
43
|
+
venv_name=os.environ["CONDA_DEFAULT_ENV"], # Works only for conda venvs
|
|
33
44
|
)
|
|
34
45
|
|
|
46
|
+
scheduler = Pbs(queue="omp") # Scheduler in which the job is submitted
|
|
47
|
+
|
|
35
48
|
project.job_builder = JobBuilder(
|
|
36
49
|
config=job_config,
|
|
50
|
+
scheduler=scheduler,
|
|
37
51
|
)
|
|
38
52
|
|
|
39
53
|
# Now the dataset has a non-None job_builder attribute,
|
|
40
54
|
# running a transform will write a PBS file in the logs directory
|
|
41
|
-
# and submit it
|
|
55
|
+
# and submit it through the selected scheduler.
|
|
42
56
|
|
|
43
|
-
project.run(...)
|
|
57
|
+
project.run(...) # See the Transform documentation
|
|
44
58
|
|
|
45
59
|
|
|
46
60
|
Core API
|
|
47
61
|
^^^^^^^^
|
|
48
62
|
|
|
49
|
-
Exporting Core API datasets with jobs is doable by explicitly instantiating a :class:`osekit.
|
|
63
|
+
Exporting Core API datasets with jobs is doable by explicitly instantiating a :class:`osekit.job.job.Job` object.
|
|
50
64
|
|
|
51
65
|
The export parameters are specified in the ``script_args`` parameter of the ``Job`` constructor,
|
|
52
66
|
and follow the console arguments of the :mod:`osekit.public.export` script.
|
|
@@ -60,11 +74,13 @@ and follow the console arguments of the :mod:`osekit.public.export` script.
|
|
|
60
74
|
|
|
61
75
|
from osekit.core.audio_dataset import AudioDataset
|
|
62
76
|
from osekit.core.spectro_dataset import SpectroDataset
|
|
77
|
+
from osekit.job.config import JobConfig
|
|
78
|
+
from osekit.job.job import Job
|
|
79
|
+
from osekit.job.scheduler.pbs import Pbs
|
|
63
80
|
from osekit.public import export_transform
|
|
64
81
|
|
|
65
82
|
# Some Public API imports are required
|
|
66
83
|
from osekit.public.transform import OutputType
|
|
67
|
-
from osekit.utils.job import Job, JobConfig
|
|
68
84
|
|
|
69
85
|
ads = AudioDataset(...) # See the AudioDataset doc
|
|
70
86
|
sds = SpectroDataset(...) # See the SpectroDataset doc
|
|
@@ -109,9 +125,11 @@ and follow the console arguments of the :mod:`osekit.public.export` script.
|
|
|
109
125
|
mem="60gb",
|
|
110
126
|
walltime=Timedelta(hours=1),
|
|
111
127
|
venv_name=os.environ["CONDA_DEFAULT_ENV"],
|
|
112
|
-
queue="omp",
|
|
113
128
|
)
|
|
114
129
|
|
|
130
|
+
# Scheduler configuration
|
|
131
|
+
scheduler = Pbs(queue="omp")
|
|
132
|
+
|
|
115
133
|
job = Job(
|
|
116
134
|
script_path=Path(export_transform.__file__),
|
|
117
135
|
script_args=args,
|
|
@@ -120,12 +138,12 @@ and follow the console arguments of the :mod:`osekit.public.export` script.
|
|
|
120
138
|
output_folder=Path(...), # Path in which the .out and .err files are written
|
|
121
139
|
)
|
|
122
140
|
|
|
123
|
-
# Write the
|
|
124
|
-
|
|
125
|
-
|
|
141
|
+
# Write the job file and submit it through the scheduler
|
|
142
|
+
scheduler.write(job=job, path=Path(...) / f"{job.name}.pbs")
|
|
143
|
+
scheduler.submit(job=job)
|
|
126
144
|
|
|
127
|
-
You can then follow the status of the submitted job:
|
|
145
|
+
You can then follow the status of the submitted job through the scheduler:
|
|
128
146
|
|
|
129
147
|
.. code-block:: python
|
|
130
148
|
|
|
131
|
-
|
|
149
|
+
scheduler.update_status(job=job)
|
|
@@ -5,6 +5,7 @@ from __future__ import annotations
|
|
|
5
5
|
import bisect
|
|
6
6
|
import copy
|
|
7
7
|
from dataclasses import dataclass, field
|
|
8
|
+
from operator import attrgetter
|
|
8
9
|
from typing import TYPE_CHECKING, TypeVar
|
|
9
10
|
|
|
10
11
|
from osekit.utils.timestamp import localize_timestamp
|
|
@@ -40,7 +41,7 @@ class Event:
|
|
|
40
41
|
@begin.setter
|
|
41
42
|
def begin(self, value: Timestamp) -> None:
|
|
42
43
|
if hasattr(self, "_end") and value > self._end:
|
|
43
|
-
msg = f"Invalid Event: `end` ({self._end}) must be greater than `begin` ({value})."
|
|
44
|
+
msg = f"Invalid Event: `end` ({self._end}) must be greater than `begin` ({value})."
|
|
44
45
|
raise ValueError(msg)
|
|
45
46
|
self._begin = value
|
|
46
47
|
|
|
@@ -52,7 +53,7 @@ class Event:
|
|
|
52
53
|
@end.setter
|
|
53
54
|
def end(self, value: Timestamp) -> None:
|
|
54
55
|
if hasattr(self, "_begin") and value < self._begin:
|
|
55
|
-
msg = f"Invalid Event: `end` ({value}) must be greater than `begin` ({self._begin})."
|
|
56
|
+
msg = f"Invalid Event: `end` ({value}) must be greater than `begin` ({self._begin})."
|
|
56
57
|
raise ValueError(msg)
|
|
57
58
|
self._end = value
|
|
58
59
|
|
|
@@ -104,7 +105,7 @@ class Event:
|
|
|
104
105
|
>>> Event(begin=Timestamp("2024-01-01 00:00:00"),end=Timestamp("2024-01-02 00:00:00")).overlaps(Event(begin=Timestamp("2024-01-02 00:00:00"),end=Timestamp("2024-01-02 12:00:00")))
|
|
105
106
|
False
|
|
106
107
|
|
|
107
|
-
"""
|
|
108
|
+
"""
|
|
108
109
|
return self.begin < other.end and self.end > other.begin
|
|
109
110
|
|
|
110
111
|
def get_overlapping_events(self, events: list[TEvent]) -> list[TEvent]:
|
|
@@ -125,19 +126,13 @@ class Event:
|
|
|
125
126
|
|
|
126
127
|
"""
|
|
127
128
|
output = []
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
continue
|
|
136
|
-
if self.overlaps(events[i]):
|
|
137
|
-
output.append(events[i])
|
|
138
|
-
continue
|
|
139
|
-
if events[i].begin >= self.end:
|
|
140
|
-
break
|
|
129
|
+
stop_index = bisect.bisect(a=events, x=self.end, key=attrgetter("begin"))
|
|
130
|
+
|
|
131
|
+
for i in range(stop_index):
|
|
132
|
+
event = events[i]
|
|
133
|
+
if event is not self and self.overlaps(event):
|
|
134
|
+
output.append(event)
|
|
135
|
+
|
|
141
136
|
return output
|
|
142
137
|
|
|
143
138
|
@classmethod
|
|
@@ -170,7 +165,7 @@ class Event:
|
|
|
170
165
|
>>> events[0].end == events[1].begin
|
|
171
166
|
True
|
|
172
167
|
|
|
173
|
-
"""
|
|
168
|
+
"""
|
|
174
169
|
events = sorted(
|
|
175
170
|
events,
|
|
176
171
|
key=lambda event: (event.begin, -1 * event.duration),
|
|
@@ -199,7 +194,7 @@ class Event:
|
|
|
199
194
|
cls,
|
|
200
195
|
events: list[TEvent],
|
|
201
196
|
filling_class: type[TEvent],
|
|
202
|
-
**kwargs,
|
|
197
|
+
**kwargs,
|
|
203
198
|
) -> list[TEvent]:
|
|
204
199
|
"""Return a list with empty events added in the gaps between items.
|
|
205
200
|
|
|
@@ -227,7 +222,7 @@ class Event:
|
|
|
227
222
|
>>> [(event.begin.second, event.end.second) for event in events]
|
|
228
223
|
[(0, 10), (10, 15), (15, 25)]
|
|
229
224
|
|
|
230
|
-
"""
|
|
225
|
+
"""
|
|
231
226
|
events = sorted(
|
|
232
227
|
[copy.copy(event) for event in events],
|
|
233
228
|
key=lambda event: event.begin,
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
|
|
3
|
+
from osekit.job.config import JobConfig
|
|
4
|
+
from osekit.job.job import Job, JobStatus
|
|
5
|
+
from osekit.job.scheduler.pbs import Pbs
|
|
6
|
+
from osekit.job.scheduler.scheduler import Scheduler
|
|
7
|
+
from osekit.utils.core import file_indexes_per_batch
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class JobBuilder:
|
|
11
|
+
"""Class that should be attached to a Public API ``Project`` for working with jobs.
|
|
12
|
+
|
|
13
|
+
If a ``Project`` has a ``JobBuilder``, it will run its transforms through jobs
|
|
14
|
+
using the specified scheduler.
|
|
15
|
+
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
def __init__(
|
|
19
|
+
self,
|
|
20
|
+
config: JobConfig | None = None,
|
|
21
|
+
scheduler: Scheduler | None = None,
|
|
22
|
+
) -> None:
|
|
23
|
+
"""Initialize a ``JobBuilder`` instance.
|
|
24
|
+
|
|
25
|
+
Parameters
|
|
26
|
+
----------
|
|
27
|
+
config: JobConfig
|
|
28
|
+
Config of the jobs built by this job builder.
|
|
29
|
+
scheduler: Scheduler
|
|
30
|
+
Scheduler used to format, write and submit jobs.
|
|
31
|
+
|
|
32
|
+
"""
|
|
33
|
+
self.config = config or JobConfig()
|
|
34
|
+
self.scheduler = scheduler or Pbs()
|
|
35
|
+
self.jobs = []
|
|
36
|
+
|
|
37
|
+
@property
|
|
38
|
+
def jobs(self) -> list[Job]:
|
|
39
|
+
"""Return the jobs created by this job builder."""
|
|
40
|
+
return self._jobs
|
|
41
|
+
|
|
42
|
+
@jobs.setter
|
|
43
|
+
def jobs(self, jobs: list[Job]) -> None:
|
|
44
|
+
self._jobs = jobs
|
|
45
|
+
|
|
46
|
+
def create_jobs(
|
|
47
|
+
self,
|
|
48
|
+
nb_tasks: int,
|
|
49
|
+
script_path: Path,
|
|
50
|
+
script_args: dict,
|
|
51
|
+
output_folder: Path,
|
|
52
|
+
job_name: str = "osekit_transform",
|
|
53
|
+
nb_jobs: int = 1,
|
|
54
|
+
) -> None:
|
|
55
|
+
"""Create the jobs corresponding to each batch.
|
|
56
|
+
|
|
57
|
+
Parameters
|
|
58
|
+
----------
|
|
59
|
+
nb_tasks:
|
|
60
|
+
The number of tasks that are distributed across ``nb_jobs`` jobs.
|
|
61
|
+
script_path: Path
|
|
62
|
+
Path to the export script.
|
|
63
|
+
script_args: dict
|
|
64
|
+
Arguments passed to the export script.
|
|
65
|
+
job_name: str
|
|
66
|
+
Name of the job.
|
|
67
|
+
If there are multiple batches, each batch will be suffixed
|
|
68
|
+
with "_{index}".
|
|
69
|
+
output_folder: Path
|
|
70
|
+
Folder in which the job output log files are saved.
|
|
71
|
+
nb_jobs: int
|
|
72
|
+
Number of batches used to run the transform.
|
|
73
|
+
Each batch will run in a separate job.
|
|
74
|
+
|
|
75
|
+
"""
|
|
76
|
+
batch_indexes = file_indexes_per_batch(
|
|
77
|
+
total_nb_files=nb_tasks,
|
|
78
|
+
nb_batches=nb_jobs,
|
|
79
|
+
)
|
|
80
|
+
for index, (start, stop) in enumerate(batch_indexes):
|
|
81
|
+
self.create_job(
|
|
82
|
+
script_path=script_path,
|
|
83
|
+
script_args=script_args | {"first": start, "last": stop},
|
|
84
|
+
name=job_name + (f"_{index}" if len(batch_indexes) > 1 else ""),
|
|
85
|
+
output_folder=output_folder,
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
def create_job(
|
|
89
|
+
self,
|
|
90
|
+
script_path: Path,
|
|
91
|
+
script_args: dict | None = None,
|
|
92
|
+
name: str = "osekit_transform",
|
|
93
|
+
output_folder: Path | None = None,
|
|
94
|
+
) -> None:
|
|
95
|
+
"""Create a new ``Job`` instance.
|
|
96
|
+
|
|
97
|
+
Parameters
|
|
98
|
+
----------
|
|
99
|
+
script_path: Path
|
|
100
|
+
Path to the script file the job must run.
|
|
101
|
+
script_args: dict | None
|
|
102
|
+
Additional arguments to pass to the script file.
|
|
103
|
+
name: str
|
|
104
|
+
Name of the job.
|
|
105
|
+
output_folder: Path | None
|
|
106
|
+
Folder in which the output files (``.out`` and ``.err``) will be written.
|
|
107
|
+
|
|
108
|
+
"""
|
|
109
|
+
job = Job(
|
|
110
|
+
script_path=script_path,
|
|
111
|
+
script_args=script_args,
|
|
112
|
+
name=name,
|
|
113
|
+
output_folder=output_folder,
|
|
114
|
+
config=self.config,
|
|
115
|
+
)
|
|
116
|
+
self.scheduler.write(
|
|
117
|
+
job=job,
|
|
118
|
+
path=output_folder / f"{name}.{self.scheduler.JOB_FILE_EXTENSION}",
|
|
119
|
+
)
|
|
120
|
+
self.jobs.append(job)
|
|
121
|
+
|
|
122
|
+
def submit(
|
|
123
|
+
self,
|
|
124
|
+
dependencies: dict[Job, dict[str, str | Job | list[str | Job]]] | None = None,
|
|
125
|
+
) -> None:
|
|
126
|
+
"""Submit all prepared jobs to the scheduler system.
|
|
127
|
+
|
|
128
|
+
Parameters
|
|
129
|
+
----------
|
|
130
|
+
dependencies: dict[Job, dict[str, str | Job | list[str | Job]]] | None
|
|
131
|
+
Optional mapping of the jobs dependencies.
|
|
132
|
+
|
|
133
|
+
For each key (which is a job), the value is a
|
|
134
|
+
dictionary explaining its dependencies.
|
|
135
|
+
|
|
136
|
+
Such dictionary follows the following format:
|
|
137
|
+
The keys of the dictionary are the dependency types,
|
|
138
|
+
that are proper to the scheduler.
|
|
139
|
+
The values are the other jobs (or their ID) ``job`` depends on
|
|
140
|
+
with the given dependency type.
|
|
141
|
+
|
|
142
|
+
If ``None``, the jobs are submitted without any dependency.
|
|
143
|
+
|
|
144
|
+
"""
|
|
145
|
+
dependencies = dependencies or {}
|
|
146
|
+
for job in self.jobs:
|
|
147
|
+
if self.scheduler.update_status(job=job) is not JobStatus.PREPARED:
|
|
148
|
+
continue
|
|
149
|
+
|
|
150
|
+
# Check if this job has dependencies
|
|
151
|
+
job_dependencies = dependencies.get(job, None)
|
|
152
|
+
|
|
153
|
+
self.scheduler.submit(job=job, dependencies=job_dependencies)
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
|
|
3
|
+
from pandas import Timedelta
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@dataclass
|
|
7
|
+
class JobConfig:
|
|
8
|
+
"""Configuration of the computing resources allowed for a job.
|
|
9
|
+
|
|
10
|
+
Parameters
|
|
11
|
+
----------
|
|
12
|
+
nb_nodes: int
|
|
13
|
+
Number of nodes on which the job runs.
|
|
14
|
+
ncpus: int
|
|
15
|
+
Number of total cores used per node.
|
|
16
|
+
ngpus: int | None
|
|
17
|
+
Number of total GPU used per node.
|
|
18
|
+
mem: str
|
|
19
|
+
Maximum amount of physical memory used by the job.
|
|
20
|
+
walltime: str | Timedelta
|
|
21
|
+
Maximum amount of real time during which the job can be running.
|
|
22
|
+
venv_name: str
|
|
23
|
+
Name (or path) of the conda virtual environment in which the job is running.
|
|
24
|
+
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
nb_nodes: int = 1
|
|
28
|
+
ncpus: int = 2
|
|
29
|
+
ngpus: int | None = None
|
|
30
|
+
mem: str = "8gb"
|
|
31
|
+
walltime: str | Timedelta = "01:00:00"
|
|
32
|
+
venv_name: str = "osekit"
|
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
"""The job module provides classes that run transforms on a remote server.
|
|
2
|
+
|
|
3
|
+
If a ``JobBuilder`` is attached to a Public API ``Project``,
|
|
4
|
+
the transforms will run through jobs, with writting/submitting of ``pbs`` files.
|
|
5
|
+
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from enum import Enum
|
|
11
|
+
from typing import TYPE_CHECKING
|
|
12
|
+
|
|
13
|
+
from pandas import Timedelta
|
|
14
|
+
|
|
15
|
+
from osekit.job.config import JobConfig
|
|
16
|
+
|
|
17
|
+
if TYPE_CHECKING:
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class JobStatus(Enum):
|
|
22
|
+
"""Status of the job.
|
|
23
|
+
|
|
24
|
+
``UNPREPARED``: The job file hasn't been written yet.
|
|
25
|
+
``PREPARED``: The job file has been written but not submitted.
|
|
26
|
+
``QUEUED``: The job has been queued.
|
|
27
|
+
``RUNNING``: The job is currently running.
|
|
28
|
+
``SUSPENDED``: The job has been suspended or is held.
|
|
29
|
+
``COMPLETED``: The job is exiting or has been completed.
|
|
30
|
+
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
UNPREPARED = 1
|
|
34
|
+
PREPARED = 2
|
|
35
|
+
QUEUED = 3
|
|
36
|
+
RUNNING = 4
|
|
37
|
+
SUSPENDED = 5
|
|
38
|
+
COMPLETED = 6
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class Job:
|
|
42
|
+
"""Job that concerns a specific transform."""
|
|
43
|
+
|
|
44
|
+
def __init__(
|
|
45
|
+
self,
|
|
46
|
+
script_path: Path,
|
|
47
|
+
script_args: dict | None = None,
|
|
48
|
+
config: JobConfig | None = None,
|
|
49
|
+
name: str = "osekit_transform",
|
|
50
|
+
output_folder: Path | None = None,
|
|
51
|
+
) -> None:
|
|
52
|
+
"""Initialize a Job.
|
|
53
|
+
|
|
54
|
+
Parameters
|
|
55
|
+
----------
|
|
56
|
+
script_path: Path
|
|
57
|
+
Path to the script file the job must run.
|
|
58
|
+
script_args: dict | None
|
|
59
|
+
Additional arguments to pass to the script file.
|
|
60
|
+
config: JobConfig | None
|
|
61
|
+
Optional configuration to pass to the server request.
|
|
62
|
+
name: str
|
|
63
|
+
Name of the job.
|
|
64
|
+
output_folder: Path | None
|
|
65
|
+
Folder in which the output files (``.out`` and ``.err``) will be written.
|
|
66
|
+
|
|
67
|
+
"""
|
|
68
|
+
config = JobConfig() if config is None else config
|
|
69
|
+
self.script_path = script_path
|
|
70
|
+
self.script_args = script_args or {}
|
|
71
|
+
self.nb_nodes = config.nb_nodes
|
|
72
|
+
self.ncpus = config.ncpus
|
|
73
|
+
self.ngpus = config.ngpus
|
|
74
|
+
self.mem = config.mem
|
|
75
|
+
self.walltime = config.walltime
|
|
76
|
+
self.venv_name = config.venv_name
|
|
77
|
+
self.name = name
|
|
78
|
+
self.output_folder = output_folder
|
|
79
|
+
self.info = {}
|
|
80
|
+
self._status = JobStatus.UNPREPARED
|
|
81
|
+
self._path = None
|
|
82
|
+
self._id = None
|
|
83
|
+
|
|
84
|
+
@property
|
|
85
|
+
def script_path(self) -> Path:
|
|
86
|
+
"""Path to the script file the job must run."""
|
|
87
|
+
return self._script_path
|
|
88
|
+
|
|
89
|
+
@script_path.setter
|
|
90
|
+
def script_path(self, path: Path) -> None:
|
|
91
|
+
self._script_path = path
|
|
92
|
+
|
|
93
|
+
@property
|
|
94
|
+
def script_args(self) -> dict:
|
|
95
|
+
"""Additional arguments to pass to the script file."""
|
|
96
|
+
return self._script_args
|
|
97
|
+
|
|
98
|
+
@script_args.setter
|
|
99
|
+
def script_args(self, args: dict) -> None:
|
|
100
|
+
self._script_args = args
|
|
101
|
+
|
|
102
|
+
@property
|
|
103
|
+
def nb_nodes(self) -> int:
|
|
104
|
+
"""Number of nodes on which the job runs."""
|
|
105
|
+
return self._chunks
|
|
106
|
+
|
|
107
|
+
@nb_nodes.setter
|
|
108
|
+
def nb_nodes(self, chunks: int) -> None:
|
|
109
|
+
self._chunks = chunks
|
|
110
|
+
|
|
111
|
+
@property
|
|
112
|
+
def ncpus(self) -> int:
|
|
113
|
+
"""Number of total cores used per node."""
|
|
114
|
+
return self._ncpus
|
|
115
|
+
|
|
116
|
+
@ncpus.setter
|
|
117
|
+
def ncpus(self, ncpus: int) -> None:
|
|
118
|
+
self._ncpus = ncpus
|
|
119
|
+
|
|
120
|
+
@property
|
|
121
|
+
def ngpus(self) -> int | None:
|
|
122
|
+
"""Number of total GPU used per node."""
|
|
123
|
+
return self._ngpus
|
|
124
|
+
|
|
125
|
+
@ngpus.setter
|
|
126
|
+
def ngpus(self, ngpus: int) -> None:
|
|
127
|
+
self._ngpus = ngpus
|
|
128
|
+
|
|
129
|
+
@property
|
|
130
|
+
def mem(self) -> str:
|
|
131
|
+
"""Maximum amount of physical memory used by the job."""
|
|
132
|
+
return self._mem
|
|
133
|
+
|
|
134
|
+
@mem.setter
|
|
135
|
+
def mem(self, mem: str) -> None:
|
|
136
|
+
self._mem = mem
|
|
137
|
+
|
|
138
|
+
@property
|
|
139
|
+
def walltime(self) -> Timedelta:
|
|
140
|
+
"""Maximum amount of real time during which the job can be running."""
|
|
141
|
+
return self._walltime
|
|
142
|
+
|
|
143
|
+
@property
|
|
144
|
+
def walltime_str(self) -> str:
|
|
145
|
+
"""String representation of the ``walltime``."""
|
|
146
|
+
total_seconds = self.walltime.total_seconds()
|
|
147
|
+
hours, remainder = divmod(total_seconds, 3600)
|
|
148
|
+
minutes, seconds = divmod(remainder, 60)
|
|
149
|
+
return ":".join(f"{t:02}" for t in map(int, (hours, minutes, seconds)))
|
|
150
|
+
|
|
151
|
+
@walltime.setter
|
|
152
|
+
def walltime(self, walltime: str | Timedelta) -> None:
|
|
153
|
+
self._walltime = (
|
|
154
|
+
walltime if type(walltime) is Timedelta else Timedelta(walltime)
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
@property
|
|
158
|
+
def venv_name(self) -> str:
|
|
159
|
+
"""Name of the conda virtual environment in which the job is running."""
|
|
160
|
+
return self._venv_name
|
|
161
|
+
|
|
162
|
+
@venv_name.setter
|
|
163
|
+
def venv_name(self, venv_name: str) -> None:
|
|
164
|
+
self._venv_name = venv_name
|
|
165
|
+
|
|
166
|
+
@property
|
|
167
|
+
def name(self) -> str:
|
|
168
|
+
"""Name of the job."""
|
|
169
|
+
return self._name
|
|
170
|
+
|
|
171
|
+
@name.setter
|
|
172
|
+
def name(self, name: str) -> None:
|
|
173
|
+
self._name = name
|
|
174
|
+
|
|
175
|
+
@property
|
|
176
|
+
def status(self) -> JobStatus:
|
|
177
|
+
"""Status of the job.
|
|
178
|
+
|
|
179
|
+
``UNPREPARED``: The job file hasn't been written yet.
|
|
180
|
+
``PREPARED``: The job file has been written but not submitted.
|
|
181
|
+
``QUEUED``: The job has been queued.
|
|
182
|
+
``RUNNING``: The job is currently running.
|
|
183
|
+
``SUSPENDED``: The job has been suspended or is held.
|
|
184
|
+
``COMPLETED``: The job is exiting or has been completed.
|
|
185
|
+
|
|
186
|
+
"""
|
|
187
|
+
return self._status
|
|
188
|
+
|
|
189
|
+
@status.setter
|
|
190
|
+
def status(self, status: JobStatus) -> None:
|
|
191
|
+
self._status = status
|
|
192
|
+
|
|
193
|
+
@property
|
|
194
|
+
def path(self) -> Path | None:
|
|
195
|
+
"""Path of the job file."""
|
|
196
|
+
return self._path
|
|
197
|
+
|
|
198
|
+
@path.setter
|
|
199
|
+
def path(self, path: Path) -> None:
|
|
200
|
+
self._path = path
|
|
201
|
+
|
|
202
|
+
@property
|
|
203
|
+
def output_folder(self) -> Path | None:
|
|
204
|
+
"""Folder in which the output files (``.out`` and ``.err``) will be written."""
|
|
205
|
+
return self._output_folder
|
|
206
|
+
|
|
207
|
+
@output_folder.setter
|
|
208
|
+
def output_folder(self, output_folder: Path | None) -> None:
|
|
209
|
+
self._output_folder = output_folder
|
|
210
|
+
|
|
211
|
+
@property
|
|
212
|
+
def job_id(self) -> str | None:
|
|
213
|
+
"""Job ID."""
|
|
214
|
+
return self._id
|
|
215
|
+
|
|
216
|
+
@job_id.setter
|
|
217
|
+
def job_id(self, job_id: str | None) -> None:
|
|
218
|
+
self._id = job_id
|
|
219
|
+
|
|
220
|
+
@property
|
|
221
|
+
def info(self) -> dict:
|
|
222
|
+
"""Information about the job."""
|
|
223
|
+
return self._info
|
|
224
|
+
|
|
225
|
+
@info.setter
|
|
226
|
+
def info(self, info: dict) -> None:
|
|
227
|
+
self._info = info
|
|
228
|
+
|
|
229
|
+
def get_arg_string(self) -> str:
|
|
230
|
+
"""Build a string representation of the job's arguments."""
|
|
231
|
+
arg_list = []
|
|
232
|
+
for key, value in self.script_args.items():
|
|
233
|
+
if isinstance(value, bool):
|
|
234
|
+
arg_list.append(f"--{'no-' if not value else ''}{key}")
|
|
235
|
+
else:
|
|
236
|
+
arg_list.append(f"--{key} {value}")
|
|
237
|
+
return " ".join(arg_list)
|