osekit 1.4.0__tar.gz → 1.5.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (201) hide show
  1. {osekit-1.4.0 → osekit-1.5.1}/CITATION.cff +1 -1
  2. {osekit-1.4.0 → osekit-1.5.1}/PKG-INFO +1 -1
  3. osekit-1.5.1/docs/source/job.rst +27 -0
  4. {osekit-1.4.0 → osekit-1.5.1}/docs/source/jobs.rst +41 -23
  5. {osekit-1.4.0 → osekit-1.5.1}/pyproject.toml +1 -1
  6. {osekit-1.4.0 → osekit-1.5.1}/src/osekit/core/event.py +14 -19
  7. osekit-1.5.1/src/osekit/job/builder.py +153 -0
  8. osekit-1.5.1/src/osekit/job/config.py +32 -0
  9. osekit-1.5.1/src/osekit/job/job.py +237 -0
  10. osekit-1.5.1/src/osekit/job/scheduler/pbs.py +209 -0
  11. osekit-1.5.1/src/osekit/job/scheduler/scheduler.py +301 -0
  12. osekit-1.5.1/src/osekit/job/scheduler/slurm.py +267 -0
  13. {osekit-1.4.0 → osekit-1.5.1}/src/osekit/public/project.py +2 -2
  14. osekit-1.4.0/tests/sample_dataset/sample1_180701_204453.wav → osekit-1.5.1/tests/__init__.py +0 -0
  15. osekit-1.5.1/tests/_static/job_status_request_results/pbs.txt +39 -0
  16. osekit-1.5.1/tests/_static/job_status_request_results/slurm_sacct.txt +1 -0
  17. osekit-1.5.1/tests/_static/job_status_request_results/slurm_squeue.txt +1 -0
  18. osekit-1.4.0/tests/sample_dataset/sample1_190222_010000.wav → osekit-1.5.1/tests/helpers/__init__.py +0 -0
  19. osekit-1.5.1/tests/sample_dataset/sample1_190222_010000.wav +0 -0
  20. osekit-1.5.1/tests/sample_dataset/sample1_220998_185532.wav +0 -0
  21. {osekit-1.4.0 → osekit-1.5.1}/tests/test_event.py +31 -0
  22. {osekit-1.4.0 → osekit-1.5.1}/tests/test_export_transform.py +2 -2
  23. osekit-1.5.1/tests/test_job.py +973 -0
  24. {osekit-1.4.0 → osekit-1.5.1}/uv.lock +248 -215
  25. osekit-1.4.0/docs/source/job.rst +0 -7
  26. osekit-1.4.0/src/osekit/core_api/__init__.py +0 -12
  27. osekit-1.4.0/src/osekit/public_api/__init__.py +0 -12
  28. osekit-1.4.0/src/osekit/utils/job.py +0 -635
  29. osekit-1.4.0/tests/test_job.py +0 -799
  30. {osekit-1.4.0 → osekit-1.5.1}/.github/dependabot.yml +0 -0
  31. {osekit-1.4.0 → osekit-1.5.1}/.github/workflows/dependabot-auto-merge.yml +0 -0
  32. {osekit-1.4.0 → osekit-1.5.1}/.github/workflows/documentation.yml +0 -0
  33. {osekit-1.4.0 → osekit-1.5.1}/.github/workflows/github_ci.yml +0 -0
  34. {osekit-1.4.0 → osekit-1.5.1}/.github/workflows/publish.yml +0 -0
  35. {osekit-1.4.0 → osekit-1.5.1}/.gitignore +0 -0
  36. {osekit-1.4.0 → osekit-1.5.1}/.pre-commit-config.yaml +0 -0
  37. {osekit-1.4.0 → osekit-1.5.1}/CHANGELOG.md +0 -0
  38. {osekit-1.4.0 → osekit-1.5.1}/CONTRIBUTING.md +0 -0
  39. {osekit-1.4.0 → osekit-1.5.1}/LICENSE +0 -0
  40. {osekit-1.4.0 → osekit-1.5.1}/README.md +0 -0
  41. {osekit-1.4.0 → osekit-1.5.1}/docs/Makefile +0 -0
  42. {osekit-1.4.0 → osekit-1.5.1}/docs/logo/osekit.png +0 -0
  43. {osekit-1.4.0 → osekit-1.5.1}/docs/logo/osekit_mid.png +0 -0
  44. {osekit-1.4.0 → osekit-1.5.1}/docs/logo/osekit_small.png +0 -0
  45. {osekit-1.4.0 → osekit-1.5.1}/docs/logo/osmose.png +0 -0
  46. {osekit-1.4.0 → osekit-1.5.1}/docs/logo/osmose_texte_sombre.png +0 -0
  47. {osekit-1.4.0 → osekit-1.5.1}/docs/logo/osmose_texte_sombre_small.png +0 -0
  48. {osekit-1.4.0 → osekit-1.5.1}/docs/make.bat +0 -0
  49. {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/audio_data/ex2.svg +0 -0
  50. {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/audio_data/ex_1.svg +0 -0
  51. {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/audio_data/ex_2.svg +0 -0
  52. {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/custom.css +0 -0
  53. {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/data_structure/data_structure.svg +0 -0
  54. {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/data_structure/optimized_IO.svg +0 -0
  55. {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/detections/aplose_results.csv +0 -0
  56. {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/detections/detection_labelled.png +0 -0
  57. {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/detections/detection_unlabelled.png +0 -0
  58. {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/detections/label_anchors.png +0 -0
  59. {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/favicon.ico +0 -0
  60. {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/ltas/ltas.svg +0 -0
  61. {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/sample_audio/id/1829.wav +0 -0
  62. {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/sample_audio/id/1830.wav +0 -0
  63. {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/sample_audio/id/1832.wav +0 -0
  64. {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/sample_audio/id/1833.wav +0 -0
  65. {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/sample_audio/id/1834.wav +0 -0
  66. {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/sample_audio/id/1835.wav +0 -0
  67. {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/sample_audio/id/1836.wav +0 -0
  68. {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/sample_audio/id/1837.wav +0 -0
  69. {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/sample_audio/id/1838.wav +0 -0
  70. {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/sample_audio/multichannel/multichannel_220925_223450.wav +0 -0
  71. {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/sample_audio/timestamped/sample_220925_223450.wav +0 -0
  72. {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/sample_audio/timestamped/sample_220925_223500.wav +0 -0
  73. {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/sample_audio/timestamped/sample_220925_223510.wav +0 -0
  74. {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/sample_audio/timestamped/sample_220925_223520.wav +0 -0
  75. {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/sample_audio/timestamped/sample_220925_223530.wav +0 -0
  76. {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/sample_audio/timestamped/sample_220925_223600.wav +0 -0
  77. {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/sample_audio/timestamped/sample_220925_223610.wav +0 -0
  78. {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/sample_audio/timestamped/sample_220925_223620.wav +0 -0
  79. {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/sample_audio/timestamped/sample_220925_223630.wav +0 -0
  80. {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/sample_audio/timestamped/sample_220925_223640.wav +0 -0
  81. {osekit-1.4.0 → osekit-1.5.1}/docs/source/_static/spectrograms/frequency_scale.svg +0 -0
  82. {osekit-1.4.0 → osekit-1.5.1}/docs/source/api.rst +0 -0
  83. {osekit-1.4.0 → osekit-1.5.1}/docs/source/aplose.rst +0 -0
  84. {osekit-1.4.0 → osekit-1.5.1}/docs/source/audiodata.rst +0 -0
  85. {osekit-1.4.0 → osekit-1.5.1}/docs/source/audiodataset.rst +0 -0
  86. {osekit-1.4.0 → osekit-1.5.1}/docs/source/audiofile.rst +0 -0
  87. {osekit-1.4.0 → osekit-1.5.1}/docs/source/audiofilemanager.rst +0 -0
  88. {osekit-1.4.0 → osekit-1.5.1}/docs/source/audioitem.rst +0 -0
  89. {osekit-1.4.0 → osekit-1.5.1}/docs/source/basedata.rst +0 -0
  90. {osekit-1.4.0 → osekit-1.5.1}/docs/source/basedataset.rst +0 -0
  91. {osekit-1.4.0 → osekit-1.5.1}/docs/source/basefile.rst +0 -0
  92. {osekit-1.4.0 → osekit-1.5.1}/docs/source/baseitem.rst +0 -0
  93. {osekit-1.4.0 → osekit-1.5.1}/docs/source/conf.py +0 -0
  94. {osekit-1.4.0 → osekit-1.5.1}/docs/source/contributing.rst +0 -0
  95. {osekit-1.4.0 → osekit-1.5.1}/docs/source/coreapi.rst +0 -0
  96. {osekit-1.4.0 → osekit-1.5.1}/docs/source/coreapi_home.rst +0 -0
  97. {osekit-1.4.0 → osekit-1.5.1}/docs/source/coreapi_introduction.rst +0 -0
  98. {osekit-1.4.0 → osekit-1.5.1}/docs/source/coreapi_usage.rst +0 -0
  99. {osekit-1.4.0 → osekit-1.5.1}/docs/source/detection.rst +0 -0
  100. {osekit-1.4.0 → osekit-1.5.1}/docs/source/event.rst +0 -0
  101. {osekit-1.4.0 → osekit-1.5.1}/docs/source/example_aplose_result.ipynb +0 -0
  102. {osekit-1.4.0 → osekit-1.5.1}/docs/source/example_ltas.rst +0 -0
  103. {osekit-1.4.0 → osekit-1.5.1}/docs/source/example_ltas_core.ipynb +0 -0
  104. {osekit-1.4.0 → osekit-1.5.1}/docs/source/example_ltas_public.ipynb +0 -0
  105. {osekit-1.4.0 → osekit-1.5.1}/docs/source/example_multichannel.ipynb +0 -0
  106. {osekit-1.4.0 → osekit-1.5.1}/docs/source/example_multiple_spectrograms.rst +0 -0
  107. {osekit-1.4.0 → osekit-1.5.1}/docs/source/example_multiple_spectrograms_core.ipynb +0 -0
  108. {osekit-1.4.0 → osekit-1.5.1}/docs/source/example_multiple_spectrograms_id.rst +0 -0
  109. {osekit-1.4.0 → osekit-1.5.1}/docs/source/example_multiple_spectrograms_id_core.ipynb +0 -0
  110. {osekit-1.4.0 → osekit-1.5.1}/docs/source/example_multiple_spectrograms_id_public.ipynb +0 -0
  111. {osekit-1.4.0 → osekit-1.5.1}/docs/source/example_multiple_spectrograms_public.ipynb +0 -0
  112. {osekit-1.4.0 → osekit-1.5.1}/docs/source/example_reshaping_multiple_files.rst +0 -0
  113. {osekit-1.4.0 → osekit-1.5.1}/docs/source/example_reshaping_multiple_files_core.ipynb +0 -0
  114. {osekit-1.4.0 → osekit-1.5.1}/docs/source/example_reshaping_multiple_files_public.ipynb +0 -0
  115. {osekit-1.4.0 → osekit-1.5.1}/docs/source/example_reshaping_one_file.ipynb +0 -0
  116. {osekit-1.4.0 → osekit-1.5.1}/docs/source/example_spectrogram.ipynb +0 -0
  117. {osekit-1.4.0 → osekit-1.5.1}/docs/source/examples.rst +0 -0
  118. {osekit-1.4.0 → osekit-1.5.1}/docs/source/export_transform.rst +0 -0
  119. {osekit-1.4.0 → osekit-1.5.1}/docs/source/frequencyscale.rst +0 -0
  120. {osekit-1.4.0 → osekit-1.5.1}/docs/source/index.rst +0 -0
  121. {osekit-1.4.0 → osekit-1.5.1}/docs/source/installation.rst +0 -0
  122. {osekit-1.4.0 → osekit-1.5.1}/docs/source/instrument.rst +0 -0
  123. {osekit-1.4.0 → osekit-1.5.1}/docs/source/ltasdata.rst +0 -0
  124. {osekit-1.4.0 → osekit-1.5.1}/docs/source/multichannel.rst +0 -0
  125. {osekit-1.4.0 → osekit-1.5.1}/docs/source/multiprocessing.rst +0 -0
  126. {osekit-1.4.0 → osekit-1.5.1}/docs/source/normalization.rst +0 -0
  127. {osekit-1.4.0 → osekit-1.5.1}/docs/source/project.rst +0 -0
  128. {osekit-1.4.0 → osekit-1.5.1}/docs/source/publicapi.rst +0 -0
  129. {osekit-1.4.0 → osekit-1.5.1}/docs/source/publicapi_usage.rst +0 -0
  130. {osekit-1.4.0 → osekit-1.5.1}/docs/source/spectrodata.rst +0 -0
  131. {osekit-1.4.0 → osekit-1.5.1}/docs/source/spectrodataset.rst +0 -0
  132. {osekit-1.4.0 → osekit-1.5.1}/docs/source/spectrofile.rst +0 -0
  133. {osekit-1.4.0 → osekit-1.5.1}/docs/source/spectroitem.rst +0 -0
  134. {osekit-1.4.0 → osekit-1.5.1}/docs/source/transform.rst +0 -0
  135. {osekit-1.4.0 → osekit-1.5.1}/docs/source/usage.rst +0 -0
  136. {osekit-1.4.0 → osekit-1.5.1}/docs/source/utils.rst +0 -0
  137. {osekit-1.4.0 → osekit-1.5.1}/src/osekit/__init__.py +0 -0
  138. {osekit-1.4.0 → osekit-1.5.1}/src/osekit/audio_backend/audio_backend.py +0 -0
  139. {osekit-1.4.0 → osekit-1.5.1}/src/osekit/audio_backend/audio_file_manager.py +0 -0
  140. {osekit-1.4.0 → osekit-1.5.1}/src/osekit/audio_backend/mseed_backend.py +0 -0
  141. {osekit-1.4.0 → osekit-1.5.1}/src/osekit/audio_backend/soundfile_backend.py +0 -0
  142. {osekit-1.4.0 → osekit-1.5.1}/src/osekit/config.py +0 -0
  143. {osekit-1.4.0 → osekit-1.5.1}/src/osekit/core/__init__.py +0 -0
  144. {osekit-1.4.0 → osekit-1.5.1}/src/osekit/core/audio_data.py +0 -0
  145. {osekit-1.4.0 → osekit-1.5.1}/src/osekit/core/audio_dataset.py +0 -0
  146. {osekit-1.4.0 → osekit-1.5.1}/src/osekit/core/audio_file.py +0 -0
  147. {osekit-1.4.0 → osekit-1.5.1}/src/osekit/core/audio_item.py +0 -0
  148. {osekit-1.4.0 → osekit-1.5.1}/src/osekit/core/base_data.py +0 -0
  149. {osekit-1.4.0 → osekit-1.5.1}/src/osekit/core/base_dataset.py +0 -0
  150. {osekit-1.4.0 → osekit-1.5.1}/src/osekit/core/base_file.py +0 -0
  151. {osekit-1.4.0 → osekit-1.5.1}/src/osekit/core/base_item.py +0 -0
  152. {osekit-1.4.0 → osekit-1.5.1}/src/osekit/core/detection.py +0 -0
  153. {osekit-1.4.0 → osekit-1.5.1}/src/osekit/core/frequency_scale.py +0 -0
  154. {osekit-1.4.0 → osekit-1.5.1}/src/osekit/core/instrument.py +0 -0
  155. {osekit-1.4.0 → osekit-1.5.1}/src/osekit/core/json_serializer.py +0 -0
  156. {osekit-1.4.0 → osekit-1.5.1}/src/osekit/core/ltas_data.py +0 -0
  157. {osekit-1.4.0 → osekit-1.5.1}/src/osekit/core/ltas_dataset.py +0 -0
  158. {osekit-1.4.0 → osekit-1.5.1}/src/osekit/core/spectro_data.py +0 -0
  159. {osekit-1.4.0 → osekit-1.5.1}/src/osekit/core/spectro_dataset.py +0 -0
  160. {osekit-1.4.0 → osekit-1.5.1}/src/osekit/core/spectro_file.py +0 -0
  161. {osekit-1.4.0 → osekit-1.5.1}/src/osekit/core/spectro_item.py +0 -0
  162. {osekit-1.4.0/src/osekit/public → osekit-1.5.1/src/osekit/job}/__init__.py +0 -0
  163. {osekit-1.4.0/src/osekit/utils → osekit-1.5.1/src/osekit/job/scheduler}/__init__.py +0 -0
  164. {osekit-1.4.0 → osekit-1.5.1}/src/osekit/logging_config.yaml +0 -0
  165. {osekit-1.4.0 → osekit-1.5.1}/src/osekit/logging_context.py +0 -0
  166. {osekit-1.4.0/tests → osekit-1.5.1/src/osekit/public}/__init__.py +0 -0
  167. {osekit-1.4.0 → osekit-1.5.1}/src/osekit/public/export_transform.py +0 -0
  168. {osekit-1.4.0 → osekit-1.5.1}/src/osekit/public/transform.py +0 -0
  169. {osekit-1.4.0/tests/helpers → osekit-1.5.1/src/osekit/utils}/__init__.py +0 -0
  170. {osekit-1.4.0 → osekit-1.5.1}/src/osekit/utils/aplose.py +0 -0
  171. {osekit-1.4.0 → osekit-1.5.1}/src/osekit/utils/audio.py +0 -0
  172. {osekit-1.4.0 → osekit-1.5.1}/src/osekit/utils/core.py +0 -0
  173. {osekit-1.4.0 → osekit-1.5.1}/src/osekit/utils/deserialization.py +0 -0
  174. {osekit-1.4.0 → osekit-1.5.1}/src/osekit/utils/multiprocess.py +0 -0
  175. {osekit-1.4.0 → osekit-1.5.1}/src/osekit/utils/path.py +0 -0
  176. {osekit-1.4.0 → osekit-1.5.1}/src/osekit/utils/plot.py +0 -0
  177. {osekit-1.4.0 → osekit-1.5.1}/src/osekit/utils/timestamp.py +0 -0
  178. {osekit-1.4.0 → osekit-1.5.1}/tests/_static/aplose_result.csv +0 -0
  179. {osekit-1.4.0 → osekit-1.5.1}/tests/_static/minimal_detector_result.csv +0 -0
  180. {osekit-1.4.0 → osekit-1.5.1}/tests/_static/retro_compatibility_result.csv +0 -0
  181. {osekit-1.4.0 → osekit-1.5.1}/tests/conftest.py +0 -0
  182. {osekit-1.4.0 → osekit-1.5.1}/tests/helpers/audio.py +0 -0
  183. {osekit-1.4.0 → osekit-1.5.1}/tests/helpers/dummy.py +0 -0
  184. /osekit-1.4.0/tests/sample_dataset/sample1_220998_185532.wav → /osekit-1.5.1/tests/sample_dataset/sample1_180701_204453.wav +0 -0
  185. {osekit-1.4.0 → osekit-1.5.1}/tests/test_audio.py +0 -0
  186. {osekit-1.4.0 → osekit-1.5.1}/tests/test_audio_file_manager.py +0 -0
  187. {osekit-1.4.0 → osekit-1.5.1}/tests/test_core_api_base.py +0 -0
  188. {osekit-1.4.0 → osekit-1.5.1}/tests/test_detection.py +0 -0
  189. {osekit-1.4.0 → osekit-1.5.1}/tests/test_files.py +0 -0
  190. {osekit-1.4.0 → osekit-1.5.1}/tests/test_frequency_scales.py +0 -0
  191. {osekit-1.4.0 → osekit-1.5.1}/tests/test_instrument.py +0 -0
  192. {osekit-1.4.0 → osekit-1.5.1}/tests/test_logging.py +0 -0
  193. {osekit-1.4.0 → osekit-1.5.1}/tests/test_mseed.py +0 -0
  194. {osekit-1.4.0 → osekit-1.5.1}/tests/test_multiprocessing.py +0 -0
  195. {osekit-1.4.0 → osekit-1.5.1}/tests/test_permissions.py +0 -0
  196. {osekit-1.4.0 → osekit-1.5.1}/tests/test_public_api.py +0 -0
  197. {osekit-1.4.0 → osekit-1.5.1}/tests/test_serialization.py +0 -0
  198. {osekit-1.4.0 → osekit-1.5.1}/tests/test_spectro.py +0 -0
  199. {osekit-1.4.0 → osekit-1.5.1}/tests/test_timestamp_utils.py +0 -0
  200. {osekit-1.4.0 → osekit-1.5.1}/tests/test_utils.py +0 -0
  201. {osekit-1.4.0 → osekit-1.5.1}/tests/test_welch.py +0 -0
@@ -38,4 +38,4 @@ message: If you use this software, please cite it using the metadata from this f
38
38
  repository-code: https://github.com/Project-OSmOSE/OSEkit
39
39
  title: OSEkit
40
40
  type: software
41
- version: v1.4.0
41
+ version: v1.5.1
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: osekit
3
- Version: 1.4.0
3
+ Version: 1.5.1
4
4
  Summary: OSEkit
5
5
  Author-email: OSmOSE team <osmose@ensta-bretagne.fr>
6
6
  License-Expression: MIT
@@ -0,0 +1,27 @@
1
+ .. _job:
2
+
3
+ Job
4
+ ---
5
+
6
+ .. automodule:: osekit.job.job
7
+ :members: Job
8
+
9
+ Job Config
10
+ ----------
11
+
12
+ .. automodule:: osekit.job.config
13
+ :members: JobConfig
14
+
15
+ Job Scheduler
16
+ -------------
17
+
18
+ .. automodule:: osekit.job.scheduler.pbs
19
+ :members: Pbs
20
+ .. automodule:: osekit.job.scheduler.slurm
21
+ :members: Slurm
22
+
23
+ Job Builder
24
+ -----------
25
+
26
+ .. automodule:: osekit.job.builder
27
+ :members: JobBuilder
@@ -2,51 +2,65 @@ Working with jobs
2
2
  -----------------
3
3
 
4
4
  **OSEkit** can be set to send transform instructions to be computed on a remote server
5
- through the PBS queuing system.
5
+ through queuing systems.
6
6
 
7
7
  This feature has mainly be thought for the Public API, but it can nonetheless be used for
8
8
  any Core API operation.
9
9
 
10
- The job module is located at :mod:`osekit.utils.job`.
10
+ This is done thanks to the job package, located in :mod:`osekit.job`.
11
11
 
12
12
  Public API
13
13
  ^^^^^^^^^^
14
14
 
15
- Running Public API Analyses through PBS jobs only requires adding a :class:`osekit.utils.job.JobBuilder`
16
- instance to the :attr:`osekit.public.project.Project.job_builder` attribute:
15
+ Running Public API Analyses through jobs only requires adding a :class:`osekit.job.builder.JobBuilder`
16
+ instance to the :attr:`osekit.public.project.Project.job_builder` attribute.
17
+
18
+ The :class:`osekit.job.builder.JobBuilder`
19
+
20
+ Here is an example for running a transform on a PBS queue:
17
21
 
18
22
  .. code-block:: python
19
23
 
20
- from osekit.utils.job import JobConfig, JobBuilder
24
+ import os
25
+
26
+ from pandas import Timedelta
27
+
28
+ from osekit.job.builder import JobBuilder
29
+ from osekit.job.config import JobConfig
30
+ from osekit.job.scheduler.pbs import Pbs
21
31
  from osekit.public.project import Project
22
32
 
23
- project = Project(...) # See the Project documentation
33
+ project = Project(...) # See the Project documentation
24
34
 
25
35
  job_config = JobConfig(
26
- nb_nodes=1, # Number of nodes on which the job runs
27
- ncpus=28, # Number of total cores used per node
28
- ngpus=1, # Number of total GPU used per node
29
- mem="60gb", # Maximum amount of physical memory used by the job
30
- walltime=Timedelta(hours=5), # Maximum amount of real itime during which the job can be running
31
- venv_name=os.environ["CONDA_DEFAULT_ENV"], # Works only for conda venvs
32
- queue="omp" # Queue in which the job will be submitted
36
+ nb_nodes=1, # Number of nodes on which the job runs
37
+ ncpus=28, # Number of total cores used per node
38
+ ngpus=1, # Number of total GPU used per node
39
+ mem="60gb", # Maximum amount of physical memory used by the job
40
+ walltime=Timedelta(
41
+ hours=5
42
+ ), # Maximum amount of real itime during which the job can be running
43
+ venv_name=os.environ["CONDA_DEFAULT_ENV"], # Works only for conda venvs
33
44
  )
34
45
 
46
+ scheduler = Pbs(queue="omp") # Scheduler in which the job is submitted
47
+
35
48
  project.job_builder = JobBuilder(
36
49
  config=job_config,
50
+ scheduler=scheduler,
37
51
  )
38
52
 
39
53
  # Now the dataset has a non-None job_builder attribute,
40
54
  # running a transform will write a PBS file in the logs directory
41
- # and submit it to the requested queue.
55
+ # and submit it through the selected scheduler.
42
56
 
43
- project.run(...) # See the Transform documentation
57
+ project.run(...) # See the Transform documentation
44
58
 
45
59
 
46
60
  Core API
47
61
  ^^^^^^^^
48
62
 
49
- Exporting Core API datasets with jobs is doable by explicitly instantiating a :class:`osekit.utils.job.Job` object.
63
+ Exporting Core API datasets with jobs is doable by explicitly instantiating a :class:`osekit.job.job.Job` object.
50
64
 
51
65
  The export parameters are specified in the ``script_args`` parameter of the ``Job`` constructor,
52
66
  and follow the console arguments of the :mod:`osekit.public.export` script.
@@ -60,11 +74,13 @@ and follow the console arguments of the :mod:`osekit.public.export` script.
60
74
 
61
75
  from osekit.core.audio_dataset import AudioDataset
62
76
  from osekit.core.spectro_dataset import SpectroDataset
77
+ from osekit.job.config import JobConfig
78
+ from osekit.job.job import Job
79
+ from osekit.job.scheduler.pbs import Pbs
63
80
  from osekit.public import export_transform
64
81
 
65
82
  # Some Public API imports are required
66
83
  from osekit.public.transform import OutputType
67
- from osekit.utils.job import Job, JobConfig
68
84
 
69
85
  ads = AudioDataset(...) # See the AudioDataset doc
70
86
  sds = SpectroDataset(...) # See the SpectroDataset doc
@@ -109,9 +125,11 @@ and follow the console arguments of the :mod:`osekit.public.export` script.
109
125
  mem="60gb",
110
126
  walltime=Timedelta(hours=1),
111
127
  venv_name=os.environ["CONDA_DEFAULT_ENV"],
112
- queue="omp",
113
128
  )
114
129
 
130
+ # Scheduler configuration
131
+ scheduler = Pbs(queue="omp")
132
+
115
133
  job = Job(
116
134
  script_path=Path(export_transform.__file__),
117
135
  script_args=args,
@@ -120,12 +138,12 @@ and follow the console arguments of the :mod:`osekit.public.export` script.
120
138
  output_folder=Path(...), # Path in which the .out and .err files are written
121
139
  )
122
140
 
123
- # Write the PBS file and submit the job
124
- job.write_pbs(Path(...) / f"{job.name}.pbs")
125
- job.submit_pbs()
141
+ # Write the job file and submit it through the scheduler
142
+ scheduler.write(job=job, path=Path(...) / f"{job.name}.pbs")
143
+ scheduler.submit(job=job)
126
144
 
127
- You can then follow the status of the submitted job:
145
+ You can then follow the status of the submitted job through the scheduler:
128
146
 
129
147
  .. code-block:: python
130
148
 
131
- job.update_status()
149
+ scheduler.update_status(job=job)
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "osekit"
3
- version = "1.4.0"
3
+ version = "1.5.1"
4
4
  description = "OSEkit"
5
5
  readme = "README.md"
6
6
  license = "MIT"
@@ -5,6 +5,7 @@ from __future__ import annotations
5
5
  import bisect
6
6
  import copy
7
7
  from dataclasses import dataclass, field
8
+ from operator import attrgetter
8
9
  from typing import TYPE_CHECKING, TypeVar
9
10
 
10
11
  from osekit.utils.timestamp import localize_timestamp
@@ -40,7 +41,7 @@ class Event:
40
41
  @begin.setter
41
42
  def begin(self, value: Timestamp) -> None:
42
43
  if hasattr(self, "_end") and value > self._end:
43
- msg = f"Invalid Event: `end` ({self._end}) must be greater than `begin` ({value})." # noqa: E501
44
+ msg = f"Invalid Event: `end` ({self._end}) must be greater than `begin` ({value})."
44
45
  raise ValueError(msg)
45
46
  self._begin = value
46
47
 
@@ -52,7 +53,7 @@ class Event:
52
53
  @end.setter
53
54
  def end(self, value: Timestamp) -> None:
54
55
  if hasattr(self, "_begin") and value < self._begin:
55
- msg = f"Invalid Event: `end` ({value}) must be greater than `begin` ({self._begin})." # noqa: E501
56
+ msg = f"Invalid Event: `end` ({value}) must be greater than `begin` ({self._begin})."
56
57
  raise ValueError(msg)
57
58
  self._end = value
58
59
 
@@ -104,7 +105,7 @@ class Event:
104
105
  >>> Event(begin=Timestamp("2024-01-01 00:00:00"),end=Timestamp("2024-01-02 00:00:00")).overlaps(Event(begin=Timestamp("2024-01-02 00:00:00"),end=Timestamp("2024-01-02 12:00:00")))
105
106
  False
106
107
 
107
- """ # noqa: E501
108
+ """
108
109
  return self.begin < other.end and self.end > other.begin
109
110
 
110
111
  def get_overlapping_events(self, events: list[TEvent]) -> list[TEvent]:
@@ -125,19 +126,13 @@ class Event:
125
126
 
126
127
  """
127
128
  output = []
128
- start_index = bisect.bisect_left(
129
- events,
130
- self.begin,
131
- key=lambda event: event.end,
132
- )
133
- for i in range(start_index, len(events)):
134
- if events[i] is self:
135
- continue
136
- if self.overlaps(events[i]):
137
- output.append(events[i])
138
- continue
139
- if events[i].begin >= self.end:
140
- break
129
+ stop_index = bisect.bisect(a=events, x=self.end, key=attrgetter("begin"))
130
+
131
+ for i in range(stop_index):
132
+ event = events[i]
133
+ if event is not self and self.overlaps(event):
134
+ output.append(event)
135
+
141
136
  return output
142
137
 
143
138
  @classmethod
@@ -170,7 +165,7 @@ class Event:
170
165
  >>> events[0].end == events[1].begin
171
166
  True
172
167
 
173
- """ # noqa: E501
168
+ """
174
169
  events = sorted(
175
170
  events,
176
171
  key=lambda event: (event.begin, -1 * event.duration),
@@ -199,7 +194,7 @@ class Event:
199
194
  cls,
200
195
  events: list[TEvent],
201
196
  filling_class: type[TEvent],
202
- **kwargs, # noqa: ANN003
197
+ **kwargs,
203
198
  ) -> list[TEvent]:
204
199
  """Return a list with empty events added in the gaps between items.
205
200
 
@@ -227,7 +222,7 @@ class Event:
227
222
  >>> [(event.begin.second, event.end.second) for event in events]
228
223
  [(0, 10), (10, 15), (15, 25)]
229
224
 
230
- """ # noqa: E501
225
+ """
231
226
  events = sorted(
232
227
  [copy.copy(event) for event in events],
233
228
  key=lambda event: event.begin,
@@ -0,0 +1,153 @@
1
+ from pathlib import Path
2
+
3
+ from osekit.job.config import JobConfig
4
+ from osekit.job.job import Job, JobStatus
5
+ from osekit.job.scheduler.pbs import Pbs
6
+ from osekit.job.scheduler.scheduler import Scheduler
7
+ from osekit.utils.core import file_indexes_per_batch
8
+
9
+
10
+ class JobBuilder:
11
+ """Class that should be attached to a Public API ``Project`` for working with jobs.
12
+
13
+ If a ``Project`` has a ``JobBuilder``, it will run its transforms through jobs
14
+ using the specified scheduler.
15
+
16
+ """
17
+
18
+ def __init__(
19
+ self,
20
+ config: JobConfig | None = None,
21
+ scheduler: Scheduler | None = None,
22
+ ) -> None:
23
+ """Initialize a ``JobBuilder`` instance.
24
+
25
+ Parameters
26
+ ----------
27
+ config: JobConfig
28
+ Config of the jobs built by this job builder.
29
+ scheduler: Scheduler
30
+ Scheduler used to format, write and submit jobs.
31
+
32
+ """
33
+ self.config = config or JobConfig()
34
+ self.scheduler = scheduler or Pbs()
35
+ self.jobs = []
36
+
37
+ @property
38
+ def jobs(self) -> list[Job]:
39
+ """Return the jobs created by this job builder."""
40
+ return self._jobs
41
+
42
+ @jobs.setter
43
+ def jobs(self, jobs: list[Job]) -> None:
44
+ self._jobs = jobs
45
+
46
+ def create_jobs(
47
+ self,
48
+ nb_tasks: int,
49
+ script_path: Path,
50
+ script_args: dict,
51
+ output_folder: Path,
52
+ job_name: str = "osekit_transform",
53
+ nb_jobs: int = 1,
54
+ ) -> None:
55
+ """Create the jobs corresponding to each batch.
56
+
57
+ Parameters
58
+ ----------
59
+ nb_tasks:
60
+ The number of tasks that are distributed across ``nb_jobs`` jobs.
61
+ script_path: Path
62
+ Path to the export script.
63
+ script_args: dict
64
+ Arguments passed to the export script.
65
+ job_name: str
66
+ Name of the job.
67
+ If there are multiple batches, each batch will be suffixed
68
+ with "_{index}".
69
+ output_folder: Path
70
+ Folder in which the job output log files are saved.
71
+ nb_jobs: int
72
+ Number of batches used to run the transform.
73
+ Each batch will run in a separate job.
74
+
75
+ """
76
+ batch_indexes = file_indexes_per_batch(
77
+ total_nb_files=nb_tasks,
78
+ nb_batches=nb_jobs,
79
+ )
80
+ for index, (start, stop) in enumerate(batch_indexes):
81
+ self.create_job(
82
+ script_path=script_path,
83
+ script_args=script_args | {"first": start, "last": stop},
84
+ name=job_name + (f"_{index}" if len(batch_indexes) > 1 else ""),
85
+ output_folder=output_folder,
86
+ )
87
+
88
+ def create_job(
89
+ self,
90
+ script_path: Path,
91
+ script_args: dict | None = None,
92
+ name: str = "osekit_transform",
93
+ output_folder: Path | None = None,
94
+ ) -> None:
95
+ """Create a new ``Job`` instance.
96
+
97
+ Parameters
98
+ ----------
99
+ script_path: Path
100
+ Path to the script file the job must run.
101
+ script_args: dict | None
102
+ Additional arguments to pass to the script file.
103
+ name: str
104
+ Name of the job.
105
+ output_folder: Path | None
106
+ Folder in which the output files (``.out`` and ``.err``) will be written.
107
+
108
+ """
109
+ job = Job(
110
+ script_path=script_path,
111
+ script_args=script_args,
112
+ name=name,
113
+ output_folder=output_folder,
114
+ config=self.config,
115
+ )
116
+ self.scheduler.write(
117
+ job=job,
118
+ path=output_folder / f"{name}.{self.scheduler.JOB_FILE_EXTENSION}",
119
+ )
120
+ self.jobs.append(job)
121
+
122
+ def submit(
123
+ self,
124
+ dependencies: dict[Job, dict[str, str | Job | list[str | Job]]] | None = None,
125
+ ) -> None:
126
+ """Submit all prepared jobs to the scheduler system.
127
+
128
+ Parameters
129
+ ----------
130
+ dependencies: dict[Job, dict[str, str | Job | list[str | Job]]] | None
131
+ Optional mapping of the jobs dependencies.
132
+
133
+ For each key (which is a job), the value is a
134
+ dictionary explaining its dependencies.
135
+
136
+ Such dictionary follows the following format:
137
+ The keys of the dictionary are the dependency types,
138
+ that are proper to the scheduler.
139
+ The values are the other jobs (or their ID) ``job`` depends on
140
+ with the given dependency type.
141
+
142
+ If ``None``, the jobs are submitted without any dependency.
143
+
144
+ """
145
+ dependencies = dependencies or {}
146
+ for job in self.jobs:
147
+ if self.scheduler.update_status(job=job) is not JobStatus.PREPARED:
148
+ continue
149
+
150
+ # Check if this job has dependencies
151
+ job_dependencies = dependencies.get(job, None)
152
+
153
+ self.scheduler.submit(job=job, dependencies=job_dependencies)
@@ -0,0 +1,32 @@
1
+ from dataclasses import dataclass
2
+
3
+ from pandas import Timedelta
4
+
5
+
6
+ @dataclass
7
+ class JobConfig:
8
+ """Configuration of the computing resources allowed for a job.
9
+
10
+ Parameters
11
+ ----------
12
+ nb_nodes: int
13
+ Number of nodes on which the job runs.
14
+ ncpus: int
15
+ Number of total cores used per node.
16
+ ngpus: int | None
17
+ Number of total GPU used per node.
18
+ mem: str
19
+ Maximum amount of physical memory used by the job.
20
+ walltime: str | Timedelta
21
+ Maximum amount of real time during which the job can be running.
22
+ venv_name: str
23
+ Name (or path) of the conda virtual environment in which the job is running.
24
+
25
+ """
26
+
27
+ nb_nodes: int = 1
28
+ ncpus: int = 2
29
+ ngpus: int | None = None
30
+ mem: str = "8gb"
31
+ walltime: str | Timedelta = "01:00:00"
32
+ venv_name: str = "osekit"
@@ -0,0 +1,237 @@
1
+ """The job module provides classes that run transforms on a remote server.
2
+
3
+ If a ``JobBuilder`` is attached to a Public API ``Project``,
4
+ the transforms will run through jobs, with writting/submitting of ``pbs`` files.
5
+
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from enum import Enum
11
+ from typing import TYPE_CHECKING
12
+
13
+ from pandas import Timedelta
14
+
15
+ from osekit.job.config import JobConfig
16
+
17
+ if TYPE_CHECKING:
18
+ from pathlib import Path
19
+
20
+
21
+ class JobStatus(Enum):
22
+ """Status of the job.
23
+
24
+ ``UNPREPARED``: The job file hasn't been written yet.
25
+ ``PREPARED``: The job file has been written but not submitted.
26
+ ``QUEUED``: The job has been queued.
27
+ ``RUNNING``: The job is currently running.
28
+ ``SUSPENDED``: The job has been suspended or is held.
29
+ ``COMPLETED``: The job is exiting or has been completed.
30
+
31
+ """
32
+
33
+ UNPREPARED = 1
34
+ PREPARED = 2
35
+ QUEUED = 3
36
+ RUNNING = 4
37
+ SUSPENDED = 5
38
+ COMPLETED = 6
39
+
40
+
41
+ class Job:
42
+ """Job that concerns a specific transform."""
43
+
44
+ def __init__(
45
+ self,
46
+ script_path: Path,
47
+ script_args: dict | None = None,
48
+ config: JobConfig | None = None,
49
+ name: str = "osekit_transform",
50
+ output_folder: Path | None = None,
51
+ ) -> None:
52
+ """Initialize a Job.
53
+
54
+ Parameters
55
+ ----------
56
+ script_path: Path
57
+ Path to the script file the job must run.
58
+ script_args: dict | None
59
+ Additional arguments to pass to the script file.
60
+ config: JobConfig | None
61
+ Optional configuration to pass to the server request.
62
+ name: str
63
+ Name of the job.
64
+ output_folder: Path | None
65
+ Folder in which the output files (``.out`` and ``.err``) will be written.
66
+
67
+ """
68
+ config = JobConfig() if config is None else config
69
+ self.script_path = script_path
70
+ self.script_args = script_args or {}
71
+ self.nb_nodes = config.nb_nodes
72
+ self.ncpus = config.ncpus
73
+ self.ngpus = config.ngpus
74
+ self.mem = config.mem
75
+ self.walltime = config.walltime
76
+ self.venv_name = config.venv_name
77
+ self.name = name
78
+ self.output_folder = output_folder
79
+ self.info = {}
80
+ self._status = JobStatus.UNPREPARED
81
+ self._path = None
82
+ self._id = None
83
+
84
+ @property
85
+ def script_path(self) -> Path:
86
+ """Path to the script file the job must run."""
87
+ return self._script_path
88
+
89
+ @script_path.setter
90
+ def script_path(self, path: Path) -> None:
91
+ self._script_path = path
92
+
93
+ @property
94
+ def script_args(self) -> dict:
95
+ """Additional arguments to pass to the script file."""
96
+ return self._script_args
97
+
98
+ @script_args.setter
99
+ def script_args(self, args: dict) -> None:
100
+ self._script_args = args
101
+
102
+ @property
103
+ def nb_nodes(self) -> int:
104
+ """Number of nodes on which the job runs."""
105
+ return self._chunks
106
+
107
+ @nb_nodes.setter
108
+ def nb_nodes(self, chunks: int) -> None:
109
+ self._chunks = chunks
110
+
111
+ @property
112
+ def ncpus(self) -> int:
113
+ """Number of total cores used per node."""
114
+ return self._ncpus
115
+
116
+ @ncpus.setter
117
+ def ncpus(self, ncpus: int) -> None:
118
+ self._ncpus = ncpus
119
+
120
+ @property
121
+ def ngpus(self) -> int | None:
122
+ """Number of total GPU used per node."""
123
+ return self._ngpus
124
+
125
+ @ngpus.setter
126
+ def ngpus(self, ngpus: int) -> None:
127
+ self._ngpus = ngpus
128
+
129
+ @property
130
+ def mem(self) -> str:
131
+ """Maximum amount of physical memory used by the job."""
132
+ return self._mem
133
+
134
+ @mem.setter
135
+ def mem(self, mem: str) -> None:
136
+ self._mem = mem
137
+
138
+ @property
139
+ def walltime(self) -> Timedelta:
140
+ """Maximum amount of real time during which the job can be running."""
141
+ return self._walltime
142
+
143
+ @property
144
+ def walltime_str(self) -> str:
145
+ """String representation of the ``walltime``."""
146
+ total_seconds = self.walltime.total_seconds()
147
+ hours, remainder = divmod(total_seconds, 3600)
148
+ minutes, seconds = divmod(remainder, 60)
149
+ return ":".join(f"{t:02}" for t in map(int, (hours, minutes, seconds)))
150
+
151
+ @walltime.setter
152
+ def walltime(self, walltime: str | Timedelta) -> None:
153
+ self._walltime = (
154
+ walltime if type(walltime) is Timedelta else Timedelta(walltime)
155
+ )
156
+
157
+ @property
158
+ def venv_name(self) -> str:
159
+ """Name of the conda virtual environment in which the job is running."""
160
+ return self._venv_name
161
+
162
+ @venv_name.setter
163
+ def venv_name(self, venv_name: str) -> None:
164
+ self._venv_name = venv_name
165
+
166
+ @property
167
+ def name(self) -> str:
168
+ """Name of the job."""
169
+ return self._name
170
+
171
+ @name.setter
172
+ def name(self, name: str) -> None:
173
+ self._name = name
174
+
175
+ @property
176
+ def status(self) -> JobStatus:
177
+ """Status of the job.
178
+
179
+ ``UNPREPARED``: The job file hasn't been written yet.
180
+ ``PREPARED``: The job file has been written but not submitted.
181
+ ``QUEUED``: The job has been queued.
182
+ ``RUNNING``: The job is currently running.
183
+ ``SUSPENDED``: The job has been suspended or is held.
184
+ ``COMPLETED``: The job is exiting or has been completed.
185
+
186
+ """
187
+ return self._status
188
+
189
+ @status.setter
190
+ def status(self, status: JobStatus) -> None:
191
+ self._status = status
192
+
193
+ @property
194
+ def path(self) -> Path | None:
195
+ """Path of the job file."""
196
+ return self._path
197
+
198
+ @path.setter
199
+ def path(self, path: Path) -> None:
200
+ self._path = path
201
+
202
+ @property
203
+ def output_folder(self) -> Path | None:
204
+ """Folder in which the output files (``.out`` and ``.err``) will be written."""
205
+ return self._output_folder
206
+
207
+ @output_folder.setter
208
+ def output_folder(self, output_folder: Path | None) -> None:
209
+ self._output_folder = output_folder
210
+
211
+ @property
212
+ def job_id(self) -> str | None:
213
+ """Job ID."""
214
+ return self._id
215
+
216
+ @job_id.setter
217
+ def job_id(self, job_id: str | None) -> None:
218
+ self._id = job_id
219
+
220
+ @property
221
+ def info(self) -> dict:
222
+ """Information about the job."""
223
+ return self._info
224
+
225
+ @info.setter
226
+ def info(self, info: dict) -> None:
227
+ self._info = info
228
+
229
+ def get_arg_string(self) -> str:
230
+ """Build a string representation of the job's arguments."""
231
+ arg_list = []
232
+ for key, value in self.script_args.items():
233
+ if isinstance(value, bool):
234
+ arg_list.append(f"--{'no-' if not value else ''}{key}")
235
+ else:
236
+ arg_list.append(f"--{key} {value}")
237
+ return " ".join(arg_list)