fractal-server 2.3.3__tar.gz → 2.3.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (203) hide show
  1. {fractal_server-2.3.3 → fractal_server-2.3.5}/PKG-INFO +1 -1
  2. fractal_server-2.3.5/fractal_server/__init__.py +1 -0
  3. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/routes/api/__init__.py +9 -0
  4. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/routes/api/v1/project.py +10 -1
  5. fractal_server-2.3.5/fractal_server/app/runner/compress_folder.py +132 -0
  6. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/executors/slurm/_slurm_config.py +16 -1
  7. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/executors/slurm/ssh/executor.py +27 -16
  8. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/executors/slurm/sudo/executor.py +6 -0
  9. fractal_server-2.3.5/fractal_server/app/runner/extract_archive.py +85 -0
  10. fractal_server-2.3.5/fractal_server/app/runner/run_subprocess.py +27 -0
  11. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/v2/__init__.py +1 -1
  12. {fractal_server-2.3.3/fractal_server/app/runner/v2/_slurm_ssh → fractal_server-2.3.5/fractal_server/app/runner/v2/_slurm_common}/get_slurm_config.py +4 -13
  13. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/v2/_slurm_ssh/_submit_setup.py +3 -3
  14. {fractal_server-2.3.3/fractal_server/app/runner/v2/_slurm → fractal_server-2.3.5/fractal_server/app/runner/v2/_slurm_sudo}/_submit_setup.py +3 -3
  15. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/config.py +22 -1
  16. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/main.py +1 -1
  17. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/ssh/_fabric.py +22 -0
  18. fractal_server-2.3.5/fractal_server/tasks/v2/__init__.py +0 -0
  19. {fractal_server-2.3.3 → fractal_server-2.3.5}/pyproject.toml +2 -2
  20. fractal_server-2.3.3/fractal_server/__init__.py +0 -1
  21. fractal_server-2.3.3/fractal_server/app/runner/compress_folder.py +0 -120
  22. fractal_server-2.3.3/fractal_server/app/runner/extract_archive.py +0 -38
  23. fractal_server-2.3.3/fractal_server/app/runner/v2/_slurm/get_slurm_config.py +0 -182
  24. {fractal_server-2.3.3 → fractal_server-2.3.5}/LICENSE +0 -0
  25. {fractal_server-2.3.3 → fractal_server-2.3.5}/README.md +0 -0
  26. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/__main__.py +0 -0
  27. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/alembic.ini +0 -0
  28. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/__init__.py +0 -0
  29. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/db/__init__.py +0 -0
  30. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/models/__init__.py +0 -0
  31. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/models/linkuserproject.py +0 -0
  32. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/models/security.py +0 -0
  33. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/models/v1/__init__.py +0 -0
  34. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/models/v1/dataset.py +0 -0
  35. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/models/v1/job.py +0 -0
  36. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/models/v1/project.py +0 -0
  37. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/models/v1/state.py +0 -0
  38. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/models/v1/task.py +0 -0
  39. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/models/v1/workflow.py +0 -0
  40. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/models/v2/__init__.py +0 -0
  41. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/models/v2/collection_state.py +0 -0
  42. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/models/v2/dataset.py +0 -0
  43. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/models/v2/job.py +0 -0
  44. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/models/v2/project.py +0 -0
  45. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/models/v2/task.py +0 -0
  46. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/models/v2/workflow.py +0 -0
  47. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/models/v2/workflowtask.py +0 -0
  48. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/routes/__init__.py +0 -0
  49. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/routes/admin/__init__.py +0 -0
  50. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/routes/admin/v1.py +0 -0
  51. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/routes/admin/v2.py +0 -0
  52. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/routes/api/v1/__init__.py +0 -0
  53. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/routes/api/v1/_aux_functions.py +0 -0
  54. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/routes/api/v1/dataset.py +0 -0
  55. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/routes/api/v1/job.py +0 -0
  56. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/routes/api/v1/task.py +0 -0
  57. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/routes/api/v1/task_collection.py +0 -0
  58. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/routes/api/v1/workflow.py +0 -0
  59. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/routes/api/v1/workflowtask.py +0 -0
  60. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/routes/api/v2/__init__.py +0 -0
  61. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/routes/api/v2/_aux_functions.py +0 -0
  62. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/routes/api/v2/dataset.py +0 -0
  63. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/routes/api/v2/images.py +0 -0
  64. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/routes/api/v2/job.py +0 -0
  65. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/routes/api/v2/project.py +0 -0
  66. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/routes/api/v2/status.py +0 -0
  67. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/routes/api/v2/submit.py +0 -0
  68. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/routes/api/v2/task.py +0 -0
  69. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/routes/api/v2/task_collection.py +0 -0
  70. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/routes/api/v2/task_collection_custom.py +0 -0
  71. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/routes/api/v2/task_legacy.py +0 -0
  72. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/routes/api/v2/workflow.py +0 -0
  73. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/routes/api/v2/workflowtask.py +0 -0
  74. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/routes/auth.py +0 -0
  75. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/routes/aux/__init__.py +0 -0
  76. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/routes/aux/_job.py +0 -0
  77. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/routes/aux/_runner.py +0 -0
  78. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/.gitignore +0 -0
  79. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/__init__.py +0 -0
  80. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/async_wrap.py +0 -0
  81. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/components.py +0 -0
  82. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/exceptions.py +0 -0
  83. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/executors/__init__.py +0 -0
  84. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/executors/slurm/__init__.py +0 -0
  85. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/executors/slurm/_batching.py +0 -0
  86. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/executors/slurm/remote.py +0 -0
  87. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/executors/slurm/ssh/__init__.py +0 -0
  88. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/executors/slurm/ssh/_executor_wait_thread.py +0 -0
  89. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/executors/slurm/ssh/_slurm_job.py +0 -0
  90. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/executors/slurm/sudo/__init__.py +0 -0
  91. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/executors/slurm/sudo/_check_jobs_status.py +0 -0
  92. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/executors/slurm/sudo/_executor_wait_thread.py +0 -0
  93. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/executors/slurm/sudo/_subprocess_run_as_user.py +0 -0
  94. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/filenames.py +0 -0
  95. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/set_start_and_last_task_index.py +0 -0
  96. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/shutdown.py +0 -0
  97. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/task_files.py +0 -0
  98. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/v1/__init__.py +0 -0
  99. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/v1/_common.py +0 -0
  100. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/v1/_local/__init__.py +0 -0
  101. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/v1/_local/_local_config.py +0 -0
  102. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/v1/_local/_submit_setup.py +0 -0
  103. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/v1/_local/executor.py +0 -0
  104. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/v1/_slurm/__init__.py +0 -0
  105. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/v1/_slurm/_submit_setup.py +0 -0
  106. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/v1/_slurm/get_slurm_config.py +0 -0
  107. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/v1/common.py +0 -0
  108. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/v1/handle_failed_job.py +0 -0
  109. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/v2/_local/__init__.py +0 -0
  110. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/v2/_local/_local_config.py +0 -0
  111. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/v2/_local/_submit_setup.py +0 -0
  112. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/v2/_local/executor.py +0 -0
  113. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/v2/_local_experimental/__init__.py +0 -0
  114. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/v2/_local_experimental/_local_config.py +0 -0
  115. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/v2/_local_experimental/_submit_setup.py +0 -0
  116. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/v2/_local_experimental/executor.py +0 -0
  117. {fractal_server-2.3.3/fractal_server/tasks/v1 → fractal_server-2.3.5/fractal_server/app/runner/v2/_slurm_common}/__init__.py +0 -0
  118. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/v2/_slurm_ssh/__init__.py +0 -0
  119. {fractal_server-2.3.3/fractal_server/app/runner/v2/_slurm → fractal_server-2.3.5/fractal_server/app/runner/v2/_slurm_sudo}/__init__.py +0 -0
  120. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/v2/deduplicate_list.py +0 -0
  121. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/v2/handle_failed_job.py +0 -0
  122. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/v2/merge_outputs.py +0 -0
  123. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/v2/runner.py +0 -0
  124. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/v2/runner_functions.py +0 -0
  125. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/v2/runner_functions_low_level.py +0 -0
  126. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/v2/task_interface.py +0 -0
  127. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/v2/v1_compat.py +0 -0
  128. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/runner/versions.py +0 -0
  129. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/schemas/__init__.py +0 -0
  130. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/schemas/_validators.py +0 -0
  131. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/schemas/user.py +0 -0
  132. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/schemas/v1/__init__.py +0 -0
  133. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/schemas/v1/applyworkflow.py +0 -0
  134. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/schemas/v1/dataset.py +0 -0
  135. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/schemas/v1/dumps.py +0 -0
  136. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/schemas/v1/manifest.py +0 -0
  137. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/schemas/v1/project.py +0 -0
  138. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/schemas/v1/state.py +0 -0
  139. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/schemas/v1/task.py +0 -0
  140. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/schemas/v1/task_collection.py +0 -0
  141. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/schemas/v1/workflow.py +0 -0
  142. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/schemas/v2/__init__.py +0 -0
  143. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/schemas/v2/dataset.py +0 -0
  144. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/schemas/v2/dumps.py +0 -0
  145. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/schemas/v2/job.py +0 -0
  146. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/schemas/v2/manifest.py +0 -0
  147. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/schemas/v2/project.py +0 -0
  148. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/schemas/v2/status.py +0 -0
  149. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/schemas/v2/task.py +0 -0
  150. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/schemas/v2/task_collection.py +0 -0
  151. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/schemas/v2/workflow.py +0 -0
  152. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/schemas/v2/workflowtask.py +0 -0
  153. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/app/security/__init__.py +0 -0
  154. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/data_migrations/README.md +0 -0
  155. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/gunicorn_fractal.py +0 -0
  156. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/images/__init__.py +0 -0
  157. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/images/models.py +0 -0
  158. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/images/tools.py +0 -0
  159. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/logger.py +0 -0
  160. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/migrations/README +0 -0
  161. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/migrations/env.py +0 -0
  162. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/migrations/script.py.mako +0 -0
  163. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/migrations/versions/4c308bcaea2b_add_task_args_schema_and_task_args_.py +0 -0
  164. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/migrations/versions/4cedeb448a53_workflowtask_foreign_keys_not_nullables.py +0 -0
  165. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/migrations/versions/50a13d6138fd_initial_schema.py +0 -0
  166. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/migrations/versions/5bf02391cfef_v2.py +0 -0
  167. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/migrations/versions/70e77f1c38b0_add_applyworkflow_first_task_index_and_.py +0 -0
  168. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/migrations/versions/71eefd1dd202_add_slurm_accounts.py +0 -0
  169. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/migrations/versions/84bf0fffde30_add_dumps_to_applyworkflow.py +0 -0
  170. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/migrations/versions/8f79bd162e35_add_docs_info_and_docs_link_to_task_.py +0 -0
  171. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/migrations/versions/97f444d47249_add_applyworkflow_project_dump.py +0 -0
  172. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/migrations/versions/99ea79d9e5d2_add_dataset_history.py +0 -0
  173. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/migrations/versions/9fd26a2b0de4_add_workflow_timestamp_created.py +0 -0
  174. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/migrations/versions/a7f4d6137b53_add_workflow_dump_to_applyworkflow.py +0 -0
  175. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/migrations/versions/d4fe3708d309_make_applyworkflow_workflow_dump_non_.py +0 -0
  176. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/migrations/versions/e75cac726012_make_applyworkflow_start_timestamp_not_.py +0 -0
  177. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/migrations/versions/efa89c30e0a4_add_project_timestamp_created.py +0 -0
  178. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/migrations/versions/f384e1c0cf5d_drop_task_default_args_columns.py +0 -0
  179. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/py.typed +0 -0
  180. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/ssh/__init__.py +0 -0
  181. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/string_tools.py +0 -0
  182. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/syringe.py +0 -0
  183. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/tasks/__init__.py +0 -0
  184. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/tasks/utils.py +0 -0
  185. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/tasks/v1/_TaskCollectPip.py +0 -0
  186. {fractal_server-2.3.3/fractal_server/tasks/v2 → fractal_server-2.3.5/fractal_server/tasks/v1}/__init__.py +0 -0
  187. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/tasks/v1/background_operations.py +0 -0
  188. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/tasks/v1/endpoint_operations.py +0 -0
  189. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/tasks/v1/get_collection_data.py +0 -0
  190. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/tasks/v1/utils.py +0 -0
  191. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/tasks/v2/_TaskCollectPip.py +0 -0
  192. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/tasks/v2/_venv_pip.py +0 -0
  193. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/tasks/v2/background_operations.py +0 -0
  194. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/tasks/v2/background_operations_ssh.py +0 -0
  195. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/tasks/v2/endpoint_operations.py +0 -0
  196. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/tasks/v2/templates/_1_create_venv.sh +0 -0
  197. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/tasks/v2/templates/_2_upgrade_pip.sh +0 -0
  198. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/tasks/v2/templates/_3_pip_install.sh +0 -0
  199. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/tasks/v2/templates/_4_pip_freeze.sh +0 -0
  200. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/tasks/v2/templates/_5_pip_show.sh +0 -0
  201. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/tasks/v2/utils.py +0 -0
  202. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/urls.py +0 -0
  203. {fractal_server-2.3.3 → fractal_server-2.3.5}/fractal_server/utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: fractal-server
3
- Version: 2.3.3
3
+ Version: 2.3.5
4
4
  Summary: Server component of the Fractal analytics platform
5
5
  Home-page: https://github.com/fractal-analytics-platform/fractal-server
6
6
  License: BSD-3-Clause
@@ -0,0 +1 @@
1
+ __VERSION__ = "2.3.5"
@@ -2,9 +2,12 @@
2
2
  `api` module
3
3
  """
4
4
  from fastapi import APIRouter
5
+ from fastapi import Depends
5
6
 
6
7
  from ....config import get_settings
7
8
  from ....syringe import Inject
9
+ from ...models.security import UserOAuth
10
+ from ...security import current_active_superuser
8
11
 
9
12
 
10
13
  router_api = APIRouter()
@@ -17,3 +20,9 @@ async def alive():
17
20
  alive=True,
18
21
  version=settings.PROJECT_VERSION,
19
22
  )
23
+
24
+
25
+ @router_api.get("/settings/")
26
+ async def view_settings(user: UserOAuth = Depends(current_active_superuser)):
27
+ settings = Inject(get_settings)
28
+ return settings.get_sanitized()
@@ -250,9 +250,18 @@ async def apply_workflow(
250
250
  db: AsyncSession = Depends(get_async_db),
251
251
  ) -> Optional[ApplyWorkflowReadV1]:
252
252
 
253
+ settings = Inject(get_settings)
254
+ if settings.FRACTAL_API_V1_MODE == "include_without_submission":
255
+ raise HTTPException(
256
+ status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
257
+ detail=(
258
+ "Legacy API is still accessible, "
259
+ "but the submission of legacy jobs is not available."
260
+ ),
261
+ )
262
+
253
263
  # Remove non-submitted V1 jobs from the app state when the list grows
254
264
  # beyond a threshold
255
- settings = Inject(get_settings)
256
265
  if (
257
266
  len(request.app.state.jobsV1)
258
267
  > settings.FRACTAL_API_MAX_JOB_LIST_LENGTH
@@ -0,0 +1,132 @@
1
+ """
2
+ Wrap `tar` compression command.
3
+
4
+ This module is used both locally (in the environment where `fractal-server`
5
+ is running) and remotely (as a standalon Python module, executed over SSH).
6
+
7
+ This is a twin-module of `extract_archive.py`.
8
+
9
+ The reason for using the `tar` command via `subprocess` rather than Python
10
+ built-in `tarfile` library has to do with performance issues we observed
11
+ when handling files which were just created within a SLURM job, and in the
12
+ context of a CephFS filesystem.
13
+ """
14
+ import shutil
15
+ import sys
16
+ from pathlib import Path
17
+
18
+ from fractal_server.app.runner.run_subprocess import run_subprocess
19
+ from fractal_server.logger import get_logger
20
+ from fractal_server.logger import set_logger
21
+
22
+
23
+ def copy_subfolder(src: Path, dest: Path, logger_name: str):
24
+ cmd_cp = f"cp -r {src.as_posix()} {dest.as_posix()}"
25
+ logger = get_logger(logger_name=logger_name)
26
+ logger.debug(f"{cmd_cp=}")
27
+ res = run_subprocess(cmd=cmd_cp, logger_name=logger_name)
28
+ return res
29
+
30
+
31
+ def create_tar_archive(
32
+ tarfile_path: Path,
33
+ subfolder_path_tmp_copy: Path,
34
+ logger_name: str,
35
+ remote_to_local: bool,
36
+ ):
37
+ logger = get_logger(logger_name)
38
+
39
+ if remote_to_local:
40
+ exclude_options = "--exclude *sbatch --exclude *_in_*.pickle "
41
+ else:
42
+ exclude_options = ""
43
+
44
+ cmd_tar = (
45
+ f"tar czf {tarfile_path} "
46
+ f"{exclude_options} "
47
+ f"--directory={subfolder_path_tmp_copy.as_posix()} "
48
+ "."
49
+ )
50
+ logger.debug(f"cmd tar:\n{cmd_tar}")
51
+ run_subprocess(cmd=cmd_tar, logger_name=logger_name)
52
+
53
+
54
+ def remove_temp_subfolder(subfolder_path_tmp_copy: Path, logger_name: str):
55
+ logger = get_logger(logger_name)
56
+ try:
57
+ logger.debug(f"Now remove {subfolder_path_tmp_copy}")
58
+ shutil.rmtree(subfolder_path_tmp_copy)
59
+ except Exception as e:
60
+ logger.debug(f"ERROR during shutil.rmtree: {e}")
61
+
62
+
63
+ def compress_folder(
64
+ subfolder_path: Path, remote_to_local: bool = False
65
+ ) -> str:
66
+ """
67
+ Compress e.g. `/path/archive` into `/path/archive.tar.gz`
68
+
69
+ Note that `/path/archive.tar.gz` may already exist. In this case, it will
70
+ be overwritten.
71
+
72
+ Args:
73
+ subfolder_path: Absolute path to the folder to compress.
74
+ remote_to_local: If `True`, exclude some files from the tar.gz archive.
75
+
76
+ Returns:
77
+ Absolute path to the tar.gz archive.
78
+ """
79
+
80
+ logger_name = "compress_folder"
81
+ logger = set_logger(logger_name)
82
+
83
+ logger.debug("START")
84
+ logger.debug(f"{subfolder_path=}")
85
+ parent_dir = subfolder_path.parent
86
+ subfolder_name = subfolder_path.name
87
+ tarfile_path = (parent_dir / f"{subfolder_name}.tar.gz").as_posix()
88
+ logger.debug(f"{tarfile_path=}")
89
+
90
+ subfolder_path_tmp_copy = (
91
+ subfolder_path.parent / f"{subfolder_path.name}_copy"
92
+ )
93
+ try:
94
+ copy_subfolder(
95
+ subfolder_path, subfolder_path_tmp_copy, logger_name=logger_name
96
+ )
97
+ create_tar_archive(
98
+ tarfile_path,
99
+ subfolder_path_tmp_copy,
100
+ logger_name=logger_name,
101
+ remote_to_local=remote_to_local,
102
+ )
103
+ return tarfile_path
104
+
105
+ except Exception as e:
106
+ logger.debug(f"ERROR: {e}")
107
+ sys.exit(1)
108
+
109
+ finally:
110
+ remove_temp_subfolder(subfolder_path_tmp_copy, logger_name=logger_name)
111
+
112
+
113
+ def main(sys_argv: list[str]):
114
+
115
+ help_msg = (
116
+ "Expected use:\n"
117
+ "python -m fractal_server.app.runner.compress_folder "
118
+ "path/to/folder [--remote-to-local]\n"
119
+ )
120
+ num_args = len(sys_argv[1:])
121
+ if num_args == 0:
122
+ sys.exit(f"Invalid argument.\n{help_msg}\nProvided: {sys_argv[1:]=}")
123
+ elif num_args == 1:
124
+ compress_folder(subfolder_path=Path(sys_argv[1]))
125
+ elif num_args == 2 and sys_argv[2] == "--remote-to-local":
126
+ compress_folder(subfolder_path=Path(sys_argv[1]), remote_to_local=True)
127
+ else:
128
+ sys.exit(f"Invalid argument.\n{help_msg}\nProvided: {sys_argv[1:]=}")
129
+
130
+
131
+ if __name__ == "__main__":
132
+ main(sys.argv)
@@ -62,6 +62,8 @@ class _SlurmConfigSet(BaseModel, extra=Extra.forbid):
62
62
  time: Optional[str]
63
63
  account: Optional[str]
64
64
  extra_lines: Optional[list[str]]
65
+ pre_submission_commands: Optional[list[str]]
66
+ gpus: Optional[str]
65
67
 
66
68
 
67
69
  class _BatchingConfigSet(BaseModel, extra=Extra.forbid):
@@ -219,6 +221,7 @@ class SlurmConfig(BaseModel, extra=Extra.forbid):
219
221
  constraint: Corresponds to SLURM option.
220
222
  gres: Corresponds to SLURM option.
221
223
  account: Corresponds to SLURM option.
224
+ gpus: Corresponds to SLURM option.
222
225
  time: Corresponds to SLURM option (WARNING: not fully supported).
223
226
  prefix: Prefix of configuration lines in SLURM submission scripts.
224
227
  shebang_line: Shebang line for SLURM submission scripts.
@@ -240,6 +243,8 @@ class SlurmConfig(BaseModel, extra=Extra.forbid):
240
243
  Key-value pairs to be included as `export`-ed variables in SLURM
241
244
  submission script, after prepending values with the user's cache
242
245
  directory.
246
+ pre_submission_commands: List of commands to be prepended to the sbatch
247
+ command.
243
248
  """
244
249
 
245
250
  # Required SLURM parameters (note that the integer attributes are those
@@ -254,6 +259,7 @@ class SlurmConfig(BaseModel, extra=Extra.forbid):
254
259
  job_name: Optional[str] = None
255
260
  constraint: Optional[str] = None
256
261
  gres: Optional[str] = None
262
+ gpus: Optional[str] = None
257
263
  time: Optional[str] = None
258
264
  account: Optional[str] = None
259
265
 
@@ -274,6 +280,8 @@ class SlurmConfig(BaseModel, extra=Extra.forbid):
274
280
  target_num_jobs: int
275
281
  max_num_jobs: int
276
282
 
283
+ pre_submission_commands: list[str] = Field(default_factory=list)
284
+
277
285
  def _sorted_extra_lines(self) -> list[str]:
278
286
  """
279
287
  Return a copy of `self.extra_lines`, where lines starting with
@@ -340,7 +348,14 @@ class SlurmConfig(BaseModel, extra=Extra.forbid):
340
348
  f"{self.prefix} --cpus-per-task={self.cpus_per_task}",
341
349
  f"{self.prefix} --mem={mem_per_job_MB}M",
342
350
  ]
343
- for key in ["job_name", "constraint", "gres", "time", "account"]:
351
+ for key in [
352
+ "job_name",
353
+ "constraint",
354
+ "gres",
355
+ "gpus",
356
+ "time",
357
+ "account",
358
+ ]:
344
359
  value = getattr(self, key)
345
360
  if value is not None:
346
361
  # Handle the `time` parameter
@@ -13,7 +13,6 @@
13
13
  import json
14
14
  import math
15
15
  import sys
16
- import tarfile
17
16
  import threading
18
17
  import time
19
18
  from concurrent.futures import Future
@@ -38,9 +37,11 @@ from ...slurm._slurm_config import SlurmConfig
38
37
  from .._batching import heuristics
39
38
  from ._executor_wait_thread import FractalSlurmWaitThread
40
39
  from fractal_server.app.runner.components import _COMPONENT_KEY_
40
+ from fractal_server.app.runner.compress_folder import compress_folder
41
41
  from fractal_server.app.runner.exceptions import JobExecutionError
42
42
  from fractal_server.app.runner.exceptions import TaskExecutionError
43
43
  from fractal_server.app.runner.executors.slurm.ssh._slurm_job import SlurmJob
44
+ from fractal_server.app.runner.extract_archive import extract_archive
44
45
  from fractal_server.config import get_settings
45
46
  from fractal_server.logger import set_logger
46
47
  from fractal_server.ssh._fabric import FractalSSH
@@ -822,17 +823,12 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
822
823
 
823
824
  # Create compressed subfolder archive (locally)
824
825
  local_subfolder = self.workflow_dir_local / subfolder_name
825
- tarfile_name = f"{subfolder_name}.tar.gz"
826
- tarfile_path_local = (
827
- self.workflow_dir_local / tarfile_name
828
- ).as_posix()
826
+ tarfile_path_local = compress_folder(local_subfolder)
827
+ tarfile_name = Path(tarfile_path_local).name
828
+ logger.info(f"Subfolder archive created at {tarfile_path_local}")
829
829
  tarfile_path_remote = (
830
830
  self.workflow_dir_remote / tarfile_name
831
831
  ).as_posix()
832
- with tarfile.open(tarfile_path_local, "w:gz") as tar:
833
- for this_file in local_subfolder.glob("*"):
834
- tar.add(this_file, arcname=this_file.name)
835
- logger.info(f"Subfolder archive created at {tarfile_path_local}")
836
832
 
837
833
  # Transfer archive
838
834
  t_0_put = time.perf_counter()
@@ -873,9 +869,22 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
873
869
 
874
870
  # Submit job to SLURM, and get jobid
875
871
  sbatch_command = f"sbatch --parsable {job.slurm_script_remote}"
876
- sbatch_stdout = self.fractal_ssh.run_command(
877
- cmd=sbatch_command,
878
- )
872
+ pre_submission_cmds = job.slurm_config.pre_submission_commands
873
+ if len(pre_submission_cmds) == 0:
874
+ sbatch_stdout = self.fractal_ssh.run_command(cmd=sbatch_command)
875
+ else:
876
+ logger.debug(f"Now using {pre_submission_cmds=}")
877
+ script_lines = pre_submission_cmds + [sbatch_command]
878
+ script_content = "\n".join(script_lines)
879
+ script_content = f"{script_content}\n"
880
+ script_path_remote = (
881
+ f"{job.slurm_script_remote.as_posix()}_wrapper.sh"
882
+ )
883
+ self.fractal_ssh.write_remote_file(
884
+ path=script_path_remote, content=script_content
885
+ )
886
+ cmd = f"bash {script_path_remote}"
887
+ sbatch_stdout = self.fractal_ssh.run_command(cmd=cmd)
879
888
 
880
889
  # Extract SLURM job ID from stdout
881
890
  try:
@@ -885,7 +894,9 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
885
894
  error_msg = (
886
895
  f"Submit command `{sbatch_command}` returned "
887
896
  f"`{stdout=}` which cannot be cast to an integer "
888
- f"SLURM-job ID. Original error:\n{str(e)}"
897
+ f"SLURM-job ID.\n"
898
+ f"Note that {pre_submission_cmds=}.\n"
899
+ f"Original error:\n{str(e)}"
889
900
  )
890
901
  logger.error(error_msg)
891
902
  raise JobExecutionError(info=error_msg)
@@ -1222,7 +1233,8 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
1222
1233
  tar_command = (
1223
1234
  f"{self.python_remote} "
1224
1235
  "-m fractal_server.app.runner.compress_folder "
1225
- f"{(self.workflow_dir_remote / subfolder_name).as_posix()}"
1236
+ f"{(self.workflow_dir_remote / subfolder_name).as_posix()} "
1237
+ "--remote-to-local"
1226
1238
  )
1227
1239
  stdout = self.fractal_ssh.run_command(cmd=tar_command)
1228
1240
  print(stdout)
@@ -1240,8 +1252,7 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
1240
1252
  )
1241
1253
 
1242
1254
  # Extract tarfile locally
1243
- with tarfile.open(tarfile_path_local) as tar:
1244
- tar.extractall(path=(self.workflow_dir_local / subfolder_name))
1255
+ extract_archive(Path(tarfile_path_local))
1245
1256
 
1246
1257
  t_1 = time.perf_counter()
1247
1258
  logger.info("[_get_subfolder_sftp] End - " f"elapsed: {t_1-t_0:.3f} s")
@@ -1121,6 +1121,12 @@ class FractalSlurmExecutor(SlurmExecutor):
1121
1121
  slurm_err_path=str(job.slurm_stderr),
1122
1122
  )
1123
1123
 
1124
+ # Print warning for ignored parameter
1125
+ if len(job.slurm_config.pre_submission_commands) > 0:
1126
+ logger.warning(
1127
+ f"Ignoring {job.slurm_config.pre_submission_commands=}."
1128
+ )
1129
+
1124
1130
  # Submit job via sbatch, and retrieve jobid
1125
1131
 
1126
1132
  # Write script content to a job.slurm_script
@@ -0,0 +1,85 @@
1
+ """
2
+ Wrap `tar` extraction command.
3
+
4
+ This module is used both locally (in the environment where `fractal-server`
5
+ is running) and remotely (as a standalon Python module, executed over SSH).
6
+
7
+ This is a twin-module of `compress_folder.py`.
8
+
9
+ The reason for using the `tar` command via `subprocess` rather than Python
10
+ built-in `tarfile` library has to do with performance issues we observed
11
+ when handling files which were just created within a SLURM job, and in the
12
+ context of a CephFS filesystem.
13
+ """
14
+ import sys
15
+ from pathlib import Path
16
+
17
+ from .run_subprocess import run_subprocess
18
+ from fractal_server.logger import set_logger
19
+
20
+
21
+ def _remove_suffix(*, string: str, suffix: str) -> str:
22
+ if string.endswith(suffix):
23
+ return string[: -len(suffix)]
24
+ else:
25
+ raise ValueError(f"Cannot remove {suffix=} from {string=}.")
26
+
27
+
28
+ def extract_archive(archive_path: Path):
29
+ """
30
+ Extract e.g. `/path/archive.tar.gz` archive into `/path/archive` folder
31
+
32
+ Note that `/path/archive` may already exist. In this case, files with
33
+ the same name are overwritten and new files are added.
34
+
35
+ Arguments:
36
+ archive_path: Absolute path to the archive file.
37
+ """
38
+
39
+ logger_name = "extract_archive"
40
+ logger = set_logger(logger_name)
41
+
42
+ logger.debug("START")
43
+ logger.debug(f"{archive_path.as_posix()=}")
44
+
45
+ # Check archive_path is valid
46
+ if not archive_path.exists():
47
+ sys.exit(f"Missing file {archive_path.as_posix()}.")
48
+
49
+ # Prepare subfolder path
50
+ parent_dir = archive_path.parent
51
+ subfolder_name = _remove_suffix(string=archive_path.name, suffix=".tar.gz")
52
+ subfolder_path = parent_dir / subfolder_name
53
+ logger.debug(f"{subfolder_path.as_posix()=}")
54
+
55
+ # Create subfolder
56
+ subfolder_path.mkdir(exist_ok=True)
57
+
58
+ # Run tar command
59
+ cmd_tar = (
60
+ f"tar -xzvf {archive_path} "
61
+ f"--directory={subfolder_path.as_posix()} "
62
+ "."
63
+ )
64
+ logger.debug(f"{cmd_tar=}")
65
+ run_subprocess(cmd=cmd_tar, logger_name=logger_name)
66
+
67
+ logger.debug("END")
68
+
69
+
70
+ def main(sys_argv: list[str]):
71
+ help_msg = (
72
+ "Expected use:\n"
73
+ "python -m fractal_server.app.runner.extract_archive "
74
+ "path/to/archive.tar.gz"
75
+ )
76
+
77
+ if len(sys_argv[1:]) != 1 or not sys_argv[1].endswith(".tar.gz"):
78
+ sys.exit(f"Invalid argument.\n{help_msg}\nProvided: {sys_argv[1:]=}")
79
+ else:
80
+ tarfile_path = Path(sys_argv[1])
81
+ extract_archive(tarfile_path)
82
+
83
+
84
+ if __name__ == "__main__":
85
+ main(sys.argv)
@@ -0,0 +1,27 @@
1
+ import shlex
2
+ import subprocess # nosec
3
+ from typing import Optional
4
+
5
+ from fractal_server.logger import get_logger
6
+
7
+
8
+ def run_subprocess(
9
+ cmd: str, logger_name: Optional[str] = None
10
+ ) -> subprocess.CompletedProcess:
11
+ logger = get_logger(logger_name)
12
+ try:
13
+ res = subprocess.run( # nosec
14
+ shlex.split(cmd), check=True, capture_output=True, encoding="utf-8"
15
+ )
16
+ return res
17
+ except subprocess.CalledProcessError as e:
18
+ logger.debug(
19
+ f"Command '{e.cmd}' returned non-zero exit status {e.returncode}."
20
+ )
21
+ logger.debug(f"stdout: {e.stdout}")
22
+ logger.debug(f"stderr: {e.stderr}")
23
+ raise e
24
+ except Exception as e:
25
+ logger.debug(f"An error occurred while running command: {cmd}")
26
+ logger.debug(str(e))
27
+ raise e
@@ -36,8 +36,8 @@ from ._local import process_workflow as local_process_workflow
36
36
  from ._local_experimental import (
37
37
  process_workflow as local_experimental_process_workflow,
38
38
  )
39
- from ._slurm import process_workflow as slurm_sudo_process_workflow
40
39
  from ._slurm_ssh import process_workflow as slurm_ssh_process_workflow
40
+ from ._slurm_sudo import process_workflow as slurm_sudo_process_workflow
41
41
  from .handle_failed_job import assemble_filters_failed_job
42
42
  from .handle_failed_job import assemble_history_failed_job
43
43
  from .handle_failed_job import assemble_images_failed_job
@@ -18,8 +18,6 @@ from fractal_server.app.runner.executors.slurm._slurm_config import (
18
18
 
19
19
  def get_slurm_config(
20
20
  wftask: WorkflowTaskV2,
21
- workflow_dir_local: Path,
22
- workflow_dir_remote: Path,
23
21
  which_type: Literal["non_parallel", "parallel"],
24
22
  config_path: Optional[Path] = None,
25
23
  ) -> SlurmConfig:
@@ -43,13 +41,6 @@ def get_slurm_config(
43
41
  wftask:
44
42
  WorkflowTask for which the SLURM configuration is is to be
45
43
  prepared.
46
- workflow_dir_local:
47
- Server-owned directory to store all task-execution-related relevant
48
- files (inputs, outputs, errors, and all meta files related to the
49
- job execution). Note: users cannot write directly to this folder.
50
- workflow_dir_remote:
51
- User-side directory with the same scope as `workflow_dir_local`,
52
- and where a user can write.
53
44
  config_path:
54
45
  Path of a Fractal SLURM configuration file; if `None`, use
55
46
  `FRACTAL_SLURM_CONFIG_FILE` variable from settings.
@@ -99,13 +90,13 @@ def get_slurm_config(
99
90
  # 1. This block of definitions takes priority over other definitions from
100
91
  # slurm_env which are not under the `needs_gpu` subgroup
101
92
  # 2. This block of definitions has lower priority than whatever comes next
102
- # (i.e. from WorkflowTask.meta).
93
+ # (i.e. from WorkflowTask.meta_parallel).
103
94
  if wftask_meta is not None:
104
95
  needs_gpu = wftask_meta.get("needs_gpu", False)
105
96
  else:
106
97
  needs_gpu = False
107
98
  logger.debug(f"[get_slurm_config] {needs_gpu=}")
108
- if needs_gpu and slurm_env.gpu_slurm_config is not None: # FIXME
99
+ if needs_gpu:
109
100
  for key, value in slurm_env.gpu_slurm_config.dict(
110
101
  exclude_unset=True, exclude={"mem"}
111
102
  ).items():
@@ -143,9 +134,9 @@ def get_slurm_config(
143
134
  )
144
135
  logger.error(error_msg)
145
136
  raise SlurmConfigError(error_msg)
146
- for key in ["time", "gres", "constraint"]:
137
+ for key in ["time", "gres", "gpus", "constraint"]:
147
138
  value = wftask_meta.get(key, None)
148
- if value:
139
+ if value is not None:
149
140
  slurm_dict[key] = value
150
141
  if wftask_meta is not None:
151
142
  extra_lines = wftask_meta.get("extra_lines", [])
@@ -17,8 +17,10 @@ from pathlib import Path
17
17
  from typing import Literal
18
18
 
19
19
  from ...task_files import get_task_file_paths
20
- from .get_slurm_config import get_slurm_config
21
20
  from fractal_server.app.models.v2 import WorkflowTaskV2
21
+ from fractal_server.app.runner.v2._slurm_common.get_slurm_config import (
22
+ get_slurm_config,
23
+ )
22
24
 
23
25
 
24
26
  def _slurm_submit_setup(
@@ -62,8 +64,6 @@ def _slurm_submit_setup(
62
64
  # Get SlurmConfig object
63
65
  slurm_config = get_slurm_config(
64
66
  wftask=wftask,
65
- workflow_dir_local=workflow_dir_local,
66
- workflow_dir_remote=workflow_dir_remote,
67
67
  which_type=which_type,
68
68
  )
69
69
 
@@ -17,8 +17,10 @@ from pathlib import Path
17
17
  from typing import Literal
18
18
 
19
19
  from ...task_files import get_task_file_paths
20
- from .get_slurm_config import get_slurm_config
21
20
  from fractal_server.app.models.v2 import WorkflowTaskV2
21
+ from fractal_server.app.runner.v2._slurm_common.get_slurm_config import (
22
+ get_slurm_config,
23
+ )
22
24
 
23
25
 
24
26
  def _slurm_submit_setup(
@@ -62,8 +64,6 @@ def _slurm_submit_setup(
62
64
  # Get SlurmConfig object
63
65
  slurm_config = get_slurm_config(
64
66
  wftask=wftask,
65
- workflow_dir_local=workflow_dir_local,
66
- workflow_dir_remote=workflow_dir_remote,
67
67
  which_type=which_type,
68
68
  )
69
69
 
@@ -546,7 +546,9 @@ class Settings(BaseSettings):
546
546
  attribute in their input-arguments JSON file.
547
547
  """
548
548
 
549
- FRACTAL_API_V1_MODE: Literal["include", "exclude"] = "include"
549
+ FRACTAL_API_V1_MODE: Literal[
550
+ "include", "include_without_submission", "exclude"
551
+ ] = "include"
550
552
  """
551
553
  Whether to include the v1 API.
552
554
  """
@@ -685,6 +687,25 @@ class Settings(BaseSettings):
685
687
  self.check_db()
686
688
  self.check_runner()
687
689
 
690
+ def get_sanitized(self) -> dict:
691
+ def _must_be_sanitized(string) -> bool:
692
+ if not string.upper().startswith("FRACTAL") or any(
693
+ s in string.upper()
694
+ for s in ["PASSWORD", "SECRET", "PWD", "TOKEN"]
695
+ ):
696
+ return True
697
+ else:
698
+ return False
699
+
700
+ sanitized_settings = {}
701
+ for k, v in self.dict().items():
702
+ if _must_be_sanitized(k):
703
+ sanitized_settings[k] = "***"
704
+ else:
705
+ sanitized_settings[k] = v
706
+
707
+ return sanitized_settings
708
+
688
709
 
689
710
  def get_settings(settings=Settings()) -> Settings:
690
711
  return settings
@@ -49,7 +49,7 @@ def collect_routers(app: FastAPI) -> None:
49
49
  settings = Inject(get_settings)
50
50
 
51
51
  app.include_router(router_api, prefix="/api")
52
- if settings.FRACTAL_API_V1_MODE == "include":
52
+ if settings.FRACTAL_API_V1_MODE.startswith("include"):
53
53
  app.include_router(router_api_v1, prefix="/api/v1")
54
54
  app.include_router(
55
55
  router_admin_v1, prefix="/admin/v1", tags=["V1 Admin area"]
@@ -306,6 +306,28 @@ class FractalSSH(object):
306
306
  cmd = f"rm -r {folder}"
307
307
  self.run_command(cmd=cmd)
308
308
 
309
+ def write_remote_file(
310
+ self,
311
+ *,
312
+ path: str,
313
+ content: str,
314
+ lock_timeout: Optional[float] = None,
315
+ ) -> None:
316
+ """
317
+ Open a remote file via SFTP and write it.
318
+
319
+ Args:
320
+ path: Absolute path
321
+ contents: File contents
322
+ lock_timeout:
323
+ """
324
+ actual_lock_timeout = self.default_lock_timeout
325
+ if lock_timeout is not None:
326
+ actual_lock_timeout = lock_timeout
327
+ with self.acquire_timeout(timeout=actual_lock_timeout):
328
+ with self.sftp().open(filename=path, mode="w") as f:
329
+ f.write(content)
330
+
309
331
 
310
332
  def get_ssh_connection(
311
333
  *,
@@ -1,6 +1,6 @@
1
1
  [tool.poetry]
2
2
  name = "fractal-server"
3
- version = "2.3.3"
3
+ version = "2.3.5"
4
4
  description = "Server component of the Fractal analytics platform"
5
5
  authors = [
6
6
  "Tommaso Comparin <tommaso.comparin@exact-lab.it>",
@@ -91,7 +91,7 @@ filterwarnings = [
91
91
  ]
92
92
 
93
93
  [tool.bumpver]
94
- current_version = "2.3.3"
94
+ current_version = "2.3.5"
95
95
  version_pattern = "MAJOR.MINOR.PATCH[PYTAGNUM]"
96
96
  commit_message = "bump version {old_version} -> {new_version}"
97
97
  commit = true
@@ -1 +0,0 @@
1
- __VERSION__ = "2.3.3"