fractal-server 2.15.7__tar.gz → 2.15.9__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fractal_server-2.15.7 → fractal_server-2.15.9}/PKG-INFO +1 -1
- fractal_server-2.15.9/fractal_server/__init__.py +1 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/models/security.py +1 -2
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/admin/v2/job.py +1 -1
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/admin/v2/task_group_lifecycle.py +2 -3
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/api/v2/_aux_functions_task_version_update.py +1 -1
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/api/v2/history.py +9 -9
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/api/v2/submit.py +22 -12
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/api/v2/task.py +1 -1
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/api/v2/task_group_lifecycle.py +1 -1
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/runner/executors/slurm_common/_slurm_config.py +0 -5
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/runner/executors/slurm_common/base_slurm_runner.py +89 -65
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/runner/executors/slurm_ssh/runner.py +59 -4
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/runner/executors/slurm_sudo/runner.py +1 -1
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/runner/v2/runner.py +1 -1
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/config.py +62 -22
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/images/tools.py +2 -2
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/ssh/_fabric.py +74 -79
- {fractal_server-2.15.7 → fractal_server-2.15.9}/pyproject.toml +3 -3
- fractal_server-2.15.7/fractal_server/__init__.py +0 -1
- {fractal_server-2.15.7 → fractal_server-2.15.9}/LICENSE +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/README.md +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/__main__.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/alembic.ini +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/__init__.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/db/__init__.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/models/__init__.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/models/linkusergroup.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/models/linkuserproject.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/models/user_settings.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/models/v2/__init__.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/models/v2/accounting.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/models/v2/dataset.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/models/v2/history.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/models/v2/job.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/models/v2/project.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/models/v2/task.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/models/v2/task_group.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/models/v2/workflow.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/models/v2/workflowtask.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/__init__.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/admin/__init__.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/admin/v2/__init__.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/admin/v2/accounting.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/admin/v2/impersonate.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/admin/v2/project.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/admin/v2/task.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/admin/v2/task_group.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/api/__init__.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/api/v2/__init__.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/api/v2/_aux_functions.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/api/v2/_aux_functions_history.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/api/v2/_aux_functions_task_lifecycle.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/api/v2/_aux_functions_tasks.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/api/v2/_aux_task_group_disambiguation.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/api/v2/dataset.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/api/v2/images.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/api/v2/job.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/api/v2/pre_submission_checks.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/api/v2/project.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/api/v2/status_legacy.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/api/v2/task_collection.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/api/v2/task_collection_custom.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/api/v2/task_collection_pixi.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/api/v2/task_group.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/api/v2/task_version_update.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/api/v2/workflow.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/api/v2/workflow_import.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/api/v2/workflowtask.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/auth/__init__.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/auth/_aux_auth.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/auth/current_user.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/auth/group.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/auth/login.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/auth/oauth.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/auth/register.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/auth/router.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/auth/users.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/aux/__init__.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/aux/_job.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/aux/_runner.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/aux/validate_user_settings.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/routes/pagination.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/runner/__init__.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/runner/components.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/runner/exceptions.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/runner/executors/__init__.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/runner/executors/base_runner.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/runner/executors/call_command_wrapper.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/runner/executors/local/__init__.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/runner/executors/local/get_local_config.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/runner/executors/local/runner.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/runner/executors/slurm_common/__init__.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/runner/executors/slurm_common/_batching.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/runner/executors/slurm_common/_job_states.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/runner/executors/slurm_common/get_slurm_config.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/runner/executors/slurm_common/remote.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/runner/executors/slurm_common/slurm_job_task_models.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/runner/executors/slurm_ssh/__init__.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/runner/executors/slurm_ssh/run_subprocess.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/runner/executors/slurm_ssh/tar_commands.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/runner/executors/slurm_sudo/__init__.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/runner/executors/slurm_sudo/_subprocess_run_as_user.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/runner/filenames.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/runner/set_start_and_last_task_index.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/runner/shutdown.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/runner/task_files.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/runner/v2/__init__.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/runner/v2/_local.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/runner/v2/_slurm_ssh.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/runner/v2/_slurm_sudo.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/runner/v2/db_tools.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/runner/v2/deduplicate_list.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/runner/v2/merge_outputs.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/runner/v2/runner_functions.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/runner/v2/submit_workflow.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/runner/v2/task_interface.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/runner/versions.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/schemas/__init__.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/schemas/user.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/schemas/user_group.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/schemas/user_settings.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/schemas/v2/__init__.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/schemas/v2/accounting.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/schemas/v2/dataset.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/schemas/v2/dumps.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/schemas/v2/history.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/schemas/v2/job.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/schemas/v2/manifest.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/schemas/v2/project.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/schemas/v2/status_legacy.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/schemas/v2/task.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/schemas/v2/task_collection.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/schemas/v2/task_group.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/schemas/v2/workflow.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/schemas/v2/workflowtask.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/security/__init__.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/security/signup_email.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/app/user_settings.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/data_migrations/2_14_10.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/data_migrations/README.md +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/data_migrations/tools.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/exceptions.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/gunicorn_fractal.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/images/__init__.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/images/models.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/images/status_tools.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/logger.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/main.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/migrations/env.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/migrations/naming_convention.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/migrations/versions/034a469ec2eb_task_groups.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/migrations/versions/091b01f51f88_add_usergroup_and_linkusergroup_table.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/migrations/versions/19eca0dd47a9_user_settings_project_dir.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/migrations/versions/1eac13a26c83_drop_v1_tables.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/migrations/versions/316140ff7ee1_remove_usersettings_cache_dir.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/migrations/versions/47351f8c7ebc_drop_dataset_filters.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/migrations/versions/4c308bcaea2b_add_task_args_schema_and_task_args_.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/migrations/versions/4cedeb448a53_workflowtask_foreign_keys_not_nullables.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/migrations/versions/501961cfcd85_remove_link_between_v1_and_v2_tasks_.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/migrations/versions/50a13d6138fd_initial_schema.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/migrations/versions/5bf02391cfef_v2.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/migrations/versions/70e77f1c38b0_add_applyworkflow_first_task_index_and_.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/migrations/versions/71eefd1dd202_add_slurm_accounts.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/migrations/versions/791ce783d3d8_add_indices.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/migrations/versions/84bf0fffde30_add_dumps_to_applyworkflow.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/migrations/versions/8e8f227a3e36_update_taskv2_post_2_7_0.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/migrations/versions/8f79bd162e35_add_docs_info_and_docs_link_to_task_.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/migrations/versions/94a47ea2d3ff_remove_cache_dir_slurm_user_and_slurm_.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/migrations/versions/969d84257cac_add_historyrun_task_id.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/migrations/versions/97f444d47249_add_applyworkflow_project_dump.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/migrations/versions/99ea79d9e5d2_add_dataset_history.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/migrations/versions/9c5ae74c9b98_add_user_settings_table.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/migrations/versions/9db60297b8b2_set_ondelete.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/migrations/versions/9fd26a2b0de4_add_workflow_timestamp_created.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/migrations/versions/a7f4d6137b53_add_workflow_dump_to_applyworkflow.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/migrations/versions/af1ef1c83c9b_add_accounting_tables.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/migrations/versions/af8673379a5c_drop_old_filter_columns.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/migrations/versions/b1e7f7a1ff71_task_group_for_pixi.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/migrations/versions/b3ffb095f973_json_to_jsonb.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/migrations/versions/c90a7c76e996_job_id_in_history_run.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/migrations/versions/d256a7379ab8_taskgroup_activity_and_venv_info_to_.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/migrations/versions/d4fe3708d309_make_applyworkflow_workflow_dump_non_.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/migrations/versions/da2cb2ac4255_user_group_viewer_paths.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/migrations/versions/db09233ad13a_split_filters_and_keep_old_columns.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/migrations/versions/e75cac726012_make_applyworkflow_start_timestamp_not_.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/migrations/versions/e81103413827_add_job_type_filters.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/migrations/versions/efa89c30e0a4_add_project_timestamp_created.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/migrations/versions/f37aceb45062_make_historyunit_logfile_required.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/migrations/versions/f384e1c0cf5d_drop_task_default_args_columns.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/migrations/versions/fbce16ff4e47_new_history_items.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/py.typed +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/ssh/__init__.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/string_tools.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/syringe.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/tasks/__init__.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/tasks/utils.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/tasks/v2/__init__.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/tasks/v2/local/__init__.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/tasks/v2/local/_utils.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/tasks/v2/local/collect.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/tasks/v2/local/collect_pixi.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/tasks/v2/local/deactivate.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/tasks/v2/local/deactivate_pixi.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/tasks/v2/local/reactivate.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/tasks/v2/local/reactivate_pixi.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/tasks/v2/ssh/__init__.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/tasks/v2/ssh/_utils.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/tasks/v2/ssh/collect.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/tasks/v2/ssh/collect_pixi.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/tasks/v2/ssh/deactivate.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/tasks/v2/ssh/deactivate_pixi.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/tasks/v2/ssh/reactivate.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/tasks/v2/ssh/reactivate_pixi.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/tasks/v2/templates/1_create_venv.sh +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/tasks/v2/templates/2_pip_install.sh +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/tasks/v2/templates/3_pip_freeze.sh +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/tasks/v2/templates/4_pip_show.sh +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/tasks/v2/templates/5_get_venv_size_and_file_number.sh +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/tasks/v2/templates/6_pip_install_from_freeze.sh +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/tasks/v2/templates/pixi_1_extract.sh +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/tasks/v2/templates/pixi_2_install.sh +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/tasks/v2/templates/pixi_3_post_install.sh +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/tasks/v2/utils_background.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/tasks/v2/utils_database.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/tasks/v2/utils_package_names.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/tasks/v2/utils_pixi.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/tasks/v2/utils_python_interpreter.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/tasks/v2/utils_templates.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/types/__init__.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/types/validators/__init__.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/types/validators/_common_validators.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/types/validators/_filter_validators.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/types/validators/_workflow_task_arguments_validators.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/urls.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/utils.py +0 -0
- {fractal_server-2.15.7 → fractal_server-2.15.9}/fractal_server/zip_tools.py +0 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__VERSION__ = "2.15.9"
|
|
@@ -109,7 +109,7 @@ async def view_job(
|
|
|
109
109
|
|
|
110
110
|
@router.get("/{job_id}/", response_model=JobReadV2)
|
|
111
111
|
async def view_single_job(
|
|
112
|
-
job_id: int
|
|
112
|
+
job_id: int,
|
|
113
113
|
show_tmp_logs: bool = False,
|
|
114
114
|
user: UserOAuth = Depends(current_active_superuser),
|
|
115
115
|
db: AsyncSession = Depends(get_async_db),
|
|
@@ -25,7 +25,6 @@ from fractal_server.app.routes.aux.validate_user_settings import (
|
|
|
25
25
|
from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
|
|
26
26
|
from fractal_server.app.schemas.v2 import TaskGroupActivityStatusV2
|
|
27
27
|
from fractal_server.app.schemas.v2 import TaskGroupActivityV2Read
|
|
28
|
-
from fractal_server.app.schemas.v2 import TaskGroupReadV2
|
|
29
28
|
from fractal_server.app.schemas.v2 import TaskGroupV2OriginEnum
|
|
30
29
|
from fractal_server.config import get_settings
|
|
31
30
|
from fractal_server.logger import set_logger
|
|
@@ -52,7 +51,7 @@ async def deactivate_task_group(
|
|
|
52
51
|
response: Response,
|
|
53
52
|
superuser: UserOAuth = Depends(current_active_superuser),
|
|
54
53
|
db: AsyncSession = Depends(get_async_db),
|
|
55
|
-
) ->
|
|
54
|
+
) -> TaskGroupActivityV2Read:
|
|
56
55
|
"""
|
|
57
56
|
Deactivate task-group venv
|
|
58
57
|
"""
|
|
@@ -157,7 +156,7 @@ async def reactivate_task_group(
|
|
|
157
156
|
response: Response,
|
|
158
157
|
superuser: UserOAuth = Depends(current_active_superuser),
|
|
159
158
|
db: AsyncSession = Depends(get_async_db),
|
|
160
|
-
) ->
|
|
159
|
+
) -> TaskGroupActivityV2Read:
|
|
161
160
|
"""
|
|
162
161
|
Deactivate task-group venv
|
|
163
162
|
"""
|
|
@@ -6,7 +6,7 @@ def get_new_workflow_task_meta(
|
|
|
6
6
|
old_workflow_task_meta: dict | None,
|
|
7
7
|
old_task_meta: dict | None,
|
|
8
8
|
new_task_meta: dict | None,
|
|
9
|
-
) -> dict[str, Any]:
|
|
9
|
+
) -> dict[str, Any] | None:
|
|
10
10
|
"""
|
|
11
11
|
Prepare new meta field based on old/new tasks and old workflow task.
|
|
12
12
|
"""
|
|
@@ -354,21 +354,21 @@ async def get_history_images(
|
|
|
354
354
|
# Setup prefix for logging
|
|
355
355
|
prefix = f"[DS{dataset.id}-WFT{workflowtask_id}-images]"
|
|
356
356
|
|
|
357
|
-
# (1)
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
type_filters=request_body.type_filters,
|
|
361
|
-
)
|
|
362
|
-
|
|
363
|
-
# (2) Extract valid values for attributes and types
|
|
364
|
-
attributes = aggregate_attributes(type_filtered_images)
|
|
357
|
+
# (1) Extract valid values for attributes and types
|
|
358
|
+
types = aggregate_types(dataset.images)
|
|
359
|
+
attributes = aggregate_attributes(dataset.images)
|
|
365
360
|
attributes[IMAGE_STATUS_KEY] = [
|
|
366
361
|
HistoryUnitStatusWithUnset.DONE,
|
|
367
362
|
HistoryUnitStatusWithUnset.SUBMITTED,
|
|
368
363
|
HistoryUnitStatusWithUnset.FAILED,
|
|
369
364
|
HistoryUnitStatusWithUnset.UNSET,
|
|
370
365
|
]
|
|
371
|
-
|
|
366
|
+
|
|
367
|
+
# (2) Apply type filters
|
|
368
|
+
type_filtered_images = filter_image_list(
|
|
369
|
+
images=dataset.images,
|
|
370
|
+
type_filters=request_body.type_filters,
|
|
371
|
+
)
|
|
372
372
|
|
|
373
373
|
# (3) Enrich images with status attribute
|
|
374
374
|
type_filtered_images_with_status = await enrich_images_unsorted_async(
|
|
@@ -156,6 +156,28 @@ async def apply_workflow(
|
|
|
156
156
|
if len(user_settings.slurm_accounts) > 0:
|
|
157
157
|
job_create.slurm_account = user_settings.slurm_accounts[0]
|
|
158
158
|
|
|
159
|
+
# User appropriate FractalSSH object
|
|
160
|
+
if settings.FRACTAL_RUNNER_BACKEND == "slurm_ssh":
|
|
161
|
+
ssh_config = dict(
|
|
162
|
+
user=user_settings.ssh_username,
|
|
163
|
+
host=user_settings.ssh_host,
|
|
164
|
+
key_path=user_settings.ssh_private_key_path,
|
|
165
|
+
)
|
|
166
|
+
fractal_ssh_list = request.app.state.fractal_ssh_list
|
|
167
|
+
try:
|
|
168
|
+
fractal_ssh = fractal_ssh_list.get(**ssh_config)
|
|
169
|
+
except Exception as e:
|
|
170
|
+
logger.error(
|
|
171
|
+
"Could not get a valid SSH connection in the submit endpoint. "
|
|
172
|
+
f"Original error: '{str(e)}'."
|
|
173
|
+
)
|
|
174
|
+
raise HTTPException(
|
|
175
|
+
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
|
176
|
+
detail="Error in setting up the SSH connection.",
|
|
177
|
+
)
|
|
178
|
+
else:
|
|
179
|
+
fractal_ssh = None
|
|
180
|
+
|
|
159
181
|
# Add new Job object to DB
|
|
160
182
|
job = JobV2(
|
|
161
183
|
project_id=project_id,
|
|
@@ -219,18 +241,6 @@ async def apply_workflow(
|
|
|
219
241
|
await db.merge(job)
|
|
220
242
|
await db.commit()
|
|
221
243
|
|
|
222
|
-
# User appropriate FractalSSH object
|
|
223
|
-
if settings.FRACTAL_RUNNER_BACKEND == "slurm_ssh":
|
|
224
|
-
ssh_config = dict(
|
|
225
|
-
user=user_settings.ssh_username,
|
|
226
|
-
host=user_settings.ssh_host,
|
|
227
|
-
key_path=user_settings.ssh_private_key_path,
|
|
228
|
-
)
|
|
229
|
-
fractal_ssh_list = request.app.state.fractal_ssh_list
|
|
230
|
-
fractal_ssh = fractal_ssh_list.get(**ssh_config)
|
|
231
|
-
else:
|
|
232
|
-
fractal_ssh = None
|
|
233
|
-
|
|
234
244
|
# Expunge user settings from db, to use in background task
|
|
235
245
|
db.expunge(user_settings)
|
|
236
246
|
|
|
@@ -69,7 +69,7 @@ async def get_list_task(
|
|
|
69
69
|
stm = stm.where(TaskV2.authors.icontains(author))
|
|
70
70
|
|
|
71
71
|
res = await db.execute(stm)
|
|
72
|
-
task_list = res.scalars().all()
|
|
72
|
+
task_list = list(res.scalars().all())
|
|
73
73
|
await db.close()
|
|
74
74
|
if args_schema is False:
|
|
75
75
|
for task in task_list:
|
|
@@ -66,7 +66,6 @@ class _SlurmConfigSet(BaseModel):
|
|
|
66
66
|
time: str | None = None
|
|
67
67
|
account: str | None = None
|
|
68
68
|
extra_lines: list[str] | None = None
|
|
69
|
-
pre_submission_commands: list[str] | None = None
|
|
70
69
|
gpus: str | None = None
|
|
71
70
|
|
|
72
71
|
|
|
@@ -253,8 +252,6 @@ class SlurmConfig(BaseModel):
|
|
|
253
252
|
Key-value pairs to be included as `export`-ed variables in SLURM
|
|
254
253
|
submission script, after prepending values with the user's cache
|
|
255
254
|
directory.
|
|
256
|
-
pre_submission_commands: List of commands to be prepended to the sbatch
|
|
257
|
-
command.
|
|
258
255
|
"""
|
|
259
256
|
|
|
260
257
|
model_config = ConfigDict(extra="forbid")
|
|
@@ -294,8 +291,6 @@ class SlurmConfig(BaseModel):
|
|
|
294
291
|
target_num_jobs: int
|
|
295
292
|
max_num_jobs: int
|
|
296
293
|
|
|
297
|
-
pre_submission_commands: list[str] = Field(default_factory=list)
|
|
298
|
-
|
|
299
294
|
def _sorted_extra_lines(self) -> list[str]:
|
|
300
295
|
"""
|
|
301
296
|
Return a copy of `self.extra_lines`, where lines starting with
|
|
@@ -137,7 +137,9 @@ class BaseSlurmRunner(BaseRunner):
|
|
|
137
137
|
def run_squeue(self, *, job_ids: list[str], **kwargs) -> str:
|
|
138
138
|
raise NotImplementedError("Implement in child class.")
|
|
139
139
|
|
|
140
|
-
def _is_squeue_error_recoverable(
|
|
140
|
+
def _is_squeue_error_recoverable(
|
|
141
|
+
self, exception: BaseException
|
|
142
|
+
) -> Literal[True]:
|
|
141
143
|
"""
|
|
142
144
|
Determine whether a `squeue` error is considered recoverable.
|
|
143
145
|
|
|
@@ -262,14 +264,25 @@ class BaseSlurmRunner(BaseRunner):
|
|
|
262
264
|
|
|
263
265
|
return new_slurm_config
|
|
264
266
|
|
|
265
|
-
def
|
|
267
|
+
def _prepare_single_slurm_job(
|
|
266
268
|
self,
|
|
267
269
|
*,
|
|
268
270
|
base_command: str,
|
|
269
271
|
slurm_job: SlurmJob,
|
|
270
272
|
slurm_config: SlurmConfig,
|
|
271
273
|
) -> str:
|
|
272
|
-
|
|
274
|
+
"""
|
|
275
|
+
Prepare submission script locally.
|
|
276
|
+
|
|
277
|
+
Args:
|
|
278
|
+
base_command: Base of task executable command.
|
|
279
|
+
slurm_job: `SlurmJob` object
|
|
280
|
+
slurm_config: Configuration for SLURM job
|
|
281
|
+
|
|
282
|
+
Returns:
|
|
283
|
+
Command to submit the SLURM job.
|
|
284
|
+
"""
|
|
285
|
+
logger.debug("[_prepare_single_slurm_job] START")
|
|
273
286
|
|
|
274
287
|
for task in slurm_job.tasks:
|
|
275
288
|
# Write input file
|
|
@@ -299,24 +312,10 @@ class BaseSlurmRunner(BaseRunner):
|
|
|
299
312
|
json.dump(task.parameters, f, indent=2)
|
|
300
313
|
|
|
301
314
|
logger.debug(
|
|
302
|
-
"[
|
|
315
|
+
"[_prepare_single_slurm_job] Written "
|
|
316
|
+
f"{task.input_file_local=}"
|
|
303
317
|
)
|
|
304
318
|
|
|
305
|
-
if self.slurm_runner_type == "ssh":
|
|
306
|
-
# Send input file (only relevant for SSH)
|
|
307
|
-
self.fractal_ssh.send_file(
|
|
308
|
-
local=task.input_file_local,
|
|
309
|
-
remote=task.input_file_remote,
|
|
310
|
-
)
|
|
311
|
-
self.fractal_ssh.send_file(
|
|
312
|
-
local=task.task_files.args_file_local,
|
|
313
|
-
remote=task.task_files.args_file_remote,
|
|
314
|
-
)
|
|
315
|
-
logger.debug(
|
|
316
|
-
"[_submit_single_sbatch] Transferred "
|
|
317
|
-
f"{task.input_file_local=}"
|
|
318
|
-
)
|
|
319
|
-
|
|
320
319
|
# Prepare commands to be included in SLURM submission script
|
|
321
320
|
cmdlines = []
|
|
322
321
|
for task in slurm_job.tasks:
|
|
@@ -353,7 +352,7 @@ class BaseSlurmRunner(BaseRunner):
|
|
|
353
352
|
]
|
|
354
353
|
)
|
|
355
354
|
script_lines = slurm_config.sort_script_lines(script_lines)
|
|
356
|
-
logger.debug(script_lines)
|
|
355
|
+
logger.debug(f"[_prepare_single_slurm_job] {script_lines=}")
|
|
357
356
|
|
|
358
357
|
# Always print output of `uname -n` and `pwd`
|
|
359
358
|
script_lines.append('\necho "Hostname: $(uname -n)"')
|
|
@@ -373,61 +372,64 @@ class BaseSlurmRunner(BaseRunner):
|
|
|
373
372
|
f"--mem={mem_per_task_MB}MB "
|
|
374
373
|
f"{cmd} &"
|
|
375
374
|
)
|
|
376
|
-
script_lines.append("wait\n")
|
|
377
|
-
script = "\n".join(script_lines)
|
|
375
|
+
script_lines.append("wait\n\n")
|
|
378
376
|
script_lines.append(
|
|
379
377
|
'echo "End time: $(date +"%Y-%m-%dT%H:%M:%S%z")"'
|
|
380
378
|
)
|
|
379
|
+
script = "\n".join(script_lines)
|
|
381
380
|
|
|
382
381
|
# Write submission script
|
|
383
382
|
with open(slurm_job.slurm_submission_script_local, "w") as f:
|
|
384
383
|
f.write(script)
|
|
385
384
|
logger.debug(
|
|
386
|
-
"[
|
|
385
|
+
"[_prepare_single_slurm_job] Written "
|
|
387
386
|
f"{slurm_job.slurm_submission_script_local=}"
|
|
388
387
|
)
|
|
389
388
|
|
|
390
389
|
if self.slurm_runner_type == "ssh":
|
|
391
|
-
self.fractal_ssh.send_file(
|
|
392
|
-
local=slurm_job.slurm_submission_script_local,
|
|
393
|
-
remote=slurm_job.slurm_submission_script_remote,
|
|
394
|
-
)
|
|
395
390
|
submit_command = (
|
|
396
|
-
"sbatch --parsable "
|
|
397
|
-
f"{slurm_job.slurm_submission_script_remote}"
|
|
391
|
+
f"sbatch --parsable {slurm_job.slurm_submission_script_remote}"
|
|
398
392
|
)
|
|
399
393
|
else:
|
|
400
394
|
submit_command = (
|
|
401
|
-
"sbatch --parsable "
|
|
402
|
-
f"{slurm_job.slurm_submission_script_local}"
|
|
395
|
+
f"sbatch --parsable {slurm_job.slurm_submission_script_local}"
|
|
403
396
|
)
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
397
|
+
logger.debug("[_prepare_single_slurm_job] END")
|
|
398
|
+
return submit_command
|
|
399
|
+
|
|
400
|
+
def _send_many_job_inputs(
|
|
401
|
+
self, *, workdir_local: Path, workdir_remote: Path
|
|
402
|
+
) -> None:
|
|
403
|
+
"""
|
|
404
|
+
Placeholder method.
|
|
405
|
+
|
|
406
|
+
This method is intentionally left unimplemented in the base class.
|
|
407
|
+
Subclasses must override it to provide the logic for transferring
|
|
408
|
+
input data.
|
|
409
|
+
"""
|
|
410
|
+
pass
|
|
411
|
+
|
|
412
|
+
def _submit_single_sbatch(
|
|
413
|
+
self,
|
|
414
|
+
*,
|
|
415
|
+
submit_command: str,
|
|
416
|
+
slurm_job: SlurmJob,
|
|
417
|
+
) -> None:
|
|
418
|
+
"""
|
|
419
|
+
Run `sbatch` and add the `slurm_job` to `self.jobs`.
|
|
420
|
+
|
|
421
|
+
Args:
|
|
422
|
+
submit_command:
|
|
423
|
+
The SLURM submission command prepared in
|
|
424
|
+
`self._prepare_single_slurm_job`.
|
|
425
|
+
slurm_job: The `SlurmJob` object.
|
|
426
|
+
"""
|
|
427
|
+
|
|
428
|
+
logger.debug("[_submit_single_sbatch] START")
|
|
429
429
|
|
|
430
430
|
# Submit SLURM job and retrieve job ID
|
|
431
|
+
logger.debug(f"[_submit_single_sbatch] Now run {submit_command=}")
|
|
432
|
+
sbatch_stdout = self._run_remote_cmd(submit_command)
|
|
431
433
|
logger.info(f"[_submit_single_sbatch] {sbatch_stdout=}")
|
|
432
434
|
stdout = sbatch_stdout.strip("\n")
|
|
433
435
|
submitted_job_id = int(stdout)
|
|
@@ -623,11 +625,19 @@ class BaseSlurmRunner(BaseRunner):
|
|
|
623
625
|
)
|
|
624
626
|
|
|
625
627
|
config.parallel_tasks_per_job = 1
|
|
626
|
-
self.
|
|
628
|
+
submit_command = self._prepare_single_slurm_job(
|
|
627
629
|
base_command=base_command,
|
|
628
630
|
slurm_job=slurm_job,
|
|
629
631
|
slurm_config=config,
|
|
630
632
|
)
|
|
633
|
+
self._send_many_job_inputs(
|
|
634
|
+
workdir_local=workdir_local,
|
|
635
|
+
workdir_remote=workdir_remote,
|
|
636
|
+
)
|
|
637
|
+
self._submit_single_sbatch(
|
|
638
|
+
submit_command=submit_command,
|
|
639
|
+
slurm_job=slurm_job,
|
|
640
|
+
)
|
|
631
641
|
logger.debug(f"[submit] END submission phase, {self.job_ids=}")
|
|
632
642
|
|
|
633
643
|
create_accounting_record_slurm(
|
|
@@ -726,8 +736,8 @@ class BaseSlurmRunner(BaseRunner):
|
|
|
726
736
|
status=HistoryUnitStatus.FAILED,
|
|
727
737
|
db_sync=db,
|
|
728
738
|
)
|
|
729
|
-
results = {}
|
|
730
|
-
exceptions = {
|
|
739
|
+
results: dict[int, Any] = {}
|
|
740
|
+
exceptions: dict[int, BaseException] = {
|
|
731
741
|
ind: SHUTDOWN_EXCEPTION
|
|
732
742
|
for ind in range(len(list_parameters))
|
|
733
743
|
}
|
|
@@ -801,13 +811,25 @@ class BaseSlurmRunner(BaseRunner):
|
|
|
801
811
|
)
|
|
802
812
|
)
|
|
803
813
|
|
|
804
|
-
|
|
805
|
-
logger.debug("[multisubmit] Transfer files and submit jobs.")
|
|
814
|
+
submit_commands = []
|
|
806
815
|
for slurm_job in jobs_to_submit:
|
|
816
|
+
submit_commands.append(
|
|
817
|
+
self._prepare_single_slurm_job(
|
|
818
|
+
base_command=base_command,
|
|
819
|
+
slurm_job=slurm_job,
|
|
820
|
+
slurm_config=config,
|
|
821
|
+
)
|
|
822
|
+
)
|
|
823
|
+
self._send_many_job_inputs(
|
|
824
|
+
workdir_local=workdir_local,
|
|
825
|
+
workdir_remote=workdir_remote,
|
|
826
|
+
)
|
|
827
|
+
for slurm_job, submit_command in zip(
|
|
828
|
+
jobs_to_submit, submit_commands
|
|
829
|
+
):
|
|
807
830
|
self._submit_single_sbatch(
|
|
808
|
-
|
|
831
|
+
submit_command=submit_command,
|
|
809
832
|
slurm_job=slurm_job,
|
|
810
|
-
slurm_config=config,
|
|
811
833
|
)
|
|
812
834
|
|
|
813
835
|
logger.info(f"[multisubmit] END submission phase, {self.job_ids=}")
|
|
@@ -830,8 +852,10 @@ class BaseSlurmRunner(BaseRunner):
|
|
|
830
852
|
status=HistoryUnitStatus.FAILED,
|
|
831
853
|
db_sync=db,
|
|
832
854
|
)
|
|
833
|
-
results = {}
|
|
834
|
-
exceptions
|
|
855
|
+
results: dict[int, Any] = {}
|
|
856
|
+
exceptions: dict[int, BaseException] = {
|
|
857
|
+
ind: e for ind in range(len(list_parameters))
|
|
858
|
+
}
|
|
835
859
|
return results, exceptions
|
|
836
860
|
|
|
837
861
|
# Retrieval phase
|
|
@@ -166,12 +166,69 @@ class SlurmSSHRunner(BaseSlurmRunner):
|
|
|
166
166
|
stdout = self.fractal_ssh.run_command(cmd=cmd)
|
|
167
167
|
return stdout
|
|
168
168
|
|
|
169
|
+
def _send_many_job_inputs(
|
|
170
|
+
self, *, workdir_local: Path, workdir_remote: Path
|
|
171
|
+
) -> None:
|
|
172
|
+
"""
|
|
173
|
+
Compress, transfer, and extract a local working directory onto a remote
|
|
174
|
+
host.
|
|
175
|
+
|
|
176
|
+
This method creates a temporary `.tar.gz` archive of the given
|
|
177
|
+
`workdir_local`, transfers it to the remote machine via the configured
|
|
178
|
+
SSH connection, extracts it into `workdir_remote`, and removes the
|
|
179
|
+
temporary archive from both local and remote filesystems.
|
|
180
|
+
"""
|
|
181
|
+
|
|
182
|
+
logger.debug("[_send_many_job_inputs] START")
|
|
183
|
+
|
|
184
|
+
tar_path_local = workdir_local.with_suffix(".tar.gz")
|
|
185
|
+
tar_name = Path(tar_path_local).name
|
|
186
|
+
tar_path_remote = workdir_remote.parent / tar_name
|
|
187
|
+
|
|
188
|
+
tar_compression_cmd = get_tar_compression_cmd(
|
|
189
|
+
subfolder_path=workdir_local, filelist_path=None
|
|
190
|
+
)
|
|
191
|
+
_, tar_extraction_cmd = get_tar_extraction_cmd(
|
|
192
|
+
archive_path=tar_path_remote
|
|
193
|
+
)
|
|
194
|
+
rm_tar_cmd = f"rm {tar_path_remote.as_posix()}"
|
|
195
|
+
|
|
196
|
+
try:
|
|
197
|
+
run_subprocess(tar_compression_cmd, logger_name=logger.name)
|
|
198
|
+
logger.debug(
|
|
199
|
+
"[_send_many_job_inputs] "
|
|
200
|
+
f"{workdir_local=} compressed to {tar_path_local=}."
|
|
201
|
+
)
|
|
202
|
+
self.fractal_ssh.send_file(
|
|
203
|
+
local=tar_path_local.as_posix(),
|
|
204
|
+
remote=tar_path_remote.as_posix(),
|
|
205
|
+
)
|
|
206
|
+
logger.debug(
|
|
207
|
+
"[_send_many_job_inputs] "
|
|
208
|
+
f"{tar_path_local=} sent via SSH to {tar_path_remote=}."
|
|
209
|
+
)
|
|
210
|
+
self.fractal_ssh.run_command(cmd=tar_extraction_cmd)
|
|
211
|
+
logger.debug(
|
|
212
|
+
"[_send_many_job_inputs] "
|
|
213
|
+
f"{tar_path_remote=} extracted to {workdir_remote=}."
|
|
214
|
+
)
|
|
215
|
+
self.fractal_ssh.run_command(cmd=rm_tar_cmd)
|
|
216
|
+
logger.debug(
|
|
217
|
+
"[_send_many_job_inputs] "
|
|
218
|
+
f"{tar_path_remote=} removed from remote server."
|
|
219
|
+
)
|
|
220
|
+
except Exception as e:
|
|
221
|
+
raise e
|
|
222
|
+
finally:
|
|
223
|
+
Path(tar_path_local).unlink(missing_ok=True)
|
|
224
|
+
logger.debug(f"[_send_many_job_inputs] {tar_path_local=} removed.")
|
|
225
|
+
|
|
226
|
+
logger.debug("[_send_many_job_inputs] END.")
|
|
227
|
+
|
|
169
228
|
def run_squeue(
|
|
170
229
|
self,
|
|
171
230
|
*,
|
|
172
231
|
job_ids: list[str],
|
|
173
|
-
base_interval: float = 2.0,
|
|
174
|
-
max_attempts: int = 7,
|
|
175
232
|
) -> str:
|
|
176
233
|
"""
|
|
177
234
|
Run `squeue` for a set of SLURM job IDs.
|
|
@@ -205,8 +262,6 @@ class SlurmSSHRunner(BaseSlurmRunner):
|
|
|
205
262
|
try:
|
|
206
263
|
stdout = self.fractal_ssh.run_command(
|
|
207
264
|
cmd=cmd,
|
|
208
|
-
base_interval=base_interval,
|
|
209
|
-
max_attempts=max_attempts,
|
|
210
265
|
)
|
|
211
266
|
return stdout
|
|
212
267
|
except FractalSSHCommandError as e:
|
|
@@ -47,7 +47,7 @@ def _remove_status_from_attributes(
|
|
|
47
47
|
Drop attribute `IMAGE_STATUS_KEY` from all images.
|
|
48
48
|
"""
|
|
49
49
|
images_copy = deepcopy(images)
|
|
50
|
-
[img["attributes"].pop(IMAGE_STATUS_KEY) for img in images_copy]
|
|
50
|
+
[img["attributes"].pop(IMAGE_STATUS_KEY, None) for img in images_copy]
|
|
51
51
|
return images_copy
|
|
52
52
|
|
|
53
53
|
|
|
@@ -66,37 +66,74 @@ class MailSettings(BaseModel):
|
|
|
66
66
|
|
|
67
67
|
class PixiSettings(BaseModel):
|
|
68
68
|
"""
|
|
69
|
-
Configuration for Pixi
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
PIXI_CONCURRENT_DOWNLOADS:
|
|
84
|
-
Value of `--concurrent-downloads for `pixi install`.
|
|
85
|
-
TOKIO_WORKER_THREADS:
|
|
86
|
-
From tokio docs, "The core threads are where all asynchronous code
|
|
87
|
-
runs, and Tokio will by default spawn one for each CPU core. You can
|
|
88
|
-
use the environment variable TOKIO_WORKER_THREADS to override the
|
|
89
|
-
default value."
|
|
69
|
+
Configuration for Pixi Task collection.
|
|
70
|
+
|
|
71
|
+
In order to use Pixi for Task collection, you must have one or more Pixi
|
|
72
|
+
binaries in your machine
|
|
73
|
+
(see
|
|
74
|
+
[example/get_pixi.sh](https://github.com/fractal-analytics-platform/fractal-server/blob/main/example/get_pixi.sh)
|
|
75
|
+
for installation example).
|
|
76
|
+
|
|
77
|
+
To let Fractal Server use these binaries for Task collection, a JSON file
|
|
78
|
+
must be prepared with the data to populate `PixiSettings` (arguments with
|
|
79
|
+
default values may be omitted).
|
|
80
|
+
|
|
81
|
+
The path to this JSON file must then be provided to Fractal via the
|
|
82
|
+
environment variable `FRACTAL_PIXI_CONFIG_FILE`.
|
|
90
83
|
"""
|
|
91
84
|
|
|
92
85
|
versions: DictStrStr
|
|
86
|
+
"""
|
|
87
|
+
A dictionary with Pixi versions as keys and paths to the corresponding
|
|
88
|
+
folder as values.
|
|
89
|
+
|
|
90
|
+
E.g. let's assume that you have Pixi v0.47.0 at
|
|
91
|
+
`/pixi-path/0.47.0/bin/pixi` and Pixi v0.48.2 at
|
|
92
|
+
`/pixi-path/0.48.2/bin/pixi`, then
|
|
93
|
+
```json
|
|
94
|
+
"versions": {
|
|
95
|
+
"0.47.0": "/pixi-path/0.47.0",
|
|
96
|
+
"0.48.2": "/pixi-path/0.48.2"
|
|
97
|
+
}
|
|
98
|
+
```
|
|
99
|
+
"""
|
|
93
100
|
default_version: str
|
|
101
|
+
"""
|
|
102
|
+
Default Pixi version to be used for Task collection.
|
|
94
103
|
|
|
104
|
+
Must be a key of the `versions` dictionary.
|
|
105
|
+
"""
|
|
95
106
|
PIXI_CONCURRENT_SOLVES: int = 4
|
|
107
|
+
"""
|
|
108
|
+
Value of
|
|
109
|
+
[`--concurrent-solves`](https://pixi.sh/latest/reference/cli/pixi/install/#arg---concurrent-solves)
|
|
110
|
+
for `pixi install`.
|
|
111
|
+
"""
|
|
96
112
|
PIXI_CONCURRENT_DOWNLOADS: int = 4
|
|
113
|
+
"""
|
|
114
|
+
Value of
|
|
115
|
+
[`--concurrent-downloads`](https://pixi.sh/latest/reference/cli/pixi/install/#arg---concurrent-downloads)
|
|
116
|
+
for `pixi install`.
|
|
117
|
+
"""
|
|
97
118
|
TOKIO_WORKER_THREADS: int = 2
|
|
119
|
+
"""
|
|
120
|
+
From
|
|
121
|
+
[Tokio documentation](
|
|
122
|
+
https://docs.rs/tokio/latest/tokio/#cpu-bound-tasks-and-blocking-code
|
|
123
|
+
)
|
|
124
|
+
:
|
|
125
|
+
|
|
126
|
+
The core threads are where all asynchronous code runs,
|
|
127
|
+
and Tokio will by default spawn one for each CPU core.
|
|
128
|
+
You can use the environment variable `TOKIO_WORKER_THREADS` to override
|
|
129
|
+
the default value.
|
|
130
|
+
"""
|
|
98
131
|
DEFAULT_ENVIRONMENT: str = "default"
|
|
132
|
+
"""
|
|
133
|
+
"""
|
|
99
134
|
DEFAULT_PLATFORM: str = "linux-64"
|
|
135
|
+
"""
|
|
136
|
+
"""
|
|
100
137
|
|
|
101
138
|
@model_validator(mode="after")
|
|
102
139
|
def check_pixi_settings(self):
|
|
@@ -542,7 +579,7 @@ class Settings(BaseSettings):
|
|
|
542
579
|
else:
|
|
543
580
|
return "--no-cache-dir"
|
|
544
581
|
|
|
545
|
-
FRACTAL_MAX_PIP_VERSION: str = "
|
|
582
|
+
FRACTAL_MAX_PIP_VERSION: str = "25.2"
|
|
546
583
|
"""
|
|
547
584
|
Maximum value at which to update `pip` before performing task collection.
|
|
548
585
|
"""
|
|
@@ -578,6 +615,9 @@ class Settings(BaseSettings):
|
|
|
578
615
|
"""
|
|
579
616
|
|
|
580
617
|
FRACTAL_PIXI_CONFIG_FILE: Path | None = None
|
|
618
|
+
"""
|
|
619
|
+
Path to the Pixi configuration JSON file that will populate `PixiSettings`.
|
|
620
|
+
"""
|
|
581
621
|
|
|
582
622
|
pixi: PixiSettings | None = None
|
|
583
623
|
|