fractal-server 2.17.1a1__py3-none-any.whl → 2.17.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fractal_server/__init__.py +1 -1
- fractal_server/__main__.py +19 -18
- fractal_server/app/db/__init__.py +3 -3
- fractal_server/app/models/__init__.py +1 -0
- fractal_server/app/models/linkuserproject.py +3 -1
- fractal_server/app/models/security.py +21 -3
- fractal_server/app/models/v2/__init__.py +3 -1
- fractal_server/app/models/v2/accounting.py +9 -1
- fractal_server/app/models/v2/dataset.py +5 -1
- fractal_server/app/models/v2/history.py +15 -1
- fractal_server/app/models/v2/job.py +4 -0
- fractal_server/app/models/v2/profile.py +29 -0
- fractal_server/app/models/v2/project.py +4 -10
- fractal_server/app/models/v2/resource.py +4 -0
- fractal_server/app/models/v2/task_group.py +4 -3
- fractal_server/app/models/v2/workflow.py +2 -1
- fractal_server/app/routes/admin/v2/__init__.py +1 -2
- fractal_server/app/routes/admin/v2/accounting.py +1 -1
- fractal_server/app/routes/admin/v2/job.py +9 -9
- fractal_server/app/routes/admin/v2/profile.py +3 -2
- fractal_server/app/routes/admin/v2/resource.py +5 -5
- fractal_server/app/routes/admin/v2/task.py +28 -18
- fractal_server/app/routes/admin/v2/task_group.py +0 -1
- fractal_server/app/routes/admin/v2/task_group_lifecycle.py +1 -2
- fractal_server/app/routes/api/__init__.py +1 -0
- fractal_server/app/routes/api/v2/__init__.py +5 -6
- fractal_server/app/routes/api/v2/_aux_functions.py +70 -63
- fractal_server/app/routes/api/v2/_aux_functions_history.py +43 -20
- fractal_server/app/routes/api/v2/_aux_functions_task_lifecycle.py +2 -4
- fractal_server/app/routes/api/v2/_aux_functions_tasks.py +5 -7
- fractal_server/app/routes/api/v2/_aux_task_group_disambiguation.py +1 -2
- fractal_server/app/routes/api/v2/dataset.py +13 -32
- fractal_server/app/routes/api/v2/history.py +35 -21
- fractal_server/app/routes/api/v2/images.py +3 -2
- fractal_server/app/routes/api/v2/job.py +17 -14
- fractal_server/app/routes/api/v2/pre_submission_checks.py +5 -4
- fractal_server/app/routes/api/v2/project.py +22 -17
- fractal_server/app/routes/api/v2/status_legacy.py +12 -11
- fractal_server/app/routes/api/v2/submit.py +11 -12
- fractal_server/app/routes/api/v2/task.py +4 -3
- fractal_server/app/routes/api/v2/task_collection.py +28 -30
- fractal_server/app/routes/api/v2/task_collection_custom.py +8 -7
- fractal_server/app/routes/api/v2/task_collection_pixi.py +1 -2
- fractal_server/app/routes/api/v2/task_group.py +7 -6
- fractal_server/app/routes/api/v2/task_group_lifecycle.py +6 -6
- fractal_server/app/routes/api/v2/task_version_update.py +13 -12
- fractal_server/app/routes/api/v2/workflow.py +14 -31
- fractal_server/app/routes/api/v2/workflow_import.py +17 -19
- fractal_server/app/routes/api/v2/workflowtask.py +10 -12
- fractal_server/app/routes/auth/__init__.py +1 -3
- fractal_server/app/routes/auth/_aux_auth.py +1 -2
- fractal_server/app/routes/auth/current_user.py +4 -5
- fractal_server/app/routes/auth/group.py +7 -5
- fractal_server/app/routes/auth/login.py +1 -0
- fractal_server/app/routes/auth/oauth.py +4 -3
- fractal_server/app/routes/auth/register.py +4 -2
- fractal_server/app/routes/auth/users.py +10 -10
- fractal_server/app/routes/aux/_job.py +1 -1
- fractal_server/app/routes/aux/_runner.py +2 -2
- fractal_server/app/routes/pagination.py +1 -1
- fractal_server/app/schemas/user.py +3 -3
- fractal_server/app/schemas/v2/accounting.py +11 -0
- fractal_server/app/schemas/v2/dataset.py +28 -4
- fractal_server/app/schemas/v2/dumps.py +1 -0
- fractal_server/app/schemas/v2/manifest.py +4 -3
- fractal_server/app/schemas/v2/profile.py +53 -2
- fractal_server/app/schemas/v2/resource.py +109 -13
- fractal_server/app/schemas/v2/task.py +0 -1
- fractal_server/app/schemas/v2/task_collection.py +1 -1
- fractal_server/app/schemas/v2/workflowtask.py +4 -3
- fractal_server/app/security/__init__.py +4 -7
- fractal_server/app/security/signup_email.py +4 -5
- fractal_server/config/_data.py +36 -25
- fractal_server/config/_database.py +19 -20
- fractal_server/config/_email.py +30 -38
- fractal_server/config/_main.py +33 -52
- fractal_server/config/_oauth.py +17 -21
- fractal_server/exceptions.py +4 -0
- fractal_server/images/models.py +3 -3
- fractal_server/images/status_tools.py +4 -2
- fractal_server/logger.py +1 -1
- fractal_server/main.py +4 -3
- fractal_server/migrations/versions/034a469ec2eb_task_groups.py +4 -8
- fractal_server/migrations/versions/091b01f51f88_add_usergroup_and_linkusergroup_table.py +1 -1
- fractal_server/migrations/versions/0f5f85bb2ae7_add_pre_pinned_packages.py +1 -0
- fractal_server/migrations/versions/19eca0dd47a9_user_settings_project_dir.py +1 -1
- fractal_server/migrations/versions/1a83a5260664_rename.py +1 -1
- fractal_server/migrations/versions/1eac13a26c83_drop_v1_tables.py +1 -0
- fractal_server/migrations/versions/316140ff7ee1_remove_usersettings_cache_dir.py +1 -1
- fractal_server/migrations/versions/40d6d6511b20_add_index_to_history_models.py +47 -0
- fractal_server/migrations/versions/45fbb391d7af_make_resource_id_fk_non_nullable.py +1 -1
- fractal_server/migrations/versions/47351f8c7ebc_drop_dataset_filters.py +1 -0
- fractal_server/migrations/versions/49d0856e9569_drop_table.py +2 -3
- fractal_server/migrations/versions/4c308bcaea2b_add_task_args_schema_and_task_args_.py +1 -1
- fractal_server/migrations/versions/4cedeb448a53_workflowtask_foreign_keys_not_nullables.py +1 -1
- fractal_server/migrations/versions/501961cfcd85_remove_link_between_v1_and_v2_tasks_.py +2 -1
- fractal_server/migrations/versions/50a13d6138fd_initial_schema.py +7 -19
- fractal_server/migrations/versions/5bf02391cfef_v2.py +4 -10
- fractal_server/migrations/versions/70e77f1c38b0_add_applyworkflow_first_task_index_and_.py +1 -0
- fractal_server/migrations/versions/71eefd1dd202_add_slurm_accounts.py +1 -1
- fractal_server/migrations/versions/7673fe18c05d_remove_project_dir_server_default.py +1 -1
- fractal_server/migrations/versions/791ce783d3d8_add_indices.py +1 -1
- fractal_server/migrations/versions/83bc2ad3ffcc_2_17_0.py +1 -0
- fractal_server/migrations/versions/84bf0fffde30_add_dumps_to_applyworkflow.py +1 -0
- fractal_server/migrations/versions/8e8f227a3e36_update_taskv2_post_2_7_0.py +2 -4
- fractal_server/migrations/versions/8f79bd162e35_add_docs_info_and_docs_link_to_task_.py +1 -1
- fractal_server/migrations/versions/94a47ea2d3ff_remove_cache_dir_slurm_user_and_slurm_.py +1 -0
- fractal_server/migrations/versions/969d84257cac_add_historyrun_task_id.py +1 -1
- fractal_server/migrations/versions/97f444d47249_add_applyworkflow_project_dump.py +1 -1
- fractal_server/migrations/versions/981d588fe248_add_executor_error_log.py +1 -1
- fractal_server/migrations/versions/99ea79d9e5d2_add_dataset_history.py +2 -4
- fractal_server/migrations/versions/9c5ae74c9b98_add_user_settings_table.py +1 -1
- fractal_server/migrations/versions/9db60297b8b2_set_ondelete.py +1 -1
- fractal_server/migrations/versions/9fd26a2b0de4_add_workflow_timestamp_created.py +1 -1
- fractal_server/migrations/versions/a7f4d6137b53_add_workflow_dump_to_applyworkflow.py +1 -1
- fractal_server/migrations/versions/af1ef1c83c9b_add_accounting_tables.py +1 -0
- fractal_server/migrations/versions/af8673379a5c_drop_old_filter_columns.py +1 -0
- fractal_server/migrations/versions/b1e7f7a1ff71_task_group_for_pixi.py +1 -1
- fractal_server/migrations/versions/b3ffb095f973_json_to_jsonb.py +1 -0
- fractal_server/migrations/versions/c90a7c76e996_job_id_in_history_run.py +1 -1
- fractal_server/migrations/versions/caba9fb1ea5e_drop_useroauth_user_settings_id.py +1 -1
- fractal_server/migrations/versions/d256a7379ab8_taskgroup_activity_and_venv_info_to_.py +4 -9
- fractal_server/migrations/versions/d4fe3708d309_make_applyworkflow_workflow_dump_non_.py +1 -0
- fractal_server/migrations/versions/da2cb2ac4255_user_group_viewer_paths.py +1 -1
- fractal_server/migrations/versions/db09233ad13a_split_filters_and_keep_old_columns.py +1 -0
- fractal_server/migrations/versions/e0e717ae2f26_delete_linkuserproject_ondelete_project.py +50 -0
- fractal_server/migrations/versions/e75cac726012_make_applyworkflow_start_timestamp_not_.py +1 -0
- fractal_server/migrations/versions/e81103413827_add_job_type_filters.py +1 -1
- fractal_server/migrations/versions/efa89c30e0a4_add_project_timestamp_created.py +1 -0
- fractal_server/migrations/versions/f37aceb45062_make_historyunit_logfile_required.py +1 -1
- fractal_server/migrations/versions/f384e1c0cf5d_drop_task_default_args_columns.py +1 -0
- fractal_server/migrations/versions/fbce16ff4e47_new_history_items.py +4 -9
- fractal_server/runner/config/_local.py +8 -5
- fractal_server/runner/config/_slurm.py +37 -33
- fractal_server/runner/config/slurm_mem_to_MB.py +0 -1
- fractal_server/runner/executors/base_runner.py +29 -4
- fractal_server/runner/executors/local/get_local_config.py +1 -0
- fractal_server/runner/executors/local/runner.py +14 -13
- fractal_server/runner/executors/slurm_common/_batching.py +5 -10
- fractal_server/runner/executors/slurm_common/base_slurm_runner.py +53 -27
- fractal_server/runner/executors/slurm_common/get_slurm_config.py +14 -7
- fractal_server/runner/executors/slurm_common/remote.py +3 -1
- fractal_server/runner/executors/slurm_common/slurm_config.py +1 -0
- fractal_server/runner/executors/slurm_common/slurm_job_task_models.py +1 -3
- fractal_server/runner/executors/slurm_ssh/runner.py +16 -11
- fractal_server/runner/executors/slurm_ssh/tar_commands.py +1 -0
- fractal_server/runner/executors/slurm_sudo/_subprocess_run_as_user.py +1 -0
- fractal_server/runner/executors/slurm_sudo/runner.py +16 -11
- fractal_server/runner/task_files.py +9 -3
- fractal_server/runner/v2/_local.py +9 -4
- fractal_server/runner/v2/_slurm_ssh.py +11 -5
- fractal_server/runner/v2/_slurm_sudo.py +11 -5
- fractal_server/runner/v2/db_tools.py +0 -1
- fractal_server/runner/v2/deduplicate_list.py +2 -1
- fractal_server/runner/v2/runner.py +11 -14
- fractal_server/runner/v2/runner_functions.py +11 -14
- fractal_server/runner/v2/submit_workflow.py +7 -6
- fractal_server/ssh/_fabric.py +6 -13
- fractal_server/string_tools.py +0 -1
- fractal_server/syringe.py +1 -1
- fractal_server/tasks/config/_pixi.py +1 -1
- fractal_server/tasks/config/_python.py +16 -9
- fractal_server/tasks/utils.py +0 -1
- fractal_server/tasks/v2/local/_utils.py +1 -1
- fractal_server/tasks/v2/local/collect.py +10 -12
- fractal_server/tasks/v2/local/collect_pixi.py +9 -10
- fractal_server/tasks/v2/local/deactivate.py +7 -8
- fractal_server/tasks/v2/local/deactivate_pixi.py +4 -4
- fractal_server/tasks/v2/local/delete.py +1 -3
- fractal_server/tasks/v2/local/reactivate.py +7 -7
- fractal_server/tasks/v2/local/reactivate_pixi.py +7 -7
- fractal_server/tasks/v2/ssh/_utils.py +3 -3
- fractal_server/tasks/v2/ssh/collect.py +14 -19
- fractal_server/tasks/v2/ssh/collect_pixi.py +17 -19
- fractal_server/tasks/v2/ssh/deactivate.py +10 -8
- fractal_server/tasks/v2/ssh/deactivate_pixi.py +6 -5
- fractal_server/tasks/v2/ssh/delete.py +7 -5
- fractal_server/tasks/v2/ssh/reactivate.py +11 -11
- fractal_server/tasks/v2/ssh/reactivate_pixi.py +8 -9
- fractal_server/tasks/v2/templates/1_create_venv.sh +2 -0
- fractal_server/tasks/v2/templates/2_pip_install.sh +2 -0
- fractal_server/tasks/v2/templates/3_pip_freeze.sh +2 -0
- fractal_server/tasks/v2/templates/4_pip_show.sh +2 -0
- fractal_server/tasks/v2/templates/5_get_venv_size_and_file_number.sh +3 -1
- fractal_server/tasks/v2/templates/6_pip_install_from_freeze.sh +2 -0
- fractal_server/tasks/v2/templates/pixi_1_extract.sh +2 -0
- fractal_server/tasks/v2/templates/pixi_2_install.sh +2 -0
- fractal_server/tasks/v2/templates/pixi_3_post_install.sh +2 -0
- fractal_server/tasks/v2/utils_background.py +3 -3
- fractal_server/tasks/v2/utils_package_names.py +1 -2
- fractal_server/tasks/v2/utils_pixi.py +1 -3
- fractal_server/types/__init__.py +76 -1
- fractal_server/types/validators/_common_validators.py +1 -3
- fractal_server/types/validators/_workflow_task_arguments_validators.py +1 -2
- fractal_server/utils.py +1 -0
- fractal_server/zip_tools.py +34 -0
- {fractal_server-2.17.1a1.dist-info → fractal_server-2.17.2.dist-info}/METADATA +1 -1
- fractal_server-2.17.2.dist-info/RECORD +265 -0
- fractal_server/app/routes/admin/v2/project.py +0 -41
- fractal_server-2.17.1a1.dist-info/RECORD +0 -264
- {fractal_server-2.17.1a1.dist-info → fractal_server-2.17.2.dist-info}/WHEEL +0 -0
- {fractal_server-2.17.1a1.dist-info → fractal_server-2.17.2.dist-info}/entry_points.txt +0 -0
- {fractal_server-2.17.1a1.dist-info → fractal_server-2.17.2.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,14 +1,15 @@
|
|
|
1
1
|
from typing import Literal
|
|
2
2
|
|
|
3
|
-
from ._batching import heuristics
|
|
4
|
-
from .slurm_config import logger
|
|
5
|
-
from .slurm_config import SlurmConfig
|
|
6
3
|
from fractal_server.app.models.v2 import WorkflowTaskV2
|
|
7
4
|
from fractal_server.runner.config import JobRunnerConfigSLURM
|
|
8
5
|
from fractal_server.runner.config.slurm_mem_to_MB import slurm_mem_to_MB
|
|
9
6
|
from fractal_server.runner.exceptions import SlurmConfigError
|
|
10
7
|
from fractal_server.string_tools import interpret_as_bool
|
|
11
8
|
|
|
9
|
+
from ._batching import heuristics
|
|
10
|
+
from .slurm_config import SlurmConfig
|
|
11
|
+
from .slurm_config import logger
|
|
12
|
+
|
|
12
13
|
|
|
13
14
|
def _get_slurm_config_internal(
|
|
14
15
|
shared_config: JobRunnerConfigSLURM,
|
|
@@ -33,7 +34,6 @@ def _get_slurm_config_internal(
|
|
|
33
34
|
which_type:
|
|
34
35
|
Whether we should look at the non-parallel or parallel part
|
|
35
36
|
of `wftask`.
|
|
36
|
-
tot_tasks: Not used here, only present as a common interface.
|
|
37
37
|
|
|
38
38
|
Returns:
|
|
39
39
|
A ready-to-use `SlurmConfig` object.
|
|
@@ -138,9 +138,7 @@ def _get_slurm_config_internal(
|
|
|
138
138
|
# Job-batching parameters (if None, they will be determined heuristically)
|
|
139
139
|
if wftask_meta is not None:
|
|
140
140
|
tasks_per_job = wftask_meta.get("tasks_per_job", None)
|
|
141
|
-
parallel_tasks_per_job = wftask_meta.get(
|
|
142
|
-
"parallel_tasks_per_job", None
|
|
143
|
-
)
|
|
141
|
+
parallel_tasks_per_job = wftask_meta.get("parallel_tasks_per_job", None)
|
|
144
142
|
else:
|
|
145
143
|
tasks_per_job = None
|
|
146
144
|
parallel_tasks_per_job = None
|
|
@@ -162,6 +160,15 @@ def get_slurm_config(
|
|
|
162
160
|
which_type: Literal["non_parallel", "parallel"],
|
|
163
161
|
tot_tasks: int = 1,
|
|
164
162
|
) -> SlurmConfig:
|
|
163
|
+
"""
|
|
164
|
+
Get `SlurmConfig` object.
|
|
165
|
+
|
|
166
|
+
Args:
|
|
167
|
+
shared_config:
|
|
168
|
+
wftask:
|
|
169
|
+
which_type:
|
|
170
|
+
tot_tasks:
|
|
171
|
+
"""
|
|
165
172
|
config = _get_slurm_config_internal(
|
|
166
173
|
shared_config=shared_config,
|
|
167
174
|
wftask=wftask,
|
|
@@ -3,8 +3,10 @@ import json
|
|
|
3
3
|
import os
|
|
4
4
|
import sys
|
|
5
5
|
|
|
6
|
-
from ..call_command_wrapper import call_command_wrapper
|
|
7
6
|
from fractal_server import __VERSION__
|
|
7
|
+
from fractal_server.runner.executors.call_command_wrapper import (
|
|
8
|
+
call_command_wrapper,
|
|
9
|
+
)
|
|
8
10
|
|
|
9
11
|
|
|
10
12
|
class FractalVersionMismatch(RuntimeError):
|
|
@@ -24,9 +24,7 @@ class SlurmTask(BaseModel):
|
|
|
24
24
|
|
|
25
25
|
@property
|
|
26
26
|
def input_file_local_path(self) -> Path:
|
|
27
|
-
return
|
|
28
|
-
self.workdir_local / f"{self.prefix}-{self.component}-input.json"
|
|
29
|
-
)
|
|
27
|
+
return self.workdir_local / f"{self.prefix}-{self.component}-input.json"
|
|
30
28
|
|
|
31
29
|
@property
|
|
32
30
|
def input_file_remote_path(self) -> Path:
|
|
@@ -1,23 +1,32 @@
|
|
|
1
1
|
import time
|
|
2
2
|
from pathlib import Path
|
|
3
3
|
|
|
4
|
-
from ..slurm_common.base_slurm_runner import BaseSlurmRunner
|
|
5
|
-
from ..slurm_common.slurm_job_task_models import SlurmJob
|
|
6
|
-
from .run_subprocess import run_subprocess
|
|
7
|
-
from .tar_commands import get_tar_compression_cmd
|
|
8
|
-
from .tar_commands import get_tar_extraction_cmd
|
|
9
4
|
from fractal_server.app.models import Profile
|
|
10
5
|
from fractal_server.app.models import Resource
|
|
11
6
|
from fractal_server.logger import set_logger
|
|
12
7
|
from fractal_server.runner.config import JobRunnerConfigSLURM
|
|
8
|
+
from fractal_server.runner.executors.slurm_common.base_slurm_runner import (
|
|
9
|
+
BaseSlurmRunner,
|
|
10
|
+
)
|
|
11
|
+
from fractal_server.runner.executors.slurm_common.slurm_job_task_models import (
|
|
12
|
+
SlurmJob,
|
|
13
|
+
)
|
|
13
14
|
from fractal_server.ssh._fabric import FractalSSH
|
|
14
15
|
from fractal_server.ssh._fabric import FractalSSHCommandError
|
|
15
16
|
from fractal_server.ssh._fabric import FractalSSHTimeoutError
|
|
16
17
|
|
|
18
|
+
from .run_subprocess import run_subprocess
|
|
19
|
+
from .tar_commands import get_tar_compression_cmd
|
|
20
|
+
from .tar_commands import get_tar_extraction_cmd
|
|
21
|
+
|
|
17
22
|
logger = set_logger(__name__)
|
|
18
23
|
|
|
19
24
|
|
|
20
25
|
class SlurmSSHRunner(BaseSlurmRunner):
|
|
26
|
+
"""
|
|
27
|
+
Runner implementation for a computational `sudo_slurm` resource.
|
|
28
|
+
"""
|
|
29
|
+
|
|
21
30
|
fractal_ssh: FractalSSH
|
|
22
31
|
|
|
23
32
|
def __init__(
|
|
@@ -39,9 +48,7 @@ class SlurmSSHRunner(BaseSlurmRunner):
|
|
|
39
48
|
different SLURM jobs/tasks.
|
|
40
49
|
"""
|
|
41
50
|
self.fractal_ssh = fractal_ssh
|
|
42
|
-
self.shared_config = JobRunnerConfigSLURM(
|
|
43
|
-
**resource.jobs_runner_config
|
|
44
|
-
)
|
|
51
|
+
self.shared_config = JobRunnerConfigSLURM(**resource.jobs_runner_config)
|
|
45
52
|
logger.warning(self.fractal_ssh)
|
|
46
53
|
|
|
47
54
|
# Check SSH connection and try to recover from a closed-socket error
|
|
@@ -80,9 +87,7 @@ class SlurmSSHRunner(BaseSlurmRunner):
|
|
|
80
87
|
return None
|
|
81
88
|
|
|
82
89
|
t_0 = time.perf_counter()
|
|
83
|
-
logger.debug(
|
|
84
|
-
f"[_fetch_artifacts] START ({len(finished_slurm_jobs)=})."
|
|
85
|
-
)
|
|
90
|
+
logger.debug(f"[_fetch_artifacts] START ({len(finished_slurm_jobs)=}).")
|
|
86
91
|
|
|
87
92
|
# Extract `workdir_remote` and `workdir_local`
|
|
88
93
|
self.validate_slurm_jobs_workdirs(finished_slurm_jobs)
|
|
@@ -15,6 +15,7 @@ This module provides a set of tools similar to `subprocess.run`, `glob.glob` or
|
|
|
15
15
|
`os.path.exists`, but extended so that they can be executed on behalf of
|
|
16
16
|
another user. Note that this requires appropriate sudo permissions.
|
|
17
17
|
"""
|
|
18
|
+
|
|
18
19
|
import shlex
|
|
19
20
|
import subprocess # nosec
|
|
20
21
|
|
|
@@ -5,15 +5,20 @@ import subprocess # nosec
|
|
|
5
5
|
from concurrent.futures import ThreadPoolExecutor
|
|
6
6
|
from pathlib import Path
|
|
7
7
|
|
|
8
|
-
from ..slurm_common.base_slurm_runner import BaseSlurmRunner
|
|
9
|
-
from ..slurm_common.slurm_job_task_models import SlurmJob
|
|
10
|
-
from ._subprocess_run_as_user import _mkdir_as_user
|
|
11
|
-
from ._subprocess_run_as_user import _run_command_as_user
|
|
12
8
|
from fractal_server.app.models import Profile
|
|
13
9
|
from fractal_server.app.models import Resource
|
|
14
10
|
from fractal_server.logger import set_logger
|
|
15
11
|
from fractal_server.runner.config import JobRunnerConfigSLURM
|
|
16
12
|
from fractal_server.runner.exceptions import JobExecutionError
|
|
13
|
+
from fractal_server.runner.executors.slurm_common.base_slurm_runner import (
|
|
14
|
+
BaseSlurmRunner,
|
|
15
|
+
)
|
|
16
|
+
from fractal_server.runner.executors.slurm_common.slurm_job_task_models import (
|
|
17
|
+
SlurmJob,
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
from ._subprocess_run_as_user import _mkdir_as_user
|
|
21
|
+
from ._subprocess_run_as_user import _run_command_as_user
|
|
17
22
|
|
|
18
23
|
logger = set_logger(__name__)
|
|
19
24
|
|
|
@@ -40,7 +45,11 @@ def _subprocess_run_or_raise(
|
|
|
40
45
|
raise JobExecutionError(info=error_msg)
|
|
41
46
|
|
|
42
47
|
|
|
43
|
-
class
|
|
48
|
+
class SlurmSudoRunner(BaseSlurmRunner):
|
|
49
|
+
"""
|
|
50
|
+
Runner implementation for a computational `sudo_slurm` resource.
|
|
51
|
+
"""
|
|
52
|
+
|
|
44
53
|
slurm_user: str
|
|
45
54
|
slurm_account: str | None = None
|
|
46
55
|
|
|
@@ -63,9 +72,7 @@ class SudoSlurmRunner(BaseSlurmRunner):
|
|
|
63
72
|
"""
|
|
64
73
|
|
|
65
74
|
self.slurm_user = profile.username
|
|
66
|
-
self.shared_config = JobRunnerConfigSLURM(
|
|
67
|
-
**resource.jobs_runner_config
|
|
68
|
-
)
|
|
75
|
+
self.shared_config = JobRunnerConfigSLURM(**resource.jobs_runner_config)
|
|
69
76
|
|
|
70
77
|
super().__init__(
|
|
71
78
|
slurm_runner_type="sudo",
|
|
@@ -90,9 +97,7 @@ class SudoSlurmRunner(BaseSlurmRunner):
|
|
|
90
97
|
"""
|
|
91
98
|
Fetch artifacts for a single SLURM jobs.
|
|
92
99
|
"""
|
|
93
|
-
logger.debug(
|
|
94
|
-
f"[_fetch_artifacts_single_job] {job.slurm_job_id=} START"
|
|
95
|
-
)
|
|
100
|
+
logger.debug(f"[_fetch_artifacts_single_job] {job.slurm_job_id=} START")
|
|
96
101
|
source_target_list = [
|
|
97
102
|
(job.slurm_stdout_remote, job.slurm_stdout_local),
|
|
98
103
|
(job.slurm_stderr_remote, job.slurm_stderr_local),
|
|
@@ -86,9 +86,7 @@ class TaskFiles(BaseModel):
|
|
|
86
86
|
@property
|
|
87
87
|
def log_file_remote_path(self) -> Path:
|
|
88
88
|
self._check_component()
|
|
89
|
-
return
|
|
90
|
-
self.wftask_subfolder_remote / f"{self.prefix_component}-log.txt"
|
|
91
|
-
)
|
|
89
|
+
return self.wftask_subfolder_remote / f"{self.prefix_component}-log.txt"
|
|
92
90
|
|
|
93
91
|
@property
|
|
94
92
|
def log_file_remote(self) -> str:
|
|
@@ -141,8 +139,16 @@ def enrich_task_files_multisubmit(
|
|
|
141
139
|
) -> list[TaskFiles]:
|
|
142
140
|
"""
|
|
143
141
|
Expand `TaskFiles` objects with `component` and `prefix`.
|
|
142
|
+
|
|
143
|
+
Args:
|
|
144
|
+
tot_tasks: Total number of images to process.
|
|
145
|
+
batch_size: Batch size, where `0` means `batch_size=tot_tasks`.
|
|
146
|
+
base_task_files: Original `TaskFiles` object to be enriched.
|
|
144
147
|
"""
|
|
145
148
|
|
|
149
|
+
# Replace `batch_size=0` with `batch_size=tot_tasks`
|
|
150
|
+
batch_size = batch_size or tot_tasks
|
|
151
|
+
|
|
146
152
|
new_list_task_files: list[TaskFiles] = []
|
|
147
153
|
for absolute_index in range(tot_tasks):
|
|
148
154
|
ind_batch = absolute_index // batch_size
|
|
@@ -1,16 +1,21 @@
|
|
|
1
1
|
from pathlib import Path
|
|
2
2
|
|
|
3
|
-
from ..executors.local.get_local_config import get_local_backend_config
|
|
4
|
-
from ..executors.local.runner import LocalRunner
|
|
5
|
-
from ..set_start_and_last_task_index import set_start_and_last_task_index
|
|
6
|
-
from .runner import execute_tasks_v2
|
|
7
3
|
from fractal_server.app.models.v2 import DatasetV2
|
|
8
4
|
from fractal_server.app.models.v2 import Profile
|
|
9
5
|
from fractal_server.app.models.v2 import Resource
|
|
10
6
|
from fractal_server.app.models.v2 import WorkflowV2
|
|
7
|
+
from fractal_server.runner.executors.local.get_local_config import (
|
|
8
|
+
get_local_backend_config,
|
|
9
|
+
)
|
|
10
|
+
from fractal_server.runner.executors.local.runner import LocalRunner
|
|
11
|
+
from fractal_server.runner.set_start_and_last_task_index import (
|
|
12
|
+
set_start_and_last_task_index,
|
|
13
|
+
)
|
|
11
14
|
from fractal_server.ssh._fabric import FractalSSH
|
|
12
15
|
from fractal_server.types import AttributeFilters
|
|
13
16
|
|
|
17
|
+
from .runner import execute_tasks_v2
|
|
18
|
+
|
|
14
19
|
|
|
15
20
|
def process_workflow(
|
|
16
21
|
*,
|
|
@@ -15,20 +15,26 @@ Slurm Backend
|
|
|
15
15
|
|
|
16
16
|
This backend runs fractal workflows in a SLURM cluster.
|
|
17
17
|
"""
|
|
18
|
+
|
|
18
19
|
from pathlib import Path
|
|
19
20
|
|
|
20
|
-
from ...ssh._fabric import FractalSSH
|
|
21
|
-
from ..executors.slurm_common.get_slurm_config import get_slurm_config
|
|
22
|
-
from ..executors.slurm_ssh.runner import SlurmSSHRunner
|
|
23
|
-
from ..set_start_and_last_task_index import set_start_and_last_task_index
|
|
24
|
-
from .runner import execute_tasks_v2
|
|
25
21
|
from fractal_server.app.models.v2 import DatasetV2
|
|
26
22
|
from fractal_server.app.models.v2 import Profile
|
|
27
23
|
from fractal_server.app.models.v2 import Resource
|
|
28
24
|
from fractal_server.app.models.v2 import WorkflowV2
|
|
29
25
|
from fractal_server.logger import set_logger
|
|
26
|
+
from fractal_server.runner.executors.slurm_common.get_slurm_config import (
|
|
27
|
+
get_slurm_config,
|
|
28
|
+
)
|
|
29
|
+
from fractal_server.runner.executors.slurm_ssh.runner import SlurmSSHRunner
|
|
30
|
+
from fractal_server.runner.set_start_and_last_task_index import (
|
|
31
|
+
set_start_and_last_task_index,
|
|
32
|
+
)
|
|
33
|
+
from fractal_server.ssh._fabric import FractalSSH
|
|
30
34
|
from fractal_server.types import AttributeFilters
|
|
31
35
|
|
|
36
|
+
from .runner import execute_tasks_v2
|
|
37
|
+
|
|
32
38
|
logger = set_logger(__name__)
|
|
33
39
|
|
|
34
40
|
|
|
@@ -15,19 +15,25 @@ Slurm Backend
|
|
|
15
15
|
|
|
16
16
|
This backend runs fractal workflows in a SLURM cluster.
|
|
17
17
|
"""
|
|
18
|
+
|
|
18
19
|
from pathlib import Path
|
|
19
20
|
|
|
20
|
-
from ..executors.slurm_common.get_slurm_config import get_slurm_config
|
|
21
|
-
from ..executors.slurm_sudo.runner import SudoSlurmRunner
|
|
22
|
-
from ..set_start_and_last_task_index import set_start_and_last_task_index
|
|
23
|
-
from .runner import execute_tasks_v2
|
|
24
21
|
from fractal_server.app.models.v2 import DatasetV2
|
|
25
22
|
from fractal_server.app.models.v2 import Profile
|
|
26
23
|
from fractal_server.app.models.v2 import Resource
|
|
27
24
|
from fractal_server.app.models.v2 import WorkflowV2
|
|
25
|
+
from fractal_server.runner.executors.slurm_common.get_slurm_config import (
|
|
26
|
+
get_slurm_config,
|
|
27
|
+
)
|
|
28
|
+
from fractal_server.runner.executors.slurm_sudo.runner import SlurmSudoRunner
|
|
29
|
+
from fractal_server.runner.set_start_and_last_task_index import (
|
|
30
|
+
set_start_and_last_task_index,
|
|
31
|
+
)
|
|
28
32
|
from fractal_server.ssh._fabric import FractalSSH
|
|
29
33
|
from fractal_server.types import AttributeFilters
|
|
30
34
|
|
|
35
|
+
from .runner import execute_tasks_v2
|
|
36
|
+
|
|
31
37
|
|
|
32
38
|
def process_workflow(
|
|
33
39
|
*,
|
|
@@ -94,7 +100,7 @@ def process_workflow(
|
|
|
94
100
|
if isinstance(worker_init, str):
|
|
95
101
|
worker_init = worker_init.split("\n")
|
|
96
102
|
|
|
97
|
-
with
|
|
103
|
+
with SlurmSudoRunner(
|
|
98
104
|
root_dir_local=workflow_dir_local,
|
|
99
105
|
root_dir_remote=workflow_dir_remote,
|
|
100
106
|
common_script_lines=worker_init,
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
from typing import TypeVar
|
|
2
2
|
|
|
3
|
-
from .task_interface import InitArgsModel
|
|
4
3
|
from fractal_server.images import SingleImage
|
|
5
4
|
from fractal_server.images import SingleImageTaskOutput
|
|
6
5
|
|
|
6
|
+
from .task_interface import InitArgsModel
|
|
7
|
+
|
|
7
8
|
T = TypeVar("T", SingleImage, SingleImageTaskOutput, InitArgsModel)
|
|
8
9
|
|
|
9
10
|
|
|
@@ -7,13 +7,6 @@ from sqlalchemy.orm.attributes import flag_modified
|
|
|
7
7
|
from sqlmodel import delete
|
|
8
8
|
from sqlmodel import update
|
|
9
9
|
|
|
10
|
-
from .merge_outputs import merge_outputs
|
|
11
|
-
from .runner_functions import GetRunnerConfigType
|
|
12
|
-
from .runner_functions import run_v2_task_compound
|
|
13
|
-
from .runner_functions import run_v2_task_non_parallel
|
|
14
|
-
from .runner_functions import run_v2_task_parallel
|
|
15
|
-
from .runner_functions import SubmissionOutcome
|
|
16
|
-
from .task_interface import TaskOutput
|
|
17
10
|
from fractal_server.app.db import get_sync_db
|
|
18
11
|
from fractal_server.app.models.v2 import AccountingRecord
|
|
19
12
|
from fractal_server.app.models.v2 import DatasetV2
|
|
@@ -28,8 +21,8 @@ from fractal_server.app.schemas.v2 import TaskDumpV2
|
|
|
28
21
|
from fractal_server.app.schemas.v2 import TaskGroupDumpV2
|
|
29
22
|
from fractal_server.app.schemas.v2 import TaskType
|
|
30
23
|
from fractal_server.images import SingleImage
|
|
31
|
-
from fractal_server.images.status_tools import enrich_images_unsorted_sync
|
|
32
24
|
from fractal_server.images.status_tools import IMAGE_STATUS_KEY
|
|
25
|
+
from fractal_server.images.status_tools import enrich_images_unsorted_sync
|
|
33
26
|
from fractal_server.images.tools import filter_image_list
|
|
34
27
|
from fractal_server.images.tools import find_image_by_zarr_url
|
|
35
28
|
from fractal_server.images.tools import merge_type_filters
|
|
@@ -39,6 +32,14 @@ from fractal_server.runner.executors.base_runner import BaseRunner
|
|
|
39
32
|
from fractal_server.runner.v2.db_tools import update_status_of_history_run
|
|
40
33
|
from fractal_server.types import AttributeFilters
|
|
41
34
|
|
|
35
|
+
from .merge_outputs import merge_outputs
|
|
36
|
+
from .runner_functions import GetRunnerConfigType
|
|
37
|
+
from .runner_functions import SubmissionOutcome
|
|
38
|
+
from .runner_functions import run_v2_task_compound
|
|
39
|
+
from .runner_functions import run_v2_task_non_parallel
|
|
40
|
+
from .runner_functions import run_v2_task_parallel
|
|
41
|
+
from .task_interface import TaskOutput
|
|
42
|
+
|
|
42
43
|
|
|
43
44
|
def _remove_status_from_attributes(
|
|
44
45
|
images: list[dict[str, Any]],
|
|
@@ -312,9 +313,7 @@ def execute_tasks_v2(
|
|
|
312
313
|
# history status.
|
|
313
314
|
for image_obj in current_task_output.image_list_updates:
|
|
314
315
|
image = image_obj.model_dump()
|
|
315
|
-
if image["zarr_url"] in [
|
|
316
|
-
img["zarr_url"] for img in tmp_images
|
|
317
|
-
]:
|
|
316
|
+
if image["zarr_url"] in [img["zarr_url"] for img in tmp_images]:
|
|
318
317
|
img_search = find_image_by_zarr_url(
|
|
319
318
|
images=tmp_images,
|
|
320
319
|
zarr_url=image["zarr_url"],
|
|
@@ -418,9 +417,7 @@ def execute_tasks_v2(
|
|
|
418
417
|
|
|
419
418
|
# Update type_filters based on task-manifest output_types
|
|
420
419
|
type_filters_from_task_manifest = task.output_types
|
|
421
|
-
current_dataset_type_filters.update(
|
|
422
|
-
type_filters_from_task_manifest
|
|
423
|
-
)
|
|
420
|
+
current_dataset_type_filters.update(type_filters_from_task_manifest)
|
|
424
421
|
except Exception as e:
|
|
425
422
|
logger.error(
|
|
426
423
|
"Unexpected error in post-task-execution block. "
|
|
@@ -6,12 +6,6 @@ from typing import Protocol
|
|
|
6
6
|
from pydantic import BaseModel
|
|
7
7
|
from pydantic import ConfigDict
|
|
8
8
|
|
|
9
|
-
from ..exceptions import JobExecutionError
|
|
10
|
-
from ..exceptions import TaskOutputValidationError
|
|
11
|
-
from .db_tools import update_status_of_history_unit
|
|
12
|
-
from .deduplicate_list import deduplicate_list
|
|
13
|
-
from .task_interface import InitTaskOutput
|
|
14
|
-
from .task_interface import TaskOutput
|
|
15
9
|
from fractal_server.app.db import get_sync_db
|
|
16
10
|
from fractal_server.app.models.v2 import HistoryUnit
|
|
17
11
|
from fractal_server.app.models.v2 import TaskV2
|
|
@@ -22,16 +16,16 @@ from fractal_server.exceptions import UnreachableBranchError
|
|
|
22
16
|
from fractal_server.logger import set_logger
|
|
23
17
|
from fractal_server.runner.config import JobRunnerConfigLocal
|
|
24
18
|
from fractal_server.runner.config import JobRunnerConfigSLURM
|
|
19
|
+
from fractal_server.runner.exceptions import JobExecutionError
|
|
20
|
+
from fractal_server.runner.exceptions import TaskOutputValidationError
|
|
25
21
|
from fractal_server.runner.executors.base_runner import BaseRunner
|
|
26
22
|
from fractal_server.runner.executors.slurm_common.slurm_config import (
|
|
27
23
|
SlurmConfig,
|
|
28
24
|
)
|
|
29
|
-
from fractal_server.runner.task_files import enrich_task_files_multisubmit
|
|
30
25
|
from fractal_server.runner.task_files import SUBMIT_PREFIX
|
|
31
26
|
from fractal_server.runner.task_files import TaskFiles
|
|
32
|
-
from fractal_server.runner.
|
|
33
|
-
|
|
34
|
-
)
|
|
27
|
+
from fractal_server.runner.task_files import enrich_task_files_multisubmit
|
|
28
|
+
from fractal_server.runner.v2.db_tools import bulk_update_status_of_history_unit
|
|
35
29
|
from fractal_server.runner.v2.db_tools import bulk_upsert_image_cache_fast
|
|
36
30
|
from fractal_server.runner.v2.task_interface import (
|
|
37
31
|
_cast_and_validate_InitTaskOutput,
|
|
@@ -40,6 +34,11 @@ from fractal_server.runner.v2.task_interface import (
|
|
|
40
34
|
_cast_and_validate_TaskOutput,
|
|
41
35
|
)
|
|
42
36
|
|
|
37
|
+
from .db_tools import update_status_of_history_unit
|
|
38
|
+
from .deduplicate_list import deduplicate_list
|
|
39
|
+
from .task_interface import InitTaskOutput
|
|
40
|
+
from .task_interface import TaskOutput
|
|
41
|
+
|
|
43
42
|
|
|
44
43
|
class GetRunnerConfigTypeLocal(Protocol):
|
|
45
44
|
def __call__(
|
|
@@ -48,8 +47,7 @@ class GetRunnerConfigTypeLocal(Protocol):
|
|
|
48
47
|
wftask: WorkflowTaskV2,
|
|
49
48
|
which_type: Literal["non_parallel", "parallel"],
|
|
50
49
|
tot_tasks: int,
|
|
51
|
-
) -> JobRunnerConfigLocal:
|
|
52
|
-
...
|
|
50
|
+
) -> JobRunnerConfigLocal: ...
|
|
53
51
|
|
|
54
52
|
|
|
55
53
|
class GetRunnerConfigTypeSLURM(Protocol):
|
|
@@ -59,8 +57,7 @@ class GetRunnerConfigTypeSLURM(Protocol):
|
|
|
59
57
|
wftask: WorkflowTaskV2,
|
|
60
58
|
which_type: Literal["non_parallel", "parallel"],
|
|
61
59
|
tot_tasks: int,
|
|
62
|
-
) -> SlurmConfig:
|
|
63
|
-
...
|
|
60
|
+
) -> SlurmConfig: ...
|
|
64
61
|
|
|
65
62
|
|
|
66
63
|
GetRunnerConfigType = GetRunnerConfigTypeLocal | GetRunnerConfigTypeSLURM
|
|
@@ -5,6 +5,7 @@ This module is the single entry point to the runner backend subsystem V2.
|
|
|
5
5
|
Other subsystems should only import this module and not its submodules or
|
|
6
6
|
the individual backends.
|
|
7
7
|
"""
|
|
8
|
+
|
|
8
9
|
import os
|
|
9
10
|
import traceback
|
|
10
11
|
from pathlib import Path
|
|
@@ -12,9 +13,6 @@ from typing import Protocol
|
|
|
12
13
|
|
|
13
14
|
from sqlalchemy.orm import Session as DBSyncSession
|
|
14
15
|
|
|
15
|
-
from ._local import process_workflow as local_process_workflow
|
|
16
|
-
from ._slurm_ssh import process_workflow as slurm_ssh_process_workflow
|
|
17
|
-
from ._slurm_sudo import process_workflow as slurm_sudo_process_workflow
|
|
18
16
|
from fractal_server import __VERSION__
|
|
19
17
|
from fractal_server.app.db import DB
|
|
20
18
|
from fractal_server.app.models.v2 import DatasetV2
|
|
@@ -34,6 +32,10 @@ from fractal_server.types import AttributeFilters
|
|
|
34
32
|
from fractal_server.utils import get_timestamp
|
|
35
33
|
from fractal_server.zip_tools import _zip_folder_to_file_and_remove
|
|
36
34
|
|
|
35
|
+
from ._local import process_workflow as local_process_workflow
|
|
36
|
+
from ._slurm_ssh import process_workflow as slurm_ssh_process_workflow
|
|
37
|
+
from ._slurm_sudo import process_workflow as slurm_sudo_process_workflow
|
|
38
|
+
|
|
37
39
|
|
|
38
40
|
class ProcessWorkflowType(Protocol):
|
|
39
41
|
def __call__(
|
|
@@ -53,8 +55,7 @@ class ProcessWorkflowType(Protocol):
|
|
|
53
55
|
resource: Resource,
|
|
54
56
|
profile: Profile,
|
|
55
57
|
user_cache_dir: str,
|
|
56
|
-
) -> None:
|
|
57
|
-
...
|
|
58
|
+
) -> None: ...
|
|
58
59
|
|
|
59
60
|
|
|
60
61
|
def fail_job(
|
|
@@ -119,7 +120,7 @@ def submit_workflow(
|
|
|
119
120
|
Computational resource to be used for this job (e.g. a SLURM
|
|
120
121
|
cluster).
|
|
121
122
|
profile:
|
|
122
|
-
|
|
123
|
+
Computational profile to be used for this job.
|
|
123
124
|
fractal_ssh: SSH object, for when `resource.type = "slurm_ssh"`.
|
|
124
125
|
"""
|
|
125
126
|
# Declare runner backend and set `process_workflow` function
|
fractal_server/ssh/_fabric.py
CHANGED
|
@@ -16,12 +16,11 @@ from invoke import UnexpectedExit
|
|
|
16
16
|
from paramiko.ssh_exception import NoValidConnectionsError
|
|
17
17
|
from pydantic import BaseModel
|
|
18
18
|
|
|
19
|
-
from
|
|
20
|
-
from
|
|
21
|
-
from
|
|
19
|
+
from fractal_server.logger import close_logger
|
|
20
|
+
from fractal_server.logger import get_logger
|
|
21
|
+
from fractal_server.logger import set_logger
|
|
22
22
|
from fractal_server.string_tools import validate_cmd
|
|
23
23
|
|
|
24
|
-
|
|
25
24
|
SSH_MONITORING_LOGGER_NAME = "ssh-log"
|
|
26
25
|
|
|
27
26
|
|
|
@@ -99,8 +98,7 @@ def _acquire_lock_with_timeout(
|
|
|
99
98
|
if not result:
|
|
100
99
|
logger.error(f"Lock for '{label}' was *not* acquired.")
|
|
101
100
|
raise FractalSSHTimeoutError(
|
|
102
|
-
f"Failed to acquire lock for '{label}' within "
|
|
103
|
-
f"{timeout} seconds"
|
|
101
|
+
f"Failed to acquire lock for '{label}' within {timeout} seconds"
|
|
104
102
|
)
|
|
105
103
|
t_lock_acquisition = time.perf_counter()
|
|
106
104
|
elapsed = t_lock_acquisition - t_lock_request
|
|
@@ -374,8 +372,6 @@ class FractalSSH:
|
|
|
374
372
|
Args:
|
|
375
373
|
cmd: Command to be run
|
|
376
374
|
allow_char: Forbidden chars to allow for this command
|
|
377
|
-
max_attempts:
|
|
378
|
-
base_interval:
|
|
379
375
|
lock_timeout:
|
|
380
376
|
|
|
381
377
|
Returns:
|
|
@@ -458,9 +454,7 @@ class FractalSSH:
|
|
|
458
454
|
logger_name=self.logger_name,
|
|
459
455
|
):
|
|
460
456
|
self._sftp_unsafe().put(local, remote)
|
|
461
|
-
self.logger.info(
|
|
462
|
-
f"[send_file] END transfer of '{local}' over SSH."
|
|
463
|
-
)
|
|
457
|
+
self.logger.info(f"[send_file] END transfer of '{local}' over SSH.")
|
|
464
458
|
except Exception as e:
|
|
465
459
|
self.log_and_raise(
|
|
466
460
|
e=e,
|
|
@@ -766,8 +760,7 @@ class FractalSSHList:
|
|
|
766
760
|
logger_name=self._logger_name,
|
|
767
761
|
):
|
|
768
762
|
self.logger.info(
|
|
769
|
-
f"Removing FractalSSH object for {user}@{host} "
|
|
770
|
-
"from collection."
|
|
763
|
+
f"Removing FractalSSH object for {user}@{host} from collection."
|
|
771
764
|
)
|
|
772
765
|
fractal_ssh_obj = self._data.pop(key)
|
|
773
766
|
self.logger.info(
|
fractal_server/string_tools.py
CHANGED
fractal_server/syringe.py
CHANGED
|
@@ -3,8 +3,8 @@ from typing import Annotated
|
|
|
3
3
|
|
|
4
4
|
from pydantic import AfterValidator
|
|
5
5
|
from pydantic import BaseModel
|
|
6
|
-
from pydantic import model_validator
|
|
7
6
|
from pydantic import PositiveInt
|
|
7
|
+
from pydantic import model_validator
|
|
8
8
|
|
|
9
9
|
from fractal_server.types import DictStrStr
|
|
10
10
|
from fractal_server.types import NonEmptyStr
|
|
@@ -14,12 +14,25 @@ class TasksPythonSettings(BaseModel):
|
|
|
14
14
|
|
|
15
15
|
For task collection to work, there must be one or more base Python
|
|
16
16
|
interpreters available on your system.
|
|
17
|
+
|
|
18
|
+
Attributes:
|
|
19
|
+
default_version:
|
|
20
|
+
Default task-collection Python version (must be a key of
|
|
21
|
+
`versions`).
|
|
22
|
+
versions:
|
|
23
|
+
Dictionary mapping Python versions to the corresponding
|
|
24
|
+
interpreters. Example:
|
|
25
|
+
```json
|
|
26
|
+
{
|
|
27
|
+
"3.11": "/path/to/python3.11",
|
|
28
|
+
"3.13": "/path/to/python3.13"
|
|
29
|
+
}
|
|
30
|
+
```
|
|
31
|
+
pip_cache_dir:
|
|
32
|
+
Argument for `--cache-dir` option of `pip install`, if set.
|
|
17
33
|
"""
|
|
18
34
|
|
|
19
35
|
default_version: NonEmptyStr
|
|
20
|
-
"""
|
|
21
|
-
Default task-collection Python version (must be a key of `versions`).
|
|
22
|
-
"""
|
|
23
36
|
versions: dict[
|
|
24
37
|
Literal[
|
|
25
38
|
"3.9",
|
|
@@ -31,14 +44,8 @@ class TasksPythonSettings(BaseModel):
|
|
|
31
44
|
],
|
|
32
45
|
AbsolutePathStr,
|
|
33
46
|
]
|
|
34
|
-
"""
|
|
35
|
-
Dictionary mapping Python versions to the corresponding interpreters.
|
|
36
|
-
"""
|
|
37
47
|
|
|
38
48
|
pip_cache_dir: AbsolutePathStr | None = None
|
|
39
|
-
"""
|
|
40
|
-
Argument for `--cache-dir` option of `pip install`, if set.
|
|
41
|
-
"""
|
|
42
49
|
|
|
43
50
|
@model_validator(mode="after")
|
|
44
51
|
def _validate_versions(self) -> Self:
|