fractal-server 2.17.1a1__py3-none-any.whl → 2.18.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fractal_server/__init__.py +1 -1
- fractal_server/__main__.py +21 -19
- fractal_server/app/db/__init__.py +3 -3
- fractal_server/app/models/__init__.py +1 -0
- fractal_server/app/models/linkuserproject.py +43 -1
- fractal_server/app/models/security.py +28 -8
- fractal_server/app/models/v2/__init__.py +3 -1
- fractal_server/app/models/v2/accounting.py +9 -1
- fractal_server/app/models/v2/dataset.py +5 -1
- fractal_server/app/models/v2/history.py +15 -1
- fractal_server/app/models/v2/job.py +17 -2
- fractal_server/app/models/v2/profile.py +29 -0
- fractal_server/app/models/v2/project.py +4 -10
- fractal_server/app/models/v2/resource.py +17 -0
- fractal_server/app/models/v2/task_group.py +4 -3
- fractal_server/app/models/v2/workflow.py +2 -1
- fractal_server/app/routes/admin/v2/__init__.py +12 -13
- fractal_server/app/routes/admin/v2/accounting.py +3 -3
- fractal_server/app/routes/admin/v2/job.py +35 -24
- fractal_server/app/routes/admin/v2/profile.py +3 -2
- fractal_server/app/routes/admin/v2/resource.py +5 -5
- fractal_server/app/routes/admin/v2/sharing.py +103 -0
- fractal_server/app/routes/admin/v2/task.py +37 -26
- fractal_server/app/routes/admin/v2/task_group.py +94 -17
- fractal_server/app/routes/admin/v2/task_group_lifecycle.py +21 -22
- fractal_server/app/routes/api/__init__.py +1 -9
- fractal_server/app/routes/api/v2/__init__.py +49 -50
- fractal_server/app/routes/api/v2/_aux_functions.py +132 -124
- fractal_server/app/routes/api/v2/_aux_functions_history.py +51 -23
- fractal_server/app/routes/api/v2/_aux_functions_sharing.py +97 -0
- fractal_server/app/routes/api/v2/_aux_functions_task_lifecycle.py +6 -8
- fractal_server/app/routes/api/v2/_aux_functions_tasks.py +7 -9
- fractal_server/app/routes/api/v2/_aux_task_group_disambiguation.py +1 -2
- fractal_server/app/routes/api/v2/dataset.py +95 -102
- fractal_server/app/routes/api/v2/history.py +59 -33
- fractal_server/app/routes/api/v2/images.py +24 -9
- fractal_server/app/routes/api/v2/job.py +52 -33
- fractal_server/app/routes/api/v2/pre_submission_checks.py +16 -8
- fractal_server/app/routes/api/v2/project.py +65 -37
- fractal_server/app/routes/api/v2/sharing.py +311 -0
- fractal_server/app/routes/api/v2/status_legacy.py +31 -41
- fractal_server/app/routes/api/v2/submit.py +82 -78
- fractal_server/app/routes/api/v2/task.py +19 -20
- fractal_server/app/routes/api/v2/task_collection.py +41 -43
- fractal_server/app/routes/api/v2/task_collection_custom.py +19 -20
- fractal_server/app/routes/api/v2/task_collection_pixi.py +10 -11
- fractal_server/app/routes/api/v2/task_group.py +25 -24
- fractal_server/app/routes/api/v2/task_group_lifecycle.py +32 -32
- fractal_server/app/routes/api/v2/task_version_update.py +23 -19
- fractal_server/app/routes/api/v2/workflow.py +50 -55
- fractal_server/app/routes/api/v2/workflow_import.py +37 -37
- fractal_server/app/routes/api/v2/workflowtask.py +32 -26
- fractal_server/app/routes/auth/__init__.py +1 -3
- fractal_server/app/routes/auth/_aux_auth.py +101 -2
- fractal_server/app/routes/auth/current_user.py +2 -66
- fractal_server/app/routes/auth/group.py +8 -35
- fractal_server/app/routes/auth/login.py +1 -0
- fractal_server/app/routes/auth/oauth.py +4 -3
- fractal_server/app/routes/auth/register.py +4 -2
- fractal_server/app/routes/auth/router.py +2 -0
- fractal_server/app/routes/auth/users.py +19 -10
- fractal_server/app/routes/auth/viewer_paths.py +43 -0
- fractal_server/app/routes/aux/_job.py +1 -1
- fractal_server/app/routes/aux/_runner.py +2 -2
- fractal_server/app/routes/pagination.py +1 -1
- fractal_server/app/schemas/user.py +29 -12
- fractal_server/app/schemas/user_group.py +0 -15
- fractal_server/app/schemas/v2/__init__.py +55 -48
- fractal_server/app/schemas/v2/accounting.py +11 -0
- fractal_server/app/schemas/v2/dataset.py +57 -11
- fractal_server/app/schemas/v2/dumps.py +10 -9
- fractal_server/app/schemas/v2/job.py +11 -11
- fractal_server/app/schemas/v2/manifest.py +4 -3
- fractal_server/app/schemas/v2/profile.py +53 -2
- fractal_server/app/schemas/v2/project.py +3 -3
- fractal_server/app/schemas/v2/resource.py +121 -16
- fractal_server/app/schemas/v2/sharing.py +99 -0
- fractal_server/app/schemas/v2/status_legacy.py +3 -3
- fractal_server/app/schemas/v2/task.py +6 -7
- fractal_server/app/schemas/v2/task_collection.py +5 -5
- fractal_server/app/schemas/v2/task_group.py +16 -16
- fractal_server/app/schemas/v2/workflow.py +16 -16
- fractal_server/app/schemas/v2/workflowtask.py +16 -15
- fractal_server/app/security/__init__.py +5 -8
- fractal_server/app/security/signup_email.py +4 -5
- fractal_server/app/shutdown.py +6 -6
- fractal_server/config/__init__.py +0 -6
- fractal_server/config/_data.py +0 -68
- fractal_server/config/_database.py +19 -20
- fractal_server/config/_email.py +30 -38
- fractal_server/config/_main.py +38 -52
- fractal_server/config/_oauth.py +17 -21
- fractal_server/data_migrations/2_18_0.py +30 -0
- fractal_server/exceptions.py +4 -0
- fractal_server/images/models.py +4 -5
- fractal_server/images/status_tools.py +4 -2
- fractal_server/logger.py +1 -1
- fractal_server/main.py +75 -13
- fractal_server/migrations/versions/034a469ec2eb_task_groups.py +4 -8
- fractal_server/migrations/versions/091b01f51f88_add_usergroup_and_linkusergroup_table.py +1 -1
- fractal_server/migrations/versions/0f5f85bb2ae7_add_pre_pinned_packages.py +1 -0
- fractal_server/migrations/versions/19eca0dd47a9_user_settings_project_dir.py +1 -1
- fractal_server/migrations/versions/1a83a5260664_rename.py +1 -1
- fractal_server/migrations/versions/1eac13a26c83_drop_v1_tables.py +1 -0
- fractal_server/migrations/versions/316140ff7ee1_remove_usersettings_cache_dir.py +1 -1
- fractal_server/migrations/versions/40d6d6511b20_add_index_to_history_models.py +47 -0
- fractal_server/migrations/versions/45fbb391d7af_make_resource_id_fk_non_nullable.py +1 -1
- fractal_server/migrations/versions/47351f8c7ebc_drop_dataset_filters.py +1 -0
- fractal_server/migrations/versions/49d0856e9569_drop_table.py +2 -3
- fractal_server/migrations/versions/4c308bcaea2b_add_task_args_schema_and_task_args_.py +1 -1
- fractal_server/migrations/versions/4cedeb448a53_workflowtask_foreign_keys_not_nullables.py +1 -1
- fractal_server/migrations/versions/501961cfcd85_remove_link_between_v1_and_v2_tasks_.py +2 -1
- fractal_server/migrations/versions/50a13d6138fd_initial_schema.py +7 -19
- fractal_server/migrations/versions/5bf02391cfef_v2.py +4 -10
- fractal_server/migrations/versions/70e77f1c38b0_add_applyworkflow_first_task_index_and_.py +1 -0
- fractal_server/migrations/versions/71eefd1dd202_add_slurm_accounts.py +1 -1
- fractal_server/migrations/versions/7673fe18c05d_remove_project_dir_server_default.py +1 -1
- fractal_server/migrations/versions/7910eed4cf97_user_project_dirs_and_usergroup_viewer_.py +60 -0
- fractal_server/migrations/versions/791ce783d3d8_add_indices.py +1 -1
- fractal_server/migrations/versions/83bc2ad3ffcc_2_17_0.py +1 -0
- fractal_server/migrations/versions/84bf0fffde30_add_dumps_to_applyworkflow.py +1 -0
- fractal_server/migrations/versions/88270f589c9b_add_prevent_new_submissions.py +39 -0
- fractal_server/migrations/versions/8e8f227a3e36_update_taskv2_post_2_7_0.py +2 -4
- fractal_server/migrations/versions/8f79bd162e35_add_docs_info_and_docs_link_to_task_.py +1 -1
- fractal_server/migrations/versions/94a47ea2d3ff_remove_cache_dir_slurm_user_and_slurm_.py +1 -0
- fractal_server/migrations/versions/969d84257cac_add_historyrun_task_id.py +1 -1
- fractal_server/migrations/versions/97f444d47249_add_applyworkflow_project_dump.py +1 -1
- fractal_server/migrations/versions/981d588fe248_add_executor_error_log.py +1 -1
- fractal_server/migrations/versions/99ea79d9e5d2_add_dataset_history.py +2 -4
- fractal_server/migrations/versions/9c5ae74c9b98_add_user_settings_table.py +1 -1
- fractal_server/migrations/versions/9db60297b8b2_set_ondelete.py +1 -1
- fractal_server/migrations/versions/9fd26a2b0de4_add_workflow_timestamp_created.py +1 -1
- fractal_server/migrations/versions/a7f4d6137b53_add_workflow_dump_to_applyworkflow.py +1 -1
- fractal_server/migrations/versions/af1ef1c83c9b_add_accounting_tables.py +1 -0
- fractal_server/migrations/versions/af8673379a5c_drop_old_filter_columns.py +1 -0
- fractal_server/migrations/versions/b1e7f7a1ff71_task_group_for_pixi.py +1 -1
- fractal_server/migrations/versions/b3ffb095f973_json_to_jsonb.py +1 -0
- fractal_server/migrations/versions/bc0e8b3327a7_project_sharing.py +72 -0
- fractal_server/migrations/versions/c90a7c76e996_job_id_in_history_run.py +1 -1
- fractal_server/migrations/versions/caba9fb1ea5e_drop_useroauth_user_settings_id.py +1 -1
- fractal_server/migrations/versions/d256a7379ab8_taskgroup_activity_and_venv_info_to_.py +4 -9
- fractal_server/migrations/versions/d4fe3708d309_make_applyworkflow_workflow_dump_non_.py +1 -0
- fractal_server/migrations/versions/da2cb2ac4255_user_group_viewer_paths.py +1 -1
- fractal_server/migrations/versions/db09233ad13a_split_filters_and_keep_old_columns.py +1 -0
- fractal_server/migrations/versions/e0e717ae2f26_delete_linkuserproject_ondelete_project.py +50 -0
- fractal_server/migrations/versions/e75cac726012_make_applyworkflow_start_timestamp_not_.py +1 -0
- fractal_server/migrations/versions/e81103413827_add_job_type_filters.py +1 -1
- fractal_server/migrations/versions/efa89c30e0a4_add_project_timestamp_created.py +1 -0
- fractal_server/migrations/versions/f0702066b007_one_submitted_job_per_dataset.py +40 -0
- fractal_server/migrations/versions/f37aceb45062_make_historyunit_logfile_required.py +1 -1
- fractal_server/migrations/versions/f384e1c0cf5d_drop_task_default_args_columns.py +1 -0
- fractal_server/migrations/versions/fbce16ff4e47_new_history_items.py +4 -9
- fractal_server/runner/config/_local.py +8 -5
- fractal_server/runner/config/_slurm.py +39 -33
- fractal_server/runner/config/slurm_mem_to_MB.py +0 -1
- fractal_server/runner/executors/base_runner.py +29 -4
- fractal_server/runner/executors/local/get_local_config.py +1 -0
- fractal_server/runner/executors/local/runner.py +14 -13
- fractal_server/runner/executors/slurm_common/_batching.py +9 -20
- fractal_server/runner/executors/slurm_common/base_slurm_runner.py +53 -27
- fractal_server/runner/executors/slurm_common/get_slurm_config.py +14 -7
- fractal_server/runner/executors/slurm_common/remote.py +3 -1
- fractal_server/runner/executors/slurm_common/slurm_config.py +2 -0
- fractal_server/runner/executors/slurm_common/slurm_job_task_models.py +1 -3
- fractal_server/runner/executors/slurm_ssh/runner.py +16 -11
- fractal_server/runner/executors/slurm_ssh/tar_commands.py +1 -0
- fractal_server/runner/executors/slurm_sudo/_subprocess_run_as_user.py +1 -0
- fractal_server/runner/executors/slurm_sudo/runner.py +16 -11
- fractal_server/runner/task_files.py +9 -3
- fractal_server/runner/v2/_local.py +12 -6
- fractal_server/runner/v2/_slurm_ssh.py +14 -7
- fractal_server/runner/v2/_slurm_sudo.py +14 -7
- fractal_server/runner/v2/db_tools.py +0 -1
- fractal_server/runner/v2/deduplicate_list.py +2 -1
- fractal_server/runner/v2/runner.py +44 -28
- fractal_server/runner/v2/runner_functions.py +22 -28
- fractal_server/runner/v2/submit_workflow.py +29 -15
- fractal_server/ssh/_fabric.py +6 -13
- fractal_server/string_tools.py +0 -1
- fractal_server/syringe.py +1 -1
- fractal_server/tasks/config/_pixi.py +1 -1
- fractal_server/tasks/config/_python.py +16 -9
- fractal_server/tasks/utils.py +0 -1
- fractal_server/tasks/v2/local/_utils.py +3 -3
- fractal_server/tasks/v2/local/collect.py +15 -18
- fractal_server/tasks/v2/local/collect_pixi.py +14 -16
- fractal_server/tasks/v2/local/deactivate.py +14 -15
- fractal_server/tasks/v2/local/deactivate_pixi.py +7 -7
- fractal_server/tasks/v2/local/delete.py +6 -8
- fractal_server/tasks/v2/local/reactivate.py +12 -12
- fractal_server/tasks/v2/local/reactivate_pixi.py +12 -12
- fractal_server/tasks/v2/ssh/_utils.py +3 -3
- fractal_server/tasks/v2/ssh/collect.py +19 -24
- fractal_server/tasks/v2/ssh/collect_pixi.py +22 -24
- fractal_server/tasks/v2/ssh/deactivate.py +17 -15
- fractal_server/tasks/v2/ssh/deactivate_pixi.py +8 -7
- fractal_server/tasks/v2/ssh/delete.py +12 -10
- fractal_server/tasks/v2/ssh/reactivate.py +16 -16
- fractal_server/tasks/v2/ssh/reactivate_pixi.py +13 -14
- fractal_server/tasks/v2/templates/1_create_venv.sh +2 -0
- fractal_server/tasks/v2/templates/2_pip_install.sh +2 -0
- fractal_server/tasks/v2/templates/3_pip_freeze.sh +2 -0
- fractal_server/tasks/v2/templates/4_pip_show.sh +2 -0
- fractal_server/tasks/v2/templates/5_get_venv_size_and_file_number.sh +3 -1
- fractal_server/tasks/v2/templates/6_pip_install_from_freeze.sh +2 -0
- fractal_server/tasks/v2/templates/pixi_1_extract.sh +2 -0
- fractal_server/tasks/v2/templates/pixi_2_install.sh +2 -0
- fractal_server/tasks/v2/templates/pixi_3_post_install.sh +2 -0
- fractal_server/tasks/v2/utils_background.py +10 -10
- fractal_server/tasks/v2/utils_database.py +5 -5
- fractal_server/tasks/v2/utils_package_names.py +1 -2
- fractal_server/tasks/v2/utils_pixi.py +1 -3
- fractal_server/types/__init__.py +98 -1
- fractal_server/types/validators/__init__.py +3 -0
- fractal_server/types/validators/_common_validators.py +33 -3
- fractal_server/types/validators/_workflow_task_arguments_validators.py +1 -2
- fractal_server/utils.py +1 -0
- fractal_server/zip_tools.py +34 -0
- {fractal_server-2.17.1a1.dist-info → fractal_server-2.18.0.dist-info}/METADATA +3 -2
- fractal_server-2.18.0.dist-info/RECORD +275 -0
- fractal_server/app/routes/admin/v2/project.py +0 -41
- fractal_server-2.17.1a1.dist-info/RECORD +0 -264
- {fractal_server-2.17.1a1.dist-info → fractal_server-2.18.0.dist-info}/WHEEL +0 -0
- {fractal_server-2.17.1a1.dist-info → fractal_server-2.18.0.dist-info}/entry_points.txt +0 -0
- {fractal_server-2.17.1a1.dist-info → fractal_server-2.18.0.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,23 +1,32 @@
|
|
|
1
1
|
import time
|
|
2
2
|
from pathlib import Path
|
|
3
3
|
|
|
4
|
-
from ..slurm_common.base_slurm_runner import BaseSlurmRunner
|
|
5
|
-
from ..slurm_common.slurm_job_task_models import SlurmJob
|
|
6
|
-
from .run_subprocess import run_subprocess
|
|
7
|
-
from .tar_commands import get_tar_compression_cmd
|
|
8
|
-
from .tar_commands import get_tar_extraction_cmd
|
|
9
4
|
from fractal_server.app.models import Profile
|
|
10
5
|
from fractal_server.app.models import Resource
|
|
11
6
|
from fractal_server.logger import set_logger
|
|
12
7
|
from fractal_server.runner.config import JobRunnerConfigSLURM
|
|
8
|
+
from fractal_server.runner.executors.slurm_common.base_slurm_runner import (
|
|
9
|
+
BaseSlurmRunner,
|
|
10
|
+
)
|
|
11
|
+
from fractal_server.runner.executors.slurm_common.slurm_job_task_models import (
|
|
12
|
+
SlurmJob,
|
|
13
|
+
)
|
|
13
14
|
from fractal_server.ssh._fabric import FractalSSH
|
|
14
15
|
from fractal_server.ssh._fabric import FractalSSHCommandError
|
|
15
16
|
from fractal_server.ssh._fabric import FractalSSHTimeoutError
|
|
16
17
|
|
|
18
|
+
from .run_subprocess import run_subprocess
|
|
19
|
+
from .tar_commands import get_tar_compression_cmd
|
|
20
|
+
from .tar_commands import get_tar_extraction_cmd
|
|
21
|
+
|
|
17
22
|
logger = set_logger(__name__)
|
|
18
23
|
|
|
19
24
|
|
|
20
25
|
class SlurmSSHRunner(BaseSlurmRunner):
|
|
26
|
+
"""
|
|
27
|
+
Runner implementation for a computational `slurm_ssh` resource.
|
|
28
|
+
"""
|
|
29
|
+
|
|
21
30
|
fractal_ssh: FractalSSH
|
|
22
31
|
|
|
23
32
|
def __init__(
|
|
@@ -39,9 +48,7 @@ class SlurmSSHRunner(BaseSlurmRunner):
|
|
|
39
48
|
different SLURM jobs/tasks.
|
|
40
49
|
"""
|
|
41
50
|
self.fractal_ssh = fractal_ssh
|
|
42
|
-
self.shared_config = JobRunnerConfigSLURM(
|
|
43
|
-
**resource.jobs_runner_config
|
|
44
|
-
)
|
|
51
|
+
self.shared_config = JobRunnerConfigSLURM(**resource.jobs_runner_config)
|
|
45
52
|
logger.warning(self.fractal_ssh)
|
|
46
53
|
|
|
47
54
|
# Check SSH connection and try to recover from a closed-socket error
|
|
@@ -80,9 +87,7 @@ class SlurmSSHRunner(BaseSlurmRunner):
|
|
|
80
87
|
return None
|
|
81
88
|
|
|
82
89
|
t_0 = time.perf_counter()
|
|
83
|
-
logger.debug(
|
|
84
|
-
f"[_fetch_artifacts] START ({len(finished_slurm_jobs)=})."
|
|
85
|
-
)
|
|
90
|
+
logger.debug(f"[_fetch_artifacts] START ({len(finished_slurm_jobs)=}).")
|
|
86
91
|
|
|
87
92
|
# Extract `workdir_remote` and `workdir_local`
|
|
88
93
|
self.validate_slurm_jobs_workdirs(finished_slurm_jobs)
|
|
@@ -15,6 +15,7 @@ This module provides a set of tools similar to `subprocess.run`, `glob.glob` or
|
|
|
15
15
|
`os.path.exists`, but extended so that they can be executed on behalf of
|
|
16
16
|
another user. Note that this requires appropriate sudo permissions.
|
|
17
17
|
"""
|
|
18
|
+
|
|
18
19
|
import shlex
|
|
19
20
|
import subprocess # nosec
|
|
20
21
|
|
|
@@ -5,15 +5,20 @@ import subprocess # nosec
|
|
|
5
5
|
from concurrent.futures import ThreadPoolExecutor
|
|
6
6
|
from pathlib import Path
|
|
7
7
|
|
|
8
|
-
from ..slurm_common.base_slurm_runner import BaseSlurmRunner
|
|
9
|
-
from ..slurm_common.slurm_job_task_models import SlurmJob
|
|
10
|
-
from ._subprocess_run_as_user import _mkdir_as_user
|
|
11
|
-
from ._subprocess_run_as_user import _run_command_as_user
|
|
12
8
|
from fractal_server.app.models import Profile
|
|
13
9
|
from fractal_server.app.models import Resource
|
|
14
10
|
from fractal_server.logger import set_logger
|
|
15
11
|
from fractal_server.runner.config import JobRunnerConfigSLURM
|
|
16
12
|
from fractal_server.runner.exceptions import JobExecutionError
|
|
13
|
+
from fractal_server.runner.executors.slurm_common.base_slurm_runner import (
|
|
14
|
+
BaseSlurmRunner,
|
|
15
|
+
)
|
|
16
|
+
from fractal_server.runner.executors.slurm_common.slurm_job_task_models import (
|
|
17
|
+
SlurmJob,
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
from ._subprocess_run_as_user import _mkdir_as_user
|
|
21
|
+
from ._subprocess_run_as_user import _run_command_as_user
|
|
17
22
|
|
|
18
23
|
logger = set_logger(__name__)
|
|
19
24
|
|
|
@@ -40,7 +45,11 @@ def _subprocess_run_or_raise(
|
|
|
40
45
|
raise JobExecutionError(info=error_msg)
|
|
41
46
|
|
|
42
47
|
|
|
43
|
-
class
|
|
48
|
+
class SlurmSudoRunner(BaseSlurmRunner):
|
|
49
|
+
"""
|
|
50
|
+
Runner implementation for a computational `slurm_sudo` resource.
|
|
51
|
+
"""
|
|
52
|
+
|
|
44
53
|
slurm_user: str
|
|
45
54
|
slurm_account: str | None = None
|
|
46
55
|
|
|
@@ -63,9 +72,7 @@ class SudoSlurmRunner(BaseSlurmRunner):
|
|
|
63
72
|
"""
|
|
64
73
|
|
|
65
74
|
self.slurm_user = profile.username
|
|
66
|
-
self.shared_config = JobRunnerConfigSLURM(
|
|
67
|
-
**resource.jobs_runner_config
|
|
68
|
-
)
|
|
75
|
+
self.shared_config = JobRunnerConfigSLURM(**resource.jobs_runner_config)
|
|
69
76
|
|
|
70
77
|
super().__init__(
|
|
71
78
|
slurm_runner_type="sudo",
|
|
@@ -90,9 +97,7 @@ class SudoSlurmRunner(BaseSlurmRunner):
|
|
|
90
97
|
"""
|
|
91
98
|
Fetch artifacts for a single SLURM jobs.
|
|
92
99
|
"""
|
|
93
|
-
logger.debug(
|
|
94
|
-
f"[_fetch_artifacts_single_job] {job.slurm_job_id=} START"
|
|
95
|
-
)
|
|
100
|
+
logger.debug(f"[_fetch_artifacts_single_job] {job.slurm_job_id=} START")
|
|
96
101
|
source_target_list = [
|
|
97
102
|
(job.slurm_stdout_remote, job.slurm_stdout_local),
|
|
98
103
|
(job.slurm_stderr_remote, job.slurm_stderr_local),
|
|
@@ -86,9 +86,7 @@ class TaskFiles(BaseModel):
|
|
|
86
86
|
@property
|
|
87
87
|
def log_file_remote_path(self) -> Path:
|
|
88
88
|
self._check_component()
|
|
89
|
-
return
|
|
90
|
-
self.wftask_subfolder_remote / f"{self.prefix_component}-log.txt"
|
|
91
|
-
)
|
|
89
|
+
return self.wftask_subfolder_remote / f"{self.prefix_component}-log.txt"
|
|
92
90
|
|
|
93
91
|
@property
|
|
94
92
|
def log_file_remote(self) -> str:
|
|
@@ -141,8 +139,16 @@ def enrich_task_files_multisubmit(
|
|
|
141
139
|
) -> list[TaskFiles]:
|
|
142
140
|
"""
|
|
143
141
|
Expand `TaskFiles` objects with `component` and `prefix`.
|
|
142
|
+
|
|
143
|
+
Args:
|
|
144
|
+
tot_tasks: Total number of images to process.
|
|
145
|
+
batch_size: Batch size, where `0` means `batch_size=tot_tasks`.
|
|
146
|
+
base_task_files: Original `TaskFiles` object to be enriched.
|
|
144
147
|
"""
|
|
145
148
|
|
|
149
|
+
# Replace `batch_size=0` with `batch_size=tot_tasks`
|
|
150
|
+
batch_size = batch_size or tot_tasks
|
|
151
|
+
|
|
146
152
|
new_list_task_files: list[TaskFiles] = []
|
|
147
153
|
for absolute_index in range(tot_tasks):
|
|
148
154
|
ind_batch = absolute_index // batch_size
|
|
@@ -1,16 +1,21 @@
|
|
|
1
1
|
from pathlib import Path
|
|
2
2
|
|
|
3
|
-
from ..executors.local.get_local_config import get_local_backend_config
|
|
4
|
-
from ..executors.local.runner import LocalRunner
|
|
5
|
-
from ..set_start_and_last_task_index import set_start_and_last_task_index
|
|
6
|
-
from .runner import execute_tasks_v2
|
|
7
3
|
from fractal_server.app.models.v2 import DatasetV2
|
|
8
4
|
from fractal_server.app.models.v2 import Profile
|
|
9
5
|
from fractal_server.app.models.v2 import Resource
|
|
10
6
|
from fractal_server.app.models.v2 import WorkflowV2
|
|
7
|
+
from fractal_server.runner.executors.local.get_local_config import (
|
|
8
|
+
get_local_backend_config,
|
|
9
|
+
)
|
|
10
|
+
from fractal_server.runner.executors.local.runner import LocalRunner
|
|
11
|
+
from fractal_server.runner.set_start_and_last_task_index import (
|
|
12
|
+
set_start_and_last_task_index,
|
|
13
|
+
)
|
|
11
14
|
from fractal_server.ssh._fabric import FractalSSH
|
|
12
15
|
from fractal_server.types import AttributeFilters
|
|
13
16
|
|
|
17
|
+
from .runner import execute_tasks
|
|
18
|
+
|
|
14
19
|
|
|
15
20
|
def process_workflow(
|
|
16
21
|
*,
|
|
@@ -54,7 +59,7 @@ def process_workflow(
|
|
|
54
59
|
resource: Computational resource for running this job.
|
|
55
60
|
profile: Computational profile for running this job.
|
|
56
61
|
user_cache_dir:
|
|
57
|
-
User-writeable folder (typically a subfolder of `
|
|
62
|
+
User-writeable folder (typically a subfolder of `project_dirs`).
|
|
58
63
|
Only relevant for `slurm_sudo` and `slurm_ssh` backends.
|
|
59
64
|
fractal_ssh:
|
|
60
65
|
`FractalSSH` object, only relevant for the `slurm_ssh` backend.
|
|
@@ -85,7 +90,7 @@ def process_workflow(
|
|
|
85
90
|
resource=resource,
|
|
86
91
|
profile=profile,
|
|
87
92
|
) as runner:
|
|
88
|
-
|
|
93
|
+
execute_tasks(
|
|
89
94
|
wf_task_list=workflow.task_list[
|
|
90
95
|
first_task_index : (last_task_index + 1)
|
|
91
96
|
],
|
|
@@ -99,4 +104,5 @@ def process_workflow(
|
|
|
99
104
|
job_attribute_filters=job_attribute_filters,
|
|
100
105
|
job_type_filters=job_type_filters,
|
|
101
106
|
user_id=user_id,
|
|
107
|
+
resource_id=resource.id,
|
|
102
108
|
)
|
|
@@ -15,20 +15,26 @@ Slurm Backend
|
|
|
15
15
|
|
|
16
16
|
This backend runs fractal workflows in a SLURM cluster.
|
|
17
17
|
"""
|
|
18
|
+
|
|
18
19
|
from pathlib import Path
|
|
19
20
|
|
|
20
|
-
from ...ssh._fabric import FractalSSH
|
|
21
|
-
from ..executors.slurm_common.get_slurm_config import get_slurm_config
|
|
22
|
-
from ..executors.slurm_ssh.runner import SlurmSSHRunner
|
|
23
|
-
from ..set_start_and_last_task_index import set_start_and_last_task_index
|
|
24
|
-
from .runner import execute_tasks_v2
|
|
25
21
|
from fractal_server.app.models.v2 import DatasetV2
|
|
26
22
|
from fractal_server.app.models.v2 import Profile
|
|
27
23
|
from fractal_server.app.models.v2 import Resource
|
|
28
24
|
from fractal_server.app.models.v2 import WorkflowV2
|
|
29
25
|
from fractal_server.logger import set_logger
|
|
26
|
+
from fractal_server.runner.executors.slurm_common.get_slurm_config import (
|
|
27
|
+
get_slurm_config,
|
|
28
|
+
)
|
|
29
|
+
from fractal_server.runner.executors.slurm_ssh.runner import SlurmSSHRunner
|
|
30
|
+
from fractal_server.runner.set_start_and_last_task_index import (
|
|
31
|
+
set_start_and_last_task_index,
|
|
32
|
+
)
|
|
33
|
+
from fractal_server.ssh._fabric import FractalSSH
|
|
30
34
|
from fractal_server.types import AttributeFilters
|
|
31
35
|
|
|
36
|
+
from .runner import execute_tasks
|
|
37
|
+
|
|
32
38
|
logger = set_logger(__name__)
|
|
33
39
|
|
|
34
40
|
|
|
@@ -74,7 +80,7 @@ def process_workflow(
|
|
|
74
80
|
resource: Computational resource for running this job.
|
|
75
81
|
profile: Computational profile for running this job.
|
|
76
82
|
user_cache_dir:
|
|
77
|
-
User-writeable folder (typically a subfolder of `
|
|
83
|
+
User-writeable folder (typically a subfolder of `project_dirs`).
|
|
78
84
|
Only relevant for `slurm_sudo` and `slurm_ssh` backends.
|
|
79
85
|
fractal_ssh:
|
|
80
86
|
`FractalSSH` object, only relevant for the `slurm_ssh` backend.
|
|
@@ -107,7 +113,7 @@ def process_workflow(
|
|
|
107
113
|
common_script_lines=worker_init,
|
|
108
114
|
user_cache_dir=user_cache_dir,
|
|
109
115
|
) as runner:
|
|
110
|
-
|
|
116
|
+
execute_tasks(
|
|
111
117
|
wf_task_list=workflow.task_list[
|
|
112
118
|
first_task_index : (last_task_index + 1)
|
|
113
119
|
],
|
|
@@ -121,4 +127,5 @@ def process_workflow(
|
|
|
121
127
|
job_attribute_filters=job_attribute_filters,
|
|
122
128
|
job_type_filters=job_type_filters,
|
|
123
129
|
user_id=user_id,
|
|
130
|
+
resource_id=resource.id,
|
|
124
131
|
)
|
|
@@ -15,19 +15,25 @@ Slurm Backend
|
|
|
15
15
|
|
|
16
16
|
This backend runs fractal workflows in a SLURM cluster.
|
|
17
17
|
"""
|
|
18
|
+
|
|
18
19
|
from pathlib import Path
|
|
19
20
|
|
|
20
|
-
from ..executors.slurm_common.get_slurm_config import get_slurm_config
|
|
21
|
-
from ..executors.slurm_sudo.runner import SudoSlurmRunner
|
|
22
|
-
from ..set_start_and_last_task_index import set_start_and_last_task_index
|
|
23
|
-
from .runner import execute_tasks_v2
|
|
24
21
|
from fractal_server.app.models.v2 import DatasetV2
|
|
25
22
|
from fractal_server.app.models.v2 import Profile
|
|
26
23
|
from fractal_server.app.models.v2 import Resource
|
|
27
24
|
from fractal_server.app.models.v2 import WorkflowV2
|
|
25
|
+
from fractal_server.runner.executors.slurm_common.get_slurm_config import (
|
|
26
|
+
get_slurm_config,
|
|
27
|
+
)
|
|
28
|
+
from fractal_server.runner.executors.slurm_sudo.runner import SlurmSudoRunner
|
|
29
|
+
from fractal_server.runner.set_start_and_last_task_index import (
|
|
30
|
+
set_start_and_last_task_index,
|
|
31
|
+
)
|
|
28
32
|
from fractal_server.ssh._fabric import FractalSSH
|
|
29
33
|
from fractal_server.types import AttributeFilters
|
|
30
34
|
|
|
35
|
+
from .runner import execute_tasks
|
|
36
|
+
|
|
31
37
|
|
|
32
38
|
def process_workflow(
|
|
33
39
|
*,
|
|
@@ -71,7 +77,7 @@ def process_workflow(
|
|
|
71
77
|
resource: Computational resource for running this job.
|
|
72
78
|
profile: Computational profile for running this job.
|
|
73
79
|
user_cache_dir:
|
|
74
|
-
User-writeable folder (typically a subfolder of `
|
|
80
|
+
User-writeable folder (typically a subfolder of `project_dirs`).
|
|
75
81
|
Only relevant for `slurm_sudo` and `slurm_ssh` backends.
|
|
76
82
|
fractal_ssh:
|
|
77
83
|
`FractalSSH` object, only relevant for the `slurm_ssh` backend.
|
|
@@ -94,7 +100,7 @@ def process_workflow(
|
|
|
94
100
|
if isinstance(worker_init, str):
|
|
95
101
|
worker_init = worker_init.split("\n")
|
|
96
102
|
|
|
97
|
-
with
|
|
103
|
+
with SlurmSudoRunner(
|
|
98
104
|
root_dir_local=workflow_dir_local,
|
|
99
105
|
root_dir_remote=workflow_dir_remote,
|
|
100
106
|
common_script_lines=worker_init,
|
|
@@ -103,7 +109,7 @@ def process_workflow(
|
|
|
103
109
|
user_cache_dir=user_cache_dir,
|
|
104
110
|
slurm_account=slurm_account,
|
|
105
111
|
) as runner:
|
|
106
|
-
|
|
112
|
+
execute_tasks(
|
|
107
113
|
wf_task_list=workflow.task_list[
|
|
108
114
|
first_task_index : (last_task_index + 1)
|
|
109
115
|
],
|
|
@@ -117,4 +123,5 @@ def process_workflow(
|
|
|
117
123
|
job_attribute_filters=job_attribute_filters,
|
|
118
124
|
job_type_filters=job_type_filters,
|
|
119
125
|
user_id=user_id,
|
|
126
|
+
resource_id=resource.id,
|
|
120
127
|
)
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
from typing import TypeVar
|
|
2
2
|
|
|
3
|
-
from .task_interface import InitArgsModel
|
|
4
3
|
from fractal_server.images import SingleImage
|
|
5
4
|
from fractal_server.images import SingleImageTaskOutput
|
|
6
5
|
|
|
6
|
+
from .task_interface import InitArgsModel
|
|
7
|
+
|
|
7
8
|
T = TypeVar("T", SingleImage, SingleImageTaskOutput, InitArgsModel)
|
|
8
9
|
|
|
9
10
|
|
|
@@ -7,13 +7,6 @@ from sqlalchemy.orm.attributes import flag_modified
|
|
|
7
7
|
from sqlmodel import delete
|
|
8
8
|
from sqlmodel import update
|
|
9
9
|
|
|
10
|
-
from .merge_outputs import merge_outputs
|
|
11
|
-
from .runner_functions import GetRunnerConfigType
|
|
12
|
-
from .runner_functions import run_v2_task_compound
|
|
13
|
-
from .runner_functions import run_v2_task_non_parallel
|
|
14
|
-
from .runner_functions import run_v2_task_parallel
|
|
15
|
-
from .runner_functions import SubmissionOutcome
|
|
16
|
-
from .task_interface import TaskOutput
|
|
17
10
|
from fractal_server.app.db import get_sync_db
|
|
18
11
|
from fractal_server.app.models.v2 import AccountingRecord
|
|
19
12
|
from fractal_server.app.models.v2 import DatasetV2
|
|
@@ -21,15 +14,16 @@ from fractal_server.app.models.v2 import HistoryImageCache
|
|
|
21
14
|
from fractal_server.app.models.v2 import HistoryRun
|
|
22
15
|
from fractal_server.app.models.v2 import HistoryUnit
|
|
23
16
|
from fractal_server.app.models.v2 import JobV2
|
|
17
|
+
from fractal_server.app.models.v2 import Resource
|
|
24
18
|
from fractal_server.app.models.v2 import TaskGroupV2
|
|
25
19
|
from fractal_server.app.models.v2 import WorkflowTaskV2
|
|
26
20
|
from fractal_server.app.schemas.v2 import HistoryUnitStatus
|
|
27
|
-
from fractal_server.app.schemas.v2 import
|
|
28
|
-
from fractal_server.app.schemas.v2 import
|
|
21
|
+
from fractal_server.app.schemas.v2 import TaskDump
|
|
22
|
+
from fractal_server.app.schemas.v2 import TaskGroupDump
|
|
29
23
|
from fractal_server.app.schemas.v2 import TaskType
|
|
30
24
|
from fractal_server.images import SingleImage
|
|
31
|
-
from fractal_server.images.status_tools import enrich_images_unsorted_sync
|
|
32
25
|
from fractal_server.images.status_tools import IMAGE_STATUS_KEY
|
|
26
|
+
from fractal_server.images.status_tools import enrich_images_unsorted_sync
|
|
33
27
|
from fractal_server.images.tools import filter_image_list
|
|
34
28
|
from fractal_server.images.tools import find_image_by_zarr_url
|
|
35
29
|
from fractal_server.images.tools import merge_type_filters
|
|
@@ -39,6 +33,14 @@ from fractal_server.runner.executors.base_runner import BaseRunner
|
|
|
39
33
|
from fractal_server.runner.v2.db_tools import update_status_of_history_run
|
|
40
34
|
from fractal_server.types import AttributeFilters
|
|
41
35
|
|
|
36
|
+
from .merge_outputs import merge_outputs
|
|
37
|
+
from .runner_functions import GetRunnerConfigType
|
|
38
|
+
from .runner_functions import SubmissionOutcome
|
|
39
|
+
from .runner_functions import run_task_compound
|
|
40
|
+
from .runner_functions import run_task_non_parallel
|
|
41
|
+
from .runner_functions import run_task_parallel
|
|
42
|
+
from .task_interface import TaskOutput
|
|
43
|
+
|
|
42
44
|
|
|
43
45
|
def _remove_status_from_attributes(
|
|
44
46
|
images: list[dict[str, Any]],
|
|
@@ -81,7 +83,7 @@ def get_origin_attribute_and_types(
|
|
|
81
83
|
return updated_attributes, updated_types
|
|
82
84
|
|
|
83
85
|
|
|
84
|
-
def
|
|
86
|
+
def execute_tasks(
|
|
85
87
|
*,
|
|
86
88
|
wf_task_list: list[WorkflowTaskV2],
|
|
87
89
|
dataset: DatasetV2,
|
|
@@ -94,6 +96,7 @@ def execute_tasks_v2(
|
|
|
94
96
|
get_runner_config: GetRunnerConfigType,
|
|
95
97
|
job_type_filters: dict[str, bool],
|
|
96
98
|
job_attribute_filters: AttributeFilters,
|
|
99
|
+
resource_id: int,
|
|
97
100
|
) -> None:
|
|
98
101
|
logger = get_logger(logger_name=logger_name)
|
|
99
102
|
|
|
@@ -164,10 +167,10 @@ def execute_tasks_v2(
|
|
|
164
167
|
# Create dumps for workflowtask and taskgroup
|
|
165
168
|
workflowtask_dump = dict(
|
|
166
169
|
**wftask.model_dump(exclude={"task"}),
|
|
167
|
-
task=
|
|
170
|
+
task=TaskDump(**wftask.task.model_dump()).model_dump(),
|
|
168
171
|
)
|
|
169
172
|
task_group = db.get(TaskGroupV2, wftask.task.taskgroupv2_id)
|
|
170
|
-
task_group_dump =
|
|
173
|
+
task_group_dump = TaskGroupDump(
|
|
171
174
|
**task_group.model_dump()
|
|
172
175
|
).model_dump()
|
|
173
176
|
# Create HistoryRun
|
|
@@ -210,20 +213,37 @@ def execute_tasks_v2(
|
|
|
210
213
|
f"attribute_filters={job_attribute_filters})."
|
|
211
214
|
)
|
|
212
215
|
logger.info(error_msg)
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
216
|
+
with next(get_sync_db()) as db:
|
|
217
|
+
update_status_of_history_run(
|
|
218
|
+
history_run_id=history_run_id,
|
|
219
|
+
status=HistoryUnitStatus.FAILED,
|
|
220
|
+
db_sync=db,
|
|
221
|
+
)
|
|
218
222
|
raise JobExecutionError(error_msg)
|
|
219
223
|
|
|
220
|
-
#
|
|
224
|
+
# Fail if the resource is not open for new submissions
|
|
225
|
+
with next(get_sync_db()) as db:
|
|
226
|
+
resource = db.get(Resource, resource_id)
|
|
227
|
+
if resource.prevent_new_submissions:
|
|
228
|
+
error_msg = (
|
|
229
|
+
f"Cannot run '{task.name}', since the '{resource.name}' "
|
|
230
|
+
"resource is not currently active."
|
|
231
|
+
)
|
|
232
|
+
logger.info(error_msg)
|
|
233
|
+
update_status_of_history_run(
|
|
234
|
+
history_run_id=history_run_id,
|
|
235
|
+
status=HistoryUnitStatus.FAILED,
|
|
236
|
+
db_sync=db,
|
|
237
|
+
)
|
|
238
|
+
raise JobExecutionError(error_msg)
|
|
239
|
+
|
|
240
|
+
# TASK EXECUTION
|
|
221
241
|
try:
|
|
222
242
|
if task.type in [
|
|
223
243
|
TaskType.NON_PARALLEL,
|
|
224
244
|
TaskType.CONVERTER_NON_PARALLEL,
|
|
225
245
|
]:
|
|
226
|
-
outcomes_dict, num_tasks =
|
|
246
|
+
outcomes_dict, num_tasks = run_task_non_parallel(
|
|
227
247
|
images=filtered_images,
|
|
228
248
|
zarr_dir=zarr_dir,
|
|
229
249
|
wftask=wftask,
|
|
@@ -238,7 +258,7 @@ def execute_tasks_v2(
|
|
|
238
258
|
user_id=user_id,
|
|
239
259
|
)
|
|
240
260
|
elif task.type == TaskType.PARALLEL:
|
|
241
|
-
outcomes_dict, num_tasks =
|
|
261
|
+
outcomes_dict, num_tasks = run_task_parallel(
|
|
242
262
|
images=filtered_images,
|
|
243
263
|
wftask=wftask,
|
|
244
264
|
task=task,
|
|
@@ -254,7 +274,7 @@ def execute_tasks_v2(
|
|
|
254
274
|
TaskType.COMPOUND,
|
|
255
275
|
TaskType.CONVERTER_COMPOUND,
|
|
256
276
|
]:
|
|
257
|
-
outcomes_dict, num_tasks =
|
|
277
|
+
outcomes_dict, num_tasks = run_task_compound(
|
|
258
278
|
images=filtered_images,
|
|
259
279
|
zarr_dir=zarr_dir,
|
|
260
280
|
wftask=wftask,
|
|
@@ -312,9 +332,7 @@ def execute_tasks_v2(
|
|
|
312
332
|
# history status.
|
|
313
333
|
for image_obj in current_task_output.image_list_updates:
|
|
314
334
|
image = image_obj.model_dump()
|
|
315
|
-
if image["zarr_url"] in [
|
|
316
|
-
img["zarr_url"] for img in tmp_images
|
|
317
|
-
]:
|
|
335
|
+
if image["zarr_url"] in [img["zarr_url"] for img in tmp_images]:
|
|
318
336
|
img_search = find_image_by_zarr_url(
|
|
319
337
|
images=tmp_images,
|
|
320
338
|
zarr_url=image["zarr_url"],
|
|
@@ -418,9 +436,7 @@ def execute_tasks_v2(
|
|
|
418
436
|
|
|
419
437
|
# Update type_filters based on task-manifest output_types
|
|
420
438
|
type_filters_from_task_manifest = task.output_types
|
|
421
|
-
current_dataset_type_filters.update(
|
|
422
|
-
type_filters_from_task_manifest
|
|
423
|
-
)
|
|
439
|
+
current_dataset_type_filters.update(type_filters_from_task_manifest)
|
|
424
440
|
except Exception as e:
|
|
425
441
|
logger.error(
|
|
426
442
|
"Unexpected error in post-task-execution block. "
|
|
@@ -6,12 +6,6 @@ from typing import Protocol
|
|
|
6
6
|
from pydantic import BaseModel
|
|
7
7
|
from pydantic import ConfigDict
|
|
8
8
|
|
|
9
|
-
from ..exceptions import JobExecutionError
|
|
10
|
-
from ..exceptions import TaskOutputValidationError
|
|
11
|
-
from .db_tools import update_status_of_history_unit
|
|
12
|
-
from .deduplicate_list import deduplicate_list
|
|
13
|
-
from .task_interface import InitTaskOutput
|
|
14
|
-
from .task_interface import TaskOutput
|
|
15
9
|
from fractal_server.app.db import get_sync_db
|
|
16
10
|
from fractal_server.app.models.v2 import HistoryUnit
|
|
17
11
|
from fractal_server.app.models.v2 import TaskV2
|
|
@@ -22,16 +16,16 @@ from fractal_server.exceptions import UnreachableBranchError
|
|
|
22
16
|
from fractal_server.logger import set_logger
|
|
23
17
|
from fractal_server.runner.config import JobRunnerConfigLocal
|
|
24
18
|
from fractal_server.runner.config import JobRunnerConfigSLURM
|
|
19
|
+
from fractal_server.runner.exceptions import JobExecutionError
|
|
20
|
+
from fractal_server.runner.exceptions import TaskOutputValidationError
|
|
25
21
|
from fractal_server.runner.executors.base_runner import BaseRunner
|
|
26
22
|
from fractal_server.runner.executors.slurm_common.slurm_config import (
|
|
27
23
|
SlurmConfig,
|
|
28
24
|
)
|
|
29
|
-
from fractal_server.runner.task_files import enrich_task_files_multisubmit
|
|
30
25
|
from fractal_server.runner.task_files import SUBMIT_PREFIX
|
|
31
26
|
from fractal_server.runner.task_files import TaskFiles
|
|
32
|
-
from fractal_server.runner.
|
|
33
|
-
|
|
34
|
-
)
|
|
27
|
+
from fractal_server.runner.task_files import enrich_task_files_multisubmit
|
|
28
|
+
from fractal_server.runner.v2.db_tools import bulk_update_status_of_history_unit
|
|
35
29
|
from fractal_server.runner.v2.db_tools import bulk_upsert_image_cache_fast
|
|
36
30
|
from fractal_server.runner.v2.task_interface import (
|
|
37
31
|
_cast_and_validate_InitTaskOutput,
|
|
@@ -40,6 +34,11 @@ from fractal_server.runner.v2.task_interface import (
|
|
|
40
34
|
_cast_and_validate_TaskOutput,
|
|
41
35
|
)
|
|
42
36
|
|
|
37
|
+
from .db_tools import update_status_of_history_unit
|
|
38
|
+
from .deduplicate_list import deduplicate_list
|
|
39
|
+
from .task_interface import InitTaskOutput
|
|
40
|
+
from .task_interface import TaskOutput
|
|
41
|
+
|
|
43
42
|
|
|
44
43
|
class GetRunnerConfigTypeLocal(Protocol):
|
|
45
44
|
def __call__(
|
|
@@ -48,8 +47,7 @@ class GetRunnerConfigTypeLocal(Protocol):
|
|
|
48
47
|
wftask: WorkflowTaskV2,
|
|
49
48
|
which_type: Literal["non_parallel", "parallel"],
|
|
50
49
|
tot_tasks: int,
|
|
51
|
-
) -> JobRunnerConfigLocal:
|
|
52
|
-
...
|
|
50
|
+
) -> JobRunnerConfigLocal: ...
|
|
53
51
|
|
|
54
52
|
|
|
55
53
|
class GetRunnerConfigTypeSLURM(Protocol):
|
|
@@ -59,17 +57,16 @@ class GetRunnerConfigTypeSLURM(Protocol):
|
|
|
59
57
|
wftask: WorkflowTaskV2,
|
|
60
58
|
which_type: Literal["non_parallel", "parallel"],
|
|
61
59
|
tot_tasks: int,
|
|
62
|
-
) -> SlurmConfig:
|
|
63
|
-
...
|
|
60
|
+
) -> SlurmConfig: ...
|
|
64
61
|
|
|
65
62
|
|
|
66
63
|
GetRunnerConfigType = GetRunnerConfigTypeLocal | GetRunnerConfigTypeSLURM
|
|
67
64
|
|
|
68
65
|
|
|
69
66
|
__all__ = [
|
|
70
|
-
"
|
|
71
|
-
"
|
|
72
|
-
"
|
|
67
|
+
"run_task_parallel",
|
|
68
|
+
"run_task_non_parallel",
|
|
69
|
+
"run_task_compound",
|
|
73
70
|
]
|
|
74
71
|
|
|
75
72
|
|
|
@@ -148,7 +145,7 @@ def _check_parallelization_list_size(my_list):
|
|
|
148
145
|
)
|
|
149
146
|
|
|
150
147
|
|
|
151
|
-
def
|
|
148
|
+
def run_task_non_parallel(
|
|
152
149
|
*,
|
|
153
150
|
images: list[dict[str, Any]],
|
|
154
151
|
zarr_dir: str,
|
|
@@ -171,9 +168,7 @@ def run_v2_task_non_parallel(
|
|
|
171
168
|
TaskType.NON_PARALLEL,
|
|
172
169
|
TaskType.CONVERTER_NON_PARALLEL,
|
|
173
170
|
]:
|
|
174
|
-
raise ValueError(
|
|
175
|
-
f"Invalid {task_type=} for `run_v2_task_non_parallel`."
|
|
176
|
-
)
|
|
171
|
+
raise ValueError(f"Invalid {task_type=} for `run_task_non_parallel`.")
|
|
177
172
|
|
|
178
173
|
# Get TaskFiles object
|
|
179
174
|
task_files = TaskFiles(
|
|
@@ -216,7 +211,7 @@ def run_v2_task_non_parallel(
|
|
|
216
211
|
db.commit()
|
|
217
212
|
db.refresh(history_unit)
|
|
218
213
|
logger.debug(
|
|
219
|
-
"[
|
|
214
|
+
"[run_task_non_parallel] Created `HistoryUnit` with "
|
|
220
215
|
f"{history_run_id=}."
|
|
221
216
|
)
|
|
222
217
|
history_unit_id = history_unit.id
|
|
@@ -268,7 +263,7 @@ def run_v2_task_non_parallel(
|
|
|
268
263
|
return outcome, num_tasks
|
|
269
264
|
|
|
270
265
|
|
|
271
|
-
def
|
|
266
|
+
def run_task_parallel(
|
|
272
267
|
*,
|
|
273
268
|
images: list[dict[str, Any]],
|
|
274
269
|
task: TaskV2,
|
|
@@ -329,7 +324,7 @@ def run_v2_task_parallel(
|
|
|
329
324
|
db.add_all(history_units)
|
|
330
325
|
db.commit()
|
|
331
326
|
logger.debug(
|
|
332
|
-
f"[
|
|
327
|
+
f"[run_task_non_parallel] Created {len(history_units)} "
|
|
333
328
|
"`HistoryUnit`s."
|
|
334
329
|
)
|
|
335
330
|
|
|
@@ -391,7 +386,7 @@ def run_v2_task_parallel(
|
|
|
391
386
|
return outcome, num_tasks
|
|
392
387
|
|
|
393
388
|
|
|
394
|
-
def
|
|
389
|
+
def run_task_compound(
|
|
395
390
|
*,
|
|
396
391
|
images: list[dict[str, Any]],
|
|
397
392
|
zarr_dir: str,
|
|
@@ -448,7 +443,7 @@ def run_v2_task_compound(
|
|
|
448
443
|
db.refresh(history_unit)
|
|
449
444
|
init_history_unit_id = history_unit.id
|
|
450
445
|
logger.debug(
|
|
451
|
-
"[
|
|
446
|
+
"[run_task_compound] Created `HistoryUnit` with "
|
|
452
447
|
f"{init_history_unit_id=}."
|
|
453
448
|
)
|
|
454
449
|
# Create one `HistoryImageCache` for each input image
|
|
@@ -560,8 +555,7 @@ def run_v2_task_compound(
|
|
|
560
555
|
for history_unit in history_units:
|
|
561
556
|
db.refresh(history_unit)
|
|
562
557
|
logger.debug(
|
|
563
|
-
f"[
|
|
564
|
-
"`HistoryUnit`s."
|
|
558
|
+
f"[run_task_compound] Created {len(history_units)} `HistoryUnit`s."
|
|
565
559
|
)
|
|
566
560
|
history_unit_ids = [history_unit.id for history_unit in history_units]
|
|
567
561
|
|