fractal-server 2.12.1__py3-none-any.whl → 2.13.1__py3-none-any.whl

fractal_server/app/runner/executors/slurm/ssh/executor.py

@@ -1,20 +1,9 @@
-# This adapts clusterfutures <https://github.com/sampsyo/clusterfutures>
-# Original Copyright
-# Copyright 2021 Adrian Sampson <asampson@cs.washington.edu>
-# License: MIT
-#
-# Modified by:
-# Jacopo Nespolo <jacopo.nespolo@exact-lab.it>
-# Tommaso Comparin <tommaso.comparin@exact-lab.it>
-# Marco Franzon <marco.franzon@exact-lab.it>
-#
-# Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
-# University of Zurich
 import json
 import math
 import sys
 import threading
 import time
+from concurrent.futures import Executor
 from concurrent.futures import Future
 from concurrent.futures import InvalidStateError
 from copy import copy
@@ -25,18 +14,18 @@ from typing import Optional
 from typing import Sequence
 
 import cloudpickle
-from cfut import SlurmExecutor
 
 from ....filenames import SHUTDOWN_FILENAME
 from ....task_files import get_task_file_paths
 from ....task_files import TaskFiles
 from ....versions import get_versions
+from ..._job_states import STATES_FINISHED
 from ...slurm._slurm_config import SlurmConfig
 from .._batching import heuristics
 from ..utils_executors import get_pickle_file_path
 from ..utils_executors import get_slurm_file_path
 from ..utils_executors import get_slurm_script_file_path
-from ._executor_wait_thread import FractalSlurmWaitThread
+from ._executor_wait_thread import FractalSlurmSSHWaitThread
 from fractal_server.app.runner.components import _COMPONENT_KEY_
 from fractal_server.app.runner.compress_folder import compress_folder
 from fractal_server.app.runner.exceptions import JobExecutionError
@@ -48,24 +37,31 @@ from fractal_server.logger import set_logger
 from fractal_server.ssh._fabric import FractalSSH
 from fractal_server.syringe import Inject
 
+
 logger = set_logger(__name__)
 
 
-class FractalSlurmSSHExecutor(SlurmExecutor):
+class FractalSlurmSSHExecutor(Executor):
     """
-    FractalSlurmSSHExecutor (inherits from cfut.SlurmExecutor)
+    Executor to submit SLURM jobs via SSH
+
+    This class is a custom re-implementation of the SLURM executor from
+
+    > clusterfutures <https://github.com/sampsyo/clusterfutures>
+    > Original Copyright
+    > Copyright 2021 Adrian Sampson <asampson@cs.washington.edu>
+    > License: MIT
 
-    FIXME: docstring
 
     Attributes:
         fractal_ssh: FractalSSH connection with custom lock
-        shutdown_file:
-        python_remote: Equal to `settings.FRACTAL_SLURM_WORKER_PYTHON`
-        wait_thread_cls: Class for waiting thread
         workflow_dir_local:
             Directory for both the cfut/SLURM and fractal-server files and logs
         workflow_dir_remote:
             Directory for both the cfut/SLURM and fractal-server files and logs
+        shutdown_file:
+        python_remote: Equal to `settings.FRACTAL_SLURM_WORKER_PYTHON`
+        wait_thread_cls: Class for waiting thread
         common_script_lines:
             Arbitrary script lines that will always be included in the
             sbatch script
@@ -82,10 +78,10 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
     shutdown_file: str
     python_remote: str
 
-    wait_thread_cls = FractalSlurmWaitThread
+    wait_thread_cls = FractalSlurmSSHWaitThread
 
     common_script_lines: list[str]
-    slurm_account: Optional[str]
+    slurm_account: Optional[str] = None
 
     jobs: dict[str, tuple[Future, SlurmJob]]
     map_jobid_to_slurm_files_local: dict[str, tuple[str, str, str]]
@@ -1159,7 +1155,7 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
             Path(tarfile_path_local).unlink()
 
         t_1 = time.perf_counter()
-        logger.info("[_get_subfolder_sftp] End - " f"elapsed: {t_1-t_0:.3f} s")
+        logger.info(f"[_get_subfolder_sftp] End - elapsed: {t_1 - t_0:.3f} s")
 
     def _prepare_sbatch_script(
         self,
@@ -1258,7 +1254,7 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
         logger.debug("Executor shutdown: end")
 
     def _stop_and_join_wait_thread(self):
-        self.wait_thread.stop()
+        self.wait_thread.shutdown = True
         self.wait_thread.join()
 
     def __exit__(self, *args, **kwargs):
@@ -1295,8 +1291,6 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
         (released under the MIT licence)
         """
 
-        from cfut.slurm import STATES_FINISHED
-
         logger.debug(
             f"[FractalSlurmSSHExecutor._jobs_finished] START ({job_ids=})"
         )
@@ -1387,6 +1381,6 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
         t_end_handshake = time.perf_counter()
         logger.info(
             "[FractalSlurmSSHExecutor.ssh_handshake] END"
-            f" - elapsed: {t_end_handshake-t_start_handshake:.3f} s"
+            f" - elapsed: {t_end_handshake - t_start_handshake:.3f} s"
         )
         return remote_versions

fractal_server/app/runner/executors/slurm/sudo/__init__.py

@@ -1,3 +0,0 @@
-from .executor import SlurmExecutor
-
-__all__ = ["SlurmExecutor"]

fractal_server/app/runner/executors/slurm/sudo/_check_jobs_status.py

@@ -1,8 +1,7 @@
 from subprocess import run  # nosec
 
-from cfut.slurm import STATES_FINISHED
-
 from ......logger import set_logger
+from ..._job_states import STATES_FINISHED
 
 
 logger = set_logger(__name__)

fractal_server/app/runner/executors/slurm/sudo/_executor_wait_thread.py

@@ -1,12 +1,10 @@
 import os
+import threading
 import time
 import traceback
 from itertools import count
-from typing import Callable
 from typing import Optional
 
-from cfut import FileWaitThread
-
 from ......logger import set_logger
 from ._check_jobs_status import _jobs_finished
 from fractal_server.app.runner.exceptions import JobExecutionError
@@ -14,33 +12,43 @@ from fractal_server.app.runner.exceptions import JobExecutionError
 logger = set_logger(__name__)
 
 
-class FractalFileWaitThread(FileWaitThread):
+class FractalSlurmSudoWaitThread(threading.Thread):
     """
-    Overrides the original clusterfutures.FileWaitThread, so that:
-
-    1. Each jobid in the waiting list is associated to a tuple of filenames,
-       rather than a single one.
-    2. In the `check` method, we avoid output-file existence checks (which
-       would require `sudo -u user ls` calls), and we rather check for the
-       existence of the shutdown file. All the logic to check whether a job is
-       complete is deferred to the `cfut.slurm.jobs_finished` function.
-    3. There are additional attributes (`slurm_user`, `shutdown_file` and
-       `shutdown_callback`).
-
-    This class is copied from clusterfutures 0.5. Original Copyright: 2022
-    Adrian Sampson, released under the MIT licence
-
-    Note: in principle we could avoid the definition of
-    `FractalFileWaitThread`, and pack all this code in
-    `FractalSlurmWaitThread`.
+    Thread that monitors a pool of SLURM jobs
+
+    This class is a custom re-implementation of the waiting thread class from:
+
+    > clusterfutures <https://github.com/sampsyo/clusterfutures>
+    > Original Copyright
+    > Copyright 2021 Adrian Sampson <asampson@cs.washington.edu>
+    > License: MIT
+
+    Attributes:
+        slurm_user:
+        shutdown_file:
+        shutdown_callback:
+        slurm_poll_interval:
+        waiting:
+        shutdown:
+        lock:
     """
 
     slurm_user: str
     shutdown_file: Optional[str] = None
-    shutdown_callback: Callable
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
+    shutdown_callback: callable
+    slurm_poll_interval: int = 30
+    waiting: dict[tuple[str, ...], str]
+    shutdown: bool
+    _lock: threading.Lock
+
+    def __init__(self, callback: callable, interval=1):
+        threading.Thread.__init__(self, daemon=True)
+        self.callback = callback
+        self.interval = interval
+        self.waiting = {}
+        self._lock = threading.Lock()  # To protect the .waiting dict
+        self.shutdown = False
+        self.active_job_ids = []
 
     def wait(
         self,
@@ -61,10 +69,10 @@ class FractalFileWaitThread(FileWaitThread):
             error_msg = "Cannot call `wait` method after executor shutdown."
             logger.warning(error_msg)
             raise JobExecutionError(info=error_msg)
-        with self.lock:
+        with self._lock:
             self.waiting[filenames] = jobid
 
-    def check(self, i):
+    def check_shutdown(self, i):
         """
         Do one shutdown-file-existence check.
 
@@ -99,30 +107,12 @@ class FractalFileWaitThread(FileWaitThread):
             if self.shutdown:
                 self.shutdown_callback()
                 return
-            with self.lock:
+            with self._lock:
                 self.check(i)
             time.sleep(self.interval)
 
-
-class FractalSlurmWaitThread(FractalFileWaitThread):
-    """
-    Replaces the original clusterfutures.SlurmWaitThread, to inherit from
-    FractalFileWaitThread instead of FileWaitThread.
-
-    The function is copied from clusterfutures 0.5. Original Copyright: 2022
-    Adrian Sampson, released under the MIT licence
-
-    **Note**: if `self.interval != 1` then this should be modified, but for
-    `clusterfutures` v0.5 `self.interval` is indeed equal to `1`.
-
-    Changed from clusterfutures:
-    * Rename `id_to_filename` to `id_to_filenames`
-    """
-
-    slurm_poll_interval = 30
-
     def check(self, i):
-        super().check(i)
+        self.check_shutdown(i)
         if i % (self.slurm_poll_interval // self.interval) == 0:
             try:
                 finished_jobs = _jobs_finished(self.waiting.values())

fractal_server/app/runner/executors/slurm/sudo/executor.py

@@ -1,21 +1,12 @@
-# This adapts clusterfutures <https://github.com/sampsyo/clusterfutures>
-# Original Copyright
-# Copyright 2021 Adrian Sampson <asampson@cs.washington.edu>
-# License: MIT
-#
-# Modified by:
-# Jacopo Nespolo <jacopo.nespolo@exact-lab.it>
-# Tommaso Comparin <tommaso.comparin@exact-lab.it>
-# Marco Franzon <marco.franzon@exact-lab.it>
-#
-# Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
-# University of Zurich
 import json
 import math
 import shlex
 import subprocess  # nosec
 import sys
+import threading
 import time
+import uuid
+from concurrent.futures import Executor
 from concurrent.futures import Future
 from concurrent.futures import InvalidStateError
 from copy import copy
@@ -27,8 +18,6 @@ from typing import Optional
 from typing import Sequence
 
 import cloudpickle
-from cfut import SlurmExecutor
-from cfut.util import random_string
 
 from ......config import get_settings
 from ......logger import set_logger
@@ -43,7 +32,7 @@ from .._batching import heuristics
 from ..utils_executors import get_pickle_file_path
 from ..utils_executors import get_slurm_file_path
 from ..utils_executors import get_slurm_script_file_path
-from ._executor_wait_thread import FractalSlurmWaitThread
+from ._executor_wait_thread import FractalSlurmSudoWaitThread
 from ._subprocess_run_as_user import _glob_as_user
 from ._subprocess_run_as_user import _glob_as_user_strict
 from ._subprocess_run_as_user import _path_exists_as_user
@@ -180,9 +169,7 @@ class SlurmJob:
             )
         else:
             self.wftask_file_prefixes = wftask_file_prefixes
-        self.workerids = tuple(
-            random_string() for i in range(self.num_tasks_tot)
-        )
+        self.workerids = tuple(uuid.uuid4() for i in range(self.num_tasks_tot))
         self.slurm_config = slurm_config
 
     def get_clean_output_pickle_files(self) -> tuple[str, ...]:
@@ -193,9 +180,17 @@ class SlurmJob:
         return tuple(str(f.as_posix()) for f in self.output_pickle_files)
 
 
-class FractalSlurmExecutor(SlurmExecutor):
+class FractalSlurmSudoExecutor(Executor):
     """
-    FractalSlurmExecutor (inherits from cfut.SlurmExecutor)
+    Executor to submit SLURM jobs as a different user, via `sudo -u`
+
+    This class is a custom re-implementation of the SLURM executor from
+
+    > clusterfutures <https://github.com/sampsyo/clusterfutures>
+    > Original Copyright
+    > Copyright 2021 Adrian Sampson <asampson@cs.washington.edu>
+    > License: MIT
+
 
     Attributes:
         slurm_user:
@@ -211,7 +206,7 @@ class FractalSlurmExecutor(SlurmExecutor):
             Dictionary with paths of slurm-related files for active jobs
     """
 
-    wait_thread_cls = FractalSlurmWaitThread
+    wait_thread_cls = FractalSlurmSudoWaitThread
     slurm_user: str
     shutdown_file: str
     common_script_lines: list[str]
@@ -219,7 +214,7 @@ class FractalSlurmExecutor(SlurmExecutor):
     workflow_dir_local: Path
     workflow_dir_remote: Path
     map_jobid_to_slurm_files: dict[str, tuple[str, str, str]]
-    slurm_account: Optional[str]
+    slurm_account: Optional[str] = None
     jobs: dict[str, tuple[Future, SlurmJob]]
 
     def __init__(
@@ -244,7 +239,13 @@ class FractalSlurmExecutor(SlurmExecutor):
                 "Missing attribute FractalSlurmExecutor.slurm_user"
             )
 
-        super().__init__(*args, **kwargs)
+        self.jobs = {}
+        self.job_outfiles = {}
+        self.jobs_lock = threading.Lock()
+        self.jobs_empty_cond = threading.Condition(self.jobs_lock)
+
+        self.wait_thread = self.wait_thread_cls(self._completion)
+        self.wait_thread.start()
 
         # Assign `wait_thread.shutdown_callback` early, since it may be called
         # from within `_stop_and_join_wait_thread` (e.g. if an exception is
@@ -1239,7 +1240,7 @@ class FractalSlurmExecutor(SlurmExecutor):
         logger.debug("Executor shutdown: end")
 
     def _stop_and_join_wait_thread(self):
-        self.wait_thread.stop()
+        self.wait_thread.shutdown = True
         self.wait_thread.join()
 
     def __exit__(self, *args, **kwargs):

fractal_server/app/runner/v2/__init__.py

@@ -31,9 +31,6 @@ from ..executors.slurm.sudo._subprocess_run_as_user import _mkdir_as_user
 from ..filenames import WORKFLOW_LOG_FILENAME
 from ..task_files import task_subfolder_name
 from ._local import process_workflow as local_process_workflow
-from ._local_experimental import (
-    process_workflow as local_experimental_process_workflow,
-)
 from ._slurm_ssh import process_workflow as slurm_ssh_process_workflow
 from ._slurm_sudo import process_workflow as slurm_sudo_process_workflow
 from .handle_failed_job import mark_last_wftask_as_failed
@@ -45,7 +42,6 @@ _backends = {}
 _backends["local"] = local_process_workflow
 _backends["slurm"] = slurm_sudo_process_workflow
 _backends["slurm_ssh"] = slurm_ssh_process_workflow
-_backends["local_experimental"] = local_experimental_process_workflow
 
 
 def fail_job(
@@ -74,6 +70,7 @@ def submit_workflow(
     workflow_id: int,
     dataset_id: int,
     job_id: int,
+    user_id: int,
     user_settings: UserSettings,
     worker_init: Optional[str] = None,
     slurm_user: Optional[str] = None,
@@ -94,6 +91,8 @@ def submit_workflow(
         job_id:
             Id of the job record which stores the state for the current
             workflow application.
+        user_id:
+            User ID.
         worker_init:
             Custom executor parameters that get parsed before the execution of
             each task.
@@ -184,8 +183,6 @@ def submit_workflow(
             # Define and create WORKFLOW_DIR_REMOTE
             if FRACTAL_RUNNER_BACKEND == "local":
                 WORKFLOW_DIR_REMOTE = WORKFLOW_DIR_LOCAL
-            elif FRACTAL_RUNNER_BACKEND == "local_experimental":
-                WORKFLOW_DIR_REMOTE = WORKFLOW_DIR_LOCAL
             elif FRACTAL_RUNNER_BACKEND == "slurm":
                 WORKFLOW_DIR_REMOTE = (
                     Path(user_cache_dir) / WORKFLOW_DIR_LOCAL.name
@@ -287,9 +284,6 @@ def submit_workflow(
         if FRACTAL_RUNNER_BACKEND == "local":
             process_workflow = local_process_workflow
             backend_specific_kwargs = {}
-        elif FRACTAL_RUNNER_BACKEND == "local_experimental":
-            process_workflow = local_experimental_process_workflow
-            backend_specific_kwargs = {}
         elif FRACTAL_RUNNER_BACKEND == "slurm":
             process_workflow = slurm_sudo_process_workflow
             backend_specific_kwargs = dict(
@@ -321,6 +315,7 @@ def submit_workflow(
         process_workflow(
             workflow=workflow,
             dataset=dataset,
+            user_id=user_id,
             workflow_dir_local=WORKFLOW_DIR_LOCAL,
             workflow_dir_remote=WORKFLOW_DIR_REMOTE,
             logger_name=logger_name,

fractal_server/app/runner/v2/_local/__init__.py

@@ -41,6 +41,7 @@ def process_workflow(
     last_task_index: Optional[int] = None,
     logger_name: str,
     job_attribute_filters: AttributeFiltersType,
+    user_id: int,
     # Slurm-specific
     user_cache_dir: Optional[str] = None,
     slurm_user: Optional[str] = None,
@@ -75,6 +76,7 @@ def process_workflow(
             Positional index of the last task to execute; if `None`, proceed
             until the last task.
         logger_name: Logger name
+        user_id:
         slurm_user:
             Username to impersonate to run the workflow. This argument is
             present for compatibility with the standard backend interface, but
@@ -126,4 +128,5 @@ def process_workflow(
             logger_name=logger_name,
             submit_setup_call=_local_submit_setup,
             job_attribute_filters=job_attribute_filters,
+            user_id=user_id,
         )

fractal_server/app/runner/v2/_local/_local_config.py

@@ -17,8 +17,8 @@ from typing import Literal
 from typing import Optional
 
 from pydantic import BaseModel
-from pydantic import Extra
-from pydantic.error_wrappers import ValidationError
+from pydantic import ConfigDict
+from pydantic import ValidationError
 
 from .....config import get_settings
 from .....syringe import Inject
@@ -33,7 +33,7 @@ class LocalBackendConfigError(ValueError):
     pass
 
 
-class LocalBackendConfig(BaseModel, extra=Extra.forbid):
+class LocalBackendConfig(BaseModel):
     """
     Specifications of the local-backend configuration
 
@@ -44,7 +44,8 @@ class LocalBackendConfig(BaseModel, extra=Extra.forbid):
             start at the same time.
     """
 
-    parallel_tasks_per_job: Optional[int]
+    model_config = ConfigDict(extra="forbid")
+    parallel_tasks_per_job: Optional[int] = None
 
 
 def get_default_local_backend_config():

fractal_server/app/runner/v2/_slurm_common/get_slurm_config.py

@@ -67,14 +67,14 @@ def get_slurm_config(
 
     # Incorporate slurm_env.default_slurm_config
     slurm_env = load_slurm_config_file(config_path=config_path)
-    slurm_dict = slurm_env.default_slurm_config.dict(
+    slurm_dict = slurm_env.default_slurm_config.model_dump(
         exclude_unset=True, exclude={"mem"}
     )
     if slurm_env.default_slurm_config.mem:
         slurm_dict["mem_per_task_MB"] = slurm_env.default_slurm_config.mem
 
     # Incorporate slurm_env.batching_config
-    for key, value in slurm_env.batching_config.dict().items():
+    for key, value in slurm_env.batching_config.model_dump().items():
         slurm_dict[key] = value
 
     # Incorporate slurm_env.user_local_exports
@@ -82,7 +82,7 @@ def get_slurm_config(
 
     logger.debug(
         "[get_slurm_config] Fractal SLURM configuration file: "
-        f"{slurm_env.dict()=}"
+        f"{slurm_env.model_dump()=}"
     )
 
     # GPU-related options
@@ -97,7 +97,7 @@ def get_slurm_config(
         needs_gpu = False
     logger.debug(f"[get_slurm_config] {needs_gpu=}")
     if needs_gpu:
-        for key, value in slurm_env.gpu_slurm_config.dict(
+        for key, value in slurm_env.gpu_slurm_config.model_dump(
             exclude_unset=True, exclude={"mem"}
         ).items():
             slurm_dict[key] = value

fractal_server/app/runner/v2/_slurm_ssh/__init__.py

@@ -45,6 +45,7 @@ def process_workflow(
     job_attribute_filters: AttributeFiltersType,
     fractal_ssh: FractalSSH,
     worker_init: Optional[str] = None,
+    user_id: int,
     # Not used
     user_cache_dir: Optional[str] = None,
     slurm_user: Optional[str] = None,
@@ -94,4 +95,5 @@ def process_workflow(
             logger_name=logger_name,
             submit_setup_call=_slurm_submit_setup,
             job_attribute_filters=job_attribute_filters,
+            user_id=user_id,
         )

fractal_server/app/runner/v2/_slurm_sudo/__init__.py

@@ -21,7 +21,7 @@ from typing import Optional
 
 from ....models.v2 import DatasetV2
 from ....models.v2 import WorkflowV2
-from ...executors.slurm.sudo.executor import FractalSlurmExecutor
+from ...executors.slurm.sudo.executor import FractalSlurmSudoExecutor
 from ...set_start_and_last_task_index import set_start_and_last_task_index
 from ..runner import execute_tasks_v2
 from ._submit_setup import _slurm_submit_setup
@@ -38,6 +38,7 @@ def process_workflow(
     last_task_index: Optional[int] = None,
     logger_name: str,
     job_attribute_filters: AttributeFiltersType,
+    user_id: int,
     # Slurm-specific
     user_cache_dir: Optional[str] = None,
     slurm_user: Optional[str] = None,
@@ -64,7 +65,7 @@ def process_workflow(
     if isinstance(worker_init, str):
         worker_init = worker_init.split("\n")
 
-    with FractalSlurmExecutor(
+    with FractalSlurmSudoExecutor(
         debug=True,
         keep_logs=True,
         slurm_user=slurm_user,
@@ -85,4 +86,5 @@ def process_workflow(
             logger_name=logger_name,
             submit_setup_call=_slurm_submit_setup,
             job_attribute_filters=job_attribute_filters,
+            user_id=user_id,
         )

fractal_server/app/runner/v2/deduplicate_list.py

@@ -16,7 +16,7 @@ def deduplicate_list(
     new_list_dict = []
     new_list_objs = []
     for this_obj in this_list:
-        this_dict = this_obj.dict()
+        this_dict = this_obj.model_dump()
         if this_dict not in new_list_dict:
             new_list_dict.append(this_dict)
             new_list_objs.append(this_obj)