fractal-server 2.12.0a1__py3-none-any.whl → 2.13.0__py3-none-any.whl

fractal_server/app/routes/auth/_aux_auth.py

@@ -58,11 +58,14 @@ async def _get_single_user_with_groups(
         group_ids_names.insert(0, default_group)
     else:
         pass
+    oauth_accounts = [
+        oauth_account.model_dump() for oauth_account in user.oauth_accounts
+    ]
 
     return UserRead(
         **user.model_dump(),
         group_ids_names=group_ids_names,
-        oauth_accounts=user.oauth_accounts,
+        oauth_accounts=oauth_accounts,
     )
 
 

fractal_server/app/routes/auth/current_user.py

@@ -57,14 +57,14 @@ async def patch_current_user(
     Note: a user cannot patch their own password (as enforced within the
     `UserUpdateStrict` schema).
     """
-    update = UserUpdate(**user_update.dict(exclude_unset=True))
+    update = UserUpdate(**user_update.model_dump(exclude_unset=True))
 
     # NOTE: here it would be relevant to catch an `InvalidPasswordException`
     # (from `fastapi_users.exceptions`), if we were to allow users change
     # their own password
 
     user = await user_manager.update(update, current_user, safe=True)
-    validated_user = schemas.model_validate(UserOAuth, user)
+    validated_user = schemas.model_validate(UserOAuth, user.model_dump())
 
     patched_user = await db.get(
         UserOAuth, validated_user.id, populate_existing=True
@@ -82,7 +82,6 @@ async def get_current_user_settings(
     current_user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
 ) -> UserSettingsReadStrict:
-
     verify_user_has_settings(current_user)
     user_settings = await db.get(UserSettings, current_user.user_settings_id)
     return user_settings
@@ -96,13 +95,12 @@ async def patch_current_user_settings(
     current_user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
 ) -> UserSettingsReadStrict:
-
     verify_user_has_settings(current_user)
     current_user_settings = await db.get(
         UserSettings, current_user.user_settings_id
     )
 
-    for k, v in settings_update.dict(exclude_unset=True).items():
+    for k, v in settings_update.model_dump(exclude_unset=True).items():
         setattr(current_user_settings, k, v)
 
     db.add(current_user_settings)

fractal_server/app/routes/auth/group.py

@@ -194,7 +194,7 @@ async def patch_user_settings_bulk(
         .where(LinkUserGroup.group_id == group_id)
     )
     settings_list = res.scalars().all()
-    update = settings_update.dict(exclude_unset=True)
+    update = settings_update.model_dump(exclude_unset=True)
     for settings in settings_list:
         for k, v in update.items():
             setattr(settings, k, v)

fractal_server/app/routes/auth/users.py

@@ -75,7 +75,7 @@ async def patch_user(
             safe=False,
             request=None,
         )
-        validated_user = schemas.model_validate(UserOAuth, user)
+        validated_user = schemas.model_validate(UserOAuth, user.model_dump())
         patched_user = await db.get(
             UserOAuth, validated_user.id, populate_existing=True
         )
@@ -139,7 +139,6 @@ async def set_user_groups(
     superuser: UserOAuth = Depends(current_active_superuser),
     db: AsyncSession = Depends(get_async_db),
 ) -> UserRead:
-
     # Preliminary check that all objects exist in the db
     user = await _user_or_404(user_id=user_id, db=db)
     target_group_ids = user_update.group_ids
@@ -209,7 +208,6 @@ async def get_user_settings(
     superuser: UserOAuth = Depends(current_active_superuser),
     db: AsyncSession = Depends(get_async_db),
 ) -> UserSettingsRead:
-
     user = await _user_or_404(user_id=user_id, db=db)
     verify_user_has_settings(user)
     user_settings = await db.get(UserSettings, user.user_settings_id)
@@ -229,7 +227,7 @@ async def patch_user_settings(
     verify_user_has_settings(user)
     user_settings = await db.get(UserSettings, user.user_settings_id)
 
-    for k, v in settings_update.dict(exclude_unset=True).items():
+    for k, v in settings_update.model_dump(exclude_unset=True).items():
         setattr(user_settings, k, v)
 
     db.add(user_settings)

fractal_server/app/routes/aux/_runner.py

@@ -6,7 +6,7 @@ from ....syringe import Inject
 
 
 def _backend_supports_shutdown(backend: str) -> bool:
-    if backend in ["slurm", "slurm_ssh", "local_experimental"]:
+    if backend in ["slurm", "slurm_ssh"]:
         return True
     else:
         return False

fractal_server/app/routes/aux/validate_user_settings.py

@@ -1,6 +1,5 @@
 from fastapi import HTTPException
 from fastapi import status
-from pydantic import BaseModel
 from pydantic import ValidationError
 
 from fractal_server.app.db import AsyncSession
@@ -55,7 +54,7 @@ async def validate_user_settings(
         UserSettingsValidationModel = SlurmSudoUserSettings
     else:
         # For other backends, we don't validate anything
-        UserSettingsValidationModel = BaseModel
+        return user_settings
 
     try:
         UserSettingsValidationModel(**user_settings.model_dump())

fractal_server/app/runner/executors/_job_states.py

@@ -0,0 +1,13 @@
+# https://slurm.schedmd.com/squeue.html#lbAG
+STATES_FINISHED = {
+    "BOOT_FAIL",
+    "CANCELLED",
+    "COMPLETED",
+    "DEADLINE",
+    "FAILED",
+    "NODE_FAIL",
+    "OUT_OF_MEMORY",
+    "PREEMPTED",
+    "SPECIAL_EXIT",
+    "TIMEOUT",
+}

fractal_server/app/runner/executors/slurm/_slurm_config.py

@@ -18,9 +18,9 @@ from typing import Optional
 from typing import Union
 
 from pydantic import BaseModel
-from pydantic import Extra
+from pydantic import ConfigDict
 from pydantic import Field
-from pydantic.error_wrappers import ValidationError
+from pydantic import ValidationError
 
 from .....config import get_settings
 from .....logger import set_logger
@@ -37,7 +37,7 @@ class SlurmConfigError(ValueError):
     pass
 
 
-class _SlurmConfigSet(BaseModel, extra=Extra.forbid):
+class _SlurmConfigSet(BaseModel):
     """
     Options that can be set in `FRACTAL_SLURM_CONFIG_FILE` for the default/gpu
     SLURM config. Only used as part of `SlurmConfigFile`.
@@ -54,19 +54,21 @@ class _SlurmConfigSet(BaseModel, extra=Extra.forbid):
         extra_lines:
     """
 
-    partition: Optional[str]
-    cpus_per_task: Optional[int]
-    mem: Optional[Union[int, str]]
-    constraint: Optional[str]
-    gres: Optional[str]
-    time: Optional[str]
-    account: Optional[str]
-    extra_lines: Optional[list[str]]
-    pre_submission_commands: Optional[list[str]]
-    gpus: Optional[str]
+    model_config = ConfigDict(extra="forbid")
 
+    partition: Optional[str] = None
+    cpus_per_task: Optional[int] = None
+    mem: Optional[Union[int, str]] = None
+    constraint: Optional[str] = None
+    gres: Optional[str] = None
+    time: Optional[str] = None
+    account: Optional[str] = None
+    extra_lines: Optional[list[str]] = None
+    pre_submission_commands: Optional[list[str]] = None
+    gpus: Optional[str] = None
 
-class _BatchingConfigSet(BaseModel, extra=Extra.forbid):
+
+class _BatchingConfigSet(BaseModel):
     """
     Options that can be set in `FRACTAL_SLURM_CONFIG_FILE` to configure the
     batching strategy (that is, how to combine several tasks in a single SLURM
@@ -83,6 +85,8 @@ class _BatchingConfigSet(BaseModel, extra=Extra.forbid):
         max_num_jobs:
     """
 
+    model_config = ConfigDict(extra="forbid")
+
     target_cpus_per_job: int
     max_cpus_per_job: int
     target_mem_per_job: Union[int, str]
@@ -91,7 +95,7 @@ class _BatchingConfigSet(BaseModel, extra=Extra.forbid):
     max_num_jobs: int
 
 
-class SlurmConfigFile(BaseModel, extra=Extra.forbid):
+class SlurmConfigFile(BaseModel):
     """
     Specifications for the content of `FRACTAL_SLURM_CONFIG_FILE`
 
@@ -136,10 +140,12 @@ class SlurmConfigFile(BaseModel, extra=Extra.forbid):
             directory.
     """
 
+    model_config = ConfigDict(extra="forbid")
+
     default_slurm_config: _SlurmConfigSet
-    gpu_slurm_config: Optional[_SlurmConfigSet]
+    gpu_slurm_config: Optional[_SlurmConfigSet] = None
     batching_config: _BatchingConfigSet
-    user_local_exports: Optional[dict[str, str]]
+    user_local_exports: Optional[dict[str, str]] = None
 
 
 def load_slurm_config_file(
@@ -196,7 +202,7 @@ def load_slurm_config_file(
     return obj
 
 
-class SlurmConfig(BaseModel, extra=Extra.forbid):
+class SlurmConfig(BaseModel):
     """
     Abstraction for SLURM parameters
 
@@ -247,6 +253,8 @@ class SlurmConfig(BaseModel, extra=Extra.forbid):
             command.
     """
 
+    model_config = ConfigDict(extra="forbid")
+
     # Required SLURM parameters (note that the integer attributes are those
     # that will need to scale up with the number of parallel tasks per job)
     partition: str

fractal_server/app/runner/executors/slurm/ssh/__init__.py

@@ -1,3 +0,0 @@
-from .executor import SlurmExecutor
-
-__all__ = ["SlurmExecutor"]

fractal_server/app/runner/executors/slurm/ssh/_executor_wait_thread.py

@@ -1,10 +1,8 @@
 import os
+import threading
 import time
 import traceback
 from itertools import count
-from typing import Callable
-
-from cfut import FileWaitThread
 
 from ......logger import set_logger
 from fractal_server.app.runner.exceptions import JobExecutionError
@@ -12,35 +10,46 @@ from fractal_server.app.runner.exceptions import JobExecutionError
 logger = set_logger(__name__)
 
 
-class FractalSlurmWaitThread(FileWaitThread):
+class FractalSlurmSSHWaitThread(threading.Thread):
     """
-    Overrides the original clusterfutures.FileWaitThread, so that:
-
-    1. Each jobid in the waiting list is associated to a tuple of filenames,
-       rather than a single one.
-    2. In the `check` method, we avoid output-file existence checks (which
-       would require `sudo -u user ls` calls), and we rather check for the
-       existence of the shutdown file. All the logic to check whether a job is
-       complete is deferred to the `cfut.slurm.jobs_finished` function.
-    3. There are additional attributes (...).
-
-    This class is based on clusterfutures 0.5. Original Copyright: 2022
-    Adrian Sampson, released under the MIT licence
+    Thread that monitors a pool of SLURM jobs
+
+    This class is a custom re-implementation of the waiting thread class from:
+
+    > clusterfutures <https://github.com/sampsyo/clusterfutures>
+    > Original Copyright
+    > Copyright 2021 Adrian Sampson <asampson@cs.washington.edu>
+    > License: MIT
+
+    Attributes:
+        shutdown_file:
+        shutdown_callback:
+        slurm_poll_interval:
+        jobs_finished_callback:
+        active_job_ids:
+        shutdown:
+        lock:
     """
 
     shutdown_file: str
-    shutdown_callback: Callable
-    jobs_finished_callback: Callable
+    shutdown_callback: callable
     slurm_poll_interval = 30
+    jobs_finished_callback: callable
     active_job_ids: list[str]
+    shutdown: bool
+    _lock: threading.Lock
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, callback: callable, interval=1):
         """
         Init method
 
         This method is executed on the main thread.
         """
-        super().__init__(*args, **kwargs)
+        threading.Thread.__init__(self, daemon=True)
+        self.callback = callback
+        self.interval = interval
+        self._lock = threading.Lock()
+        self.shutdown = False
         self.active_job_ids = []
 
     def wait(self, *, job_id: str):
@@ -53,7 +62,7 @@ class FractalSlurmWaitThread(FileWaitThread):
             error_msg = "Cannot call `wait` method after executor shutdown."
             logger.warning(error_msg)
             raise JobExecutionError(info=error_msg)
-        with self.lock:
+        with self._lock:
             self.active_job_ids.append(job_id)
 
     def check_shutdown(self):
@@ -109,7 +118,7 @@ class FractalSlurmWaitThread(FileWaitThread):
                     pass
                 return
             if ind % skip == 0:
-                with self.lock:
+                with self._lock:
                     try:
                         self.check_jobs()
                     except Exception:  # nosec

fractal_server/app/runner/executors/slurm/ssh/_slurm_job.py

@@ -1,8 +1,7 @@
+import uuid
 from pathlib import Path
 from typing import Optional
 
-from cfut.util import random_string
-
 from fractal_server.app.runner.executors.slurm._slurm_config import (
     SlurmConfig,
 )
@@ -88,7 +87,6 @@ class SlurmJob:
         self,
         num_tasks_tot: int,
         slurm_config: SlurmConfig,
-        workflow_task_file_prefix: Optional[str] = None,
         slurm_file_prefix: Optional[str] = None,
         wftask_file_prefixes: Optional[tuple[str, ...]] = None,
         single_task_submission: bool = False,
@@ -107,9 +105,7 @@ class SlurmJob:
             )
         else:
             self.wftask_file_prefixes = wftask_file_prefixes
-        self.workerids = tuple(
-            random_string() for i in range(self.num_tasks_tot)
-        )
+        self.workerids = tuple(uuid.uuid4() for i in range(self.num_tasks_tot))
         self.slurm_config = slurm_config
 
     def get_clean_output_pickle_files(self) -> tuple[str, ...]:

fractal_server/app/runner/executors/slurm/ssh/executor.py

@@ -1,20 +1,9 @@
-# This adapts clusterfutures <https://github.com/sampsyo/clusterfutures>
-# Original Copyright
-# Copyright 2021 Adrian Sampson <asampson@cs.washington.edu>
-# License: MIT
-#
-# Modified by:
-# Jacopo Nespolo <jacopo.nespolo@exact-lab.it>
-# Tommaso Comparin <tommaso.comparin@exact-lab.it>
-# Marco Franzon <marco.franzon@exact-lab.it>
-#
-# Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
-# University of Zurich
 import json
 import math
 import sys
 import threading
 import time
+from concurrent.futures import Executor
 from concurrent.futures import Future
 from concurrent.futures import InvalidStateError
 from copy import copy
@@ -25,18 +14,18 @@ from typing import Optional
 from typing import Sequence
 
 import cloudpickle
-from cfut import SlurmExecutor
 
 from ....filenames import SHUTDOWN_FILENAME
 from ....task_files import get_task_file_paths
 from ....task_files import TaskFiles
 from ....versions import get_versions
+from ..._job_states import STATES_FINISHED
 from ...slurm._slurm_config import SlurmConfig
 from .._batching import heuristics
 from ..utils_executors import get_pickle_file_path
 from ..utils_executors import get_slurm_file_path
 from ..utils_executors import get_slurm_script_file_path
-from ._executor_wait_thread import FractalSlurmWaitThread
+from ._executor_wait_thread import FractalSlurmSSHWaitThread
 from fractal_server.app.runner.components import _COMPONENT_KEY_
 from fractal_server.app.runner.compress_folder import compress_folder
 from fractal_server.app.runner.exceptions import JobExecutionError
@@ -48,25 +37,31 @@ from fractal_server.logger import set_logger
 from fractal_server.ssh._fabric import FractalSSH
 from fractal_server.syringe import Inject
 
+
 logger = set_logger(__name__)
 
 
-class FractalSlurmSSHExecutor(SlurmExecutor):
+class FractalSlurmSSHExecutor(Executor):
     """
-    FractalSlurmSSHExecutor (inherits from cfut.SlurmExecutor)
+    Executor to submit SLURM jobs via SSH
+
+    This class is a custom re-implementation of the SLURM executor from
+
+    > clusterfutures <https://github.com/sampsyo/clusterfutures>
+    > Original Copyright
+    > Copyright 2021 Adrian Sampson <asampson@cs.washington.edu>
+    > License: MIT
 
-    FIXME: docstring
 
     Attributes:
         fractal_ssh: FractalSSH connection with custom lock
-        shutdown_file:
-        python_remote: Equal to `settings.FRACTAL_SLURM_WORKER_PYTHON`
-        wait_thread_cls: Class for waiting thread
-        keep_pickle_files:
         workflow_dir_local:
             Directory for both the cfut/SLURM and fractal-server files and logs
         workflow_dir_remote:
             Directory for both the cfut/SLURM and fractal-server files and logs
+        shutdown_file:
+        python_remote: Equal to `settings.FRACTAL_SLURM_WORKER_PYTHON`
+        wait_thread_cls: Class for waiting thread
         common_script_lines:
             Arbitrary script lines that will always be included in the
             sbatch script
@@ -83,11 +78,10 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
     shutdown_file: str
     python_remote: str
 
-    wait_thread_cls = FractalSlurmWaitThread
-    keep_pickle_files: bool
+    wait_thread_cls = FractalSlurmSSHWaitThread
 
     common_script_lines: list[str]
-    slurm_account: Optional[str]
+    slurm_account: Optional[str] = None
 
     jobs: dict[str, tuple[Future, SlurmJob]]
     map_jobid_to_slurm_files_local: dict[str, tuple[str, str, str]]
@@ -100,8 +94,6 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
         # Folders and files
         workflow_dir_local: Path,
         workflow_dir_remote: Path,
-        # Runner options
-        keep_pickle_files: bool = False,
         # Monitoring options
         slurm_poll_interval: Optional[int] = None,
         # SLURM submission script options
@@ -120,7 +112,6 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
             fractal_ssh:
             workflow_dir_local:
             workflow_dir_remote:
-            keep_pickle_files:
             slurm_poll_interval:
             common_script_lines:
             slurm_account:
@@ -194,7 +185,6 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
             raise e
 
         # Set/initialize some more options
-        self.keep_pickle_files = keep_pickle_files
         self.map_jobid_to_slurm_files_local = {}
 
     def _validate_common_script_lines(self):
@@ -901,12 +891,11 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
                 pass
         for job_id in remaining_job_ids:
             self._cleanup(job_id)
-        if not self.keep_pickle_files:
-            for job in remaining_jobs:
-                for path in job.output_pickle_files_local:
-                    path.unlink()
-                for path in job.input_pickle_files_local:
-                    path.unlink()
+        for job in remaining_jobs:
+            for path in job.output_pickle_files_local:
+                path.unlink()
+            for path in job.input_pickle_files_local:
+                path.unlink()
 
     def _completion(self, job_ids: list[str]) -> None:
         """
@@ -1001,8 +990,7 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
                                 f"Future {future} (SLURM job ID: {job_id}) "
                                 "was already cancelled."
                             )
-                            if not self.keep_pickle_files:
-                                in_path.unlink()
+                            in_path.unlink()
                             self._cleanup(job_id)
                             self._handle_remaining_jobs(
                                 remaining_futures=remaining_futures,
@@ -1062,17 +1050,15 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
                                     remaining_job_ids=remaining_job_ids,
                                 )
                                 return
-                        if not self.keep_pickle_files:
-                            out_path.unlink()
+                        out_path.unlink()
                     except InvalidStateError:
                         logger.warning(
                             f"Future {future} (SLURM job ID: {job_id}) was "
                             "already cancelled, exit from "
                             "FractalSlurmSSHExecutor._completion."
                         )
-                        if not self.keep_pickle_files:
-                            out_path.unlink()
-                            in_path.unlink()
+                        out_path.unlink()
+                        in_path.unlink()
 
                         self._cleanup(job_id)
                         self._handle_remaining_jobs(
@@ -1082,8 +1068,7 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
                         return
 
                     # Clean up input pickle file
-                    if not self.keep_pickle_files:
-                        in_path.unlink()
+                    in_path.unlink()
                 self._cleanup(job_id)
                 if job.single_task_submission:
                     future.set_result(outputs[0])
@@ -1170,7 +1155,7 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
             Path(tarfile_path_local).unlink()
 
         t_1 = time.perf_counter()
-        logger.info("[_get_subfolder_sftp] End - " f"elapsed: {t_1-t_0:.3f} s")
+        logger.info(f"[_get_subfolder_sftp] End - elapsed: {t_1 - t_0:.3f} s")
 
     def _prepare_sbatch_script(
         self,
@@ -1210,8 +1195,10 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
         script_lines = slurm_config.sort_script_lines(script_lines)
         logger.debug(script_lines)
 
-        # Always print output of `pwd`
-        script_lines.append('echo "Working directory (pwd): `pwd`"\n')
+        # Always print output of `uname -n` and `pwd`
+        script_lines.append(
+            '"Hostname: `uname -n`; current directory: `pwd`"\n'
+        )
 
         # Complete script preamble
         script_lines.append("\n")
@@ -1267,7 +1254,7 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
         logger.debug("Executor shutdown: end")
 
     def _stop_and_join_wait_thread(self):
-        self.wait_thread.stop()
+        self.wait_thread.shutdown = True
         self.wait_thread.join()
 
     def __exit__(self, *args, **kwargs):
@@ -1304,8 +1291,6 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
         (released under the MIT licence)
         """
 
-        from cfut.slurm import STATES_FINISHED
-
         logger.debug(
             f"[FractalSlurmSSHExecutor._jobs_finished] START ({job_ids=})"
         )
@@ -1396,6 +1381,6 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
         t_end_handshake = time.perf_counter()
         logger.info(
             "[FractalSlurmSSHExecutor.ssh_handshake] END"
-            f" - elapsed: {t_end_handshake-t_start_handshake:.3f} s"
+            f" - elapsed: {t_end_handshake - t_start_handshake:.3f} s"
         )
         return remote_versions

fractal_server/app/runner/executors/slurm/sudo/__init__.py

@@ -1,3 +0,0 @@
-from .executor import SlurmExecutor
-
-__all__ = ["SlurmExecutor"]

fractal_server/app/runner/executors/slurm/sudo/_check_jobs_status.py

@@ -1,8 +1,7 @@
 from subprocess import run  # nosec
 
-from cfut.slurm import STATES_FINISHED
-
 from ......logger import set_logger
+from ..._job_states import STATES_FINISHED
 
 
 logger = set_logger(__name__)