fractal-server 2.7.0a9__py3-none-any.whl → 2.7.0a11__py3-none-any.whl

fractal_server/__init__.py

@@ -1 +1 @@
-__VERSION__ = "2.7.0a9"
+__VERSION__ = "2.7.0a11"

fractal_server/app/routes/api/v2/workflow_import.py

@@ -241,7 +241,10 @@ async def _get_task_by_taskimport(
             "Found many task groups, after filtering by version."
         )
         final_task_group = await _disambiguate_task_groups(
-            matching_task_groups, user_id, db, default_group_id
+            matching_task_groups=matching_task_groups,
+            user_id=user_id,
+            db=db,
+            default_group_id=default_group_id,
         )
         if final_task_group is None:
             logger.info(

fractal_server/app/runner/executors/slurm/ssh/executor.py

@@ -861,7 +861,7 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
 
         # Transfer archive
         t_0_put = time.perf_counter()
-        self.fractal_ssh.put(
+        self.fractal_ssh.send_file(
             local=tarfile_path_local,
             remote=tarfile_path_remote,
         )
@@ -1055,55 +1055,59 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
         Arguments:
             jobid: ID of the SLURM job
         """
-
-        # Loop over all job_ids, and fetch future and job objects
-        futures: list[Future] = []
-        jobs: list[SlurmJob] = []
-        with self.jobs_lock:
-            for job_id in job_ids:
-                future, job = self.jobs.pop(job_id)
-                futures.append(future)
-                jobs.append(job)
-            if not self.jobs:
-                self.jobs_empty_cond.notify_all()
-
-        # Fetch subfolder from remote host
+        # Handle all uncaught exceptions in this broad try/except block
         try:
-            self._get_subfolder_sftp(jobs=jobs)
-        except NoValidConnectionsError as e:
-            logger.error("NoValidConnectionError")
-            logger.error(f"{str(e)=}")
-            logger.error(f"{e.errors=}")
-            for err in e.errors:
-                logger.error(f"{str(err)}")
-
-            raise e
-
-        # First round of checking whether all output files exist
-        missing_out_paths = []
-        for job in jobs:
-            for ind_out_path, out_path in enumerate(
-                job.output_pickle_files_local
-            ):
-                if not out_path.exists():
-                    missing_out_paths.append(out_path)
-        num_missing = len(missing_out_paths)
-        if num_missing > 0:
-            # Output pickle files may be missing e.g. because of some slow
-            # filesystem operation; wait some time before re-trying
-            settings = Inject(get_settings)
-            sleep_time = settings.FRACTAL_SLURM_ERROR_HANDLING_INTERVAL
             logger.info(
-                f"{num_missing} output pickle files are missing; "
-                f"sleep {sleep_time} seconds."
+                f"[FractalSlurmSSHExecutor._completion] START, for {job_ids=}."
             )
-            for missing_file in missing_out_paths:
-                logger.debug(f"Missing output pickle file: {missing_file}")
-            time.sleep(sleep_time)
 
-        # Handle all jobs
-        for ind_job, job_id in enumerate(job_ids):
+            # Loop over all job_ids, and fetch future and job objects
+            futures: list[Future] = []
+            jobs: list[SlurmJob] = []
+            with self.jobs_lock:
+                for job_id in job_ids:
+                    future, job = self.jobs.pop(job_id)
+                    futures.append(future)
+                    jobs.append(job)
+                if not self.jobs:
+                    self.jobs_empty_cond.notify_all()
+
+            # Fetch subfolder from remote host
             try:
+                self._get_subfolder_sftp(jobs=jobs)
+            except NoValidConnectionsError as e:
+                logger.error("NoValidConnectionError")
+                logger.error(f"{str(e)=}")
+                logger.error(f"{e.errors=}")
+                for err in e.errors:
+                    logger.error(f"{str(err)}")
+
+                raise e
+
+            # First round of checking whether all output files exist
+            missing_out_paths = []
+            for job in jobs:
+                for ind_out_path, out_path in enumerate(
+                    job.output_pickle_files_local
+                ):
+                    if not out_path.exists():
+                        missing_out_paths.append(out_path)
+            num_missing = len(missing_out_paths)
+            if num_missing > 0:
+                # Output pickle files may be missing e.g. because of some slow
+                # filesystem operation; wait some time before re-trying
+                settings = Inject(get_settings)
+                sleep_time = settings.FRACTAL_SLURM_ERROR_HANDLING_INTERVAL
+                logger.info(
+                    f"{num_missing} output pickle files are missing; "
+                    f"sleep {sleep_time} seconds."
+                )
+                for missing_file in missing_out_paths:
+                    logger.debug(f"Missing output pickle file: {missing_file}")
+                time.sleep(sleep_time)
+
+            # Handle all jobs
+            for ind_job, job_id in enumerate(job_ids):
                 # Retrieve job and future objects
                 job = jobs[ind_job]
                 future = futures[ind_job]
@@ -1128,6 +1132,11 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
                                 remaining_futures=remaining_futures,
                                 remaining_job_ids=remaining_job_ids,
                             )
+                            logger.info(
+                                "[FractalSlurmSSHExecutor._completion] END, "
+                                f"for {job_ids=}, with JobExecutionError due "
+                                f"to missing {out_path.as_posix()}."
+                            )
                             return
                         except InvalidStateError:
                             logger.warning(
@@ -1141,6 +1150,12 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
                                 remaining_futures=remaining_futures,
                                 remaining_job_ids=remaining_job_ids,
                             )
+                            logger.info(
+                                "[FractalSlurmSSHExecutor._completion] END, "
+                                f"for {job_ids=}, with JobExecutionError/"
+                                "InvalidStateError due to "
+                                f"missing {out_path.as_posix()}."
+                            )
                             return
 
                     # Read the task output
@@ -1217,16 +1232,22 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
                 else:
                     future.set_result(outputs)
 
-            except Exception as e:
+        except Exception as e:
+            logger.warning(
+                "[FractalSlurmSSHExecutor._completion] "
+                f"An exception took place: {str(e)}."
+            )
+            for future in futures:
                 try:
+                    logger.info(f"Set exception for {future=}")
                     future.set_exception(e)
-                    return
                 except InvalidStateError:
-                    logger.warning(
-                        f"Future {future} (SLURM job ID: {job_id}) was already"
-                        " cancelled, exit from"
-                        " FractalSlurmSSHExecutor._completion."
-                    )
+                    logger.info(f"Future {future} was already cancelled.")
+            logger.info(
+                f"[FractalSlurmSSHExecutor._completion] END, for {job_ids=}, "
+                "from within exception handling."
+            )
+            return
 
     def _get_subfolder_sftp(self, jobs: list[SlurmJob]) -> None:
         """
@@ -1255,16 +1276,9 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
             self.workflow_dir_remote / f"{subfolder_name}.tar.gz"
         ).as_posix()
 
-        # Remove local tarfile - FIXME SSH: is this needed?
-        logger.warning(f"In principle I just removed {tarfile_path_local}")
-        logger.warning(f"{Path(tarfile_path_local).exists()=}")
-
-        # Remove remote tarfile - FIXME SSH: is this needed?
-        # rm_command = f"rm {tarfile_path_remote}"
-        # _run_command_over_ssh(cmd=rm_command, fractal_ssh=self.fractal_ssh)
-        logger.warning(f"Unlink {tarfile_path_remote=} - START")
-        self.fractal_ssh.sftp().unlink(tarfile_path_remote)
-        logger.warning(f"Unlink {tarfile_path_remote=} - STOP")
+        # Remove remote tarfile
+        rm_command = f"rm {tarfile_path_remote}"
+        self.fractal_ssh.run_command(cmd=rm_command)
 
         # Create remote tarfile
         tar_command = (
@@ -1278,7 +1292,7 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
 
         # Fetch tarfile
         t_0_get = time.perf_counter()
-        self.fractal_ssh.get(
+        self.fractal_ssh.fetch_file(
             remote=tarfile_path_remote,
             local=tarfile_path_local,
         )
@@ -1291,6 +1305,11 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
         # Extract tarfile locally
         extract_archive(Path(tarfile_path_local))
 
+        # Remove local tarfile
+        if Path(tarfile_path_local).exists():
+            logger.warning(f"Remove existing file {tarfile_path_local}.")
+            Path(tarfile_path_local).unlink()
+
         t_1 = time.perf_counter()
         logger.info("[_get_subfolder_sftp] End - " f"elapsed: {t_1-t_0:.3f} s")
 

fractal_server/ssh/_fabric.py

@@ -1,3 +1,4 @@
+import json
 import logging
 import time
 from contextlib import contextmanager
@@ -23,21 +24,58 @@ class FractalSSHTimeoutError(RuntimeError):
     pass
 
 
-class FractalSSHListTimeoutError(RuntimeError):
-    pass
+logger = set_logger(__name__)
 
 
-logger = set_logger(__name__)
+@contextmanager
+def _acquire_lock_with_timeout(
+    lock: Lock,
+    label: str,
+    timeout: float,
+    logger_name: str = __name__,
+) -> Generator[Literal[True], Any, None]:
+    """
+    Given a `threading.Lock` object, try to acquire it within a given timeout.
+
+    Arguments:
+        lock:
+        label:
+        timeout:
+        logger_name:
+    """
+    logger = get_logger(logger_name)
+    logger.info(f"Trying to acquire lock for '{label}', with {timeout=}")
+    result = lock.acquire(timeout=timeout)
+    try:
+        if not result:
+            logger.error(f"Lock for '{label}' was *not* acquired.")
+            raise FractalSSHTimeoutError(
+                f"Failed to acquire lock for '{label}' within "
+                f"{timeout} seconds"
+            )
+        logger.info(f"Lock for '{label}' was acquired.")
+        yield result
+    finally:
+        if result:
+            lock.release()
+            logger.info(f"Lock for '{label}' was released.")
 
 
 class FractalSSH(object):
     """
-    FIXME SSH: Fix docstring
+    Wrapper of `fabric.Connection` object, enriched with locks.
+
+    Note: methods marked as `_unsafe` should not be used directly,
+    since they do not enforce locking.
 
     Attributes:
         _lock:
-        connection:
+        _connection:
         default_lock_timeout:
+        default_max_attempts:
+        default_base_interval:
+        sftp_get_prefetch:
+        sftp_get_max_requests:
         logger_name:
     """
 
@@ -46,6 +84,8 @@ class FractalSSH(object):
     default_lock_timeout: float
     default_max_attempts: int
     default_base_interval: float
+    sftp_get_prefetch: bool
+    sftp_get_max_requests: int
     logger_name: str
 
     def __init__(
@@ -54,6 +94,8 @@ class FractalSSH(object):
         default_timeout: float = 250,
         default_max_attempts: int = 5,
         default_base_interval: float = 3.0,
+        sftp_get_prefetch: bool = False,
+        sftp_get_max_requests: int = 64,
         logger_name: str = __name__,
     ):
         self._lock = Lock()
@@ -61,28 +103,11 @@ class FractalSSH(object):
         self.default_lock_timeout = default_timeout
         self.default_base_interval = default_base_interval
         self.default_max_attempts = default_max_attempts
+        self.sftp_get_prefetch = sftp_get_prefetch
+        self.sftp_get_max_requests = sftp_get_max_requests
         self.logger_name = logger_name
         set_logger(self.logger_name)
 
-    @contextmanager
-    def acquire_timeout(
-        self, timeout: float
-    ) -> Generator[Literal[True], Any, None]:
-        self.logger.debug(f"Trying to acquire lock, with {timeout=}")
-        result = self._lock.acquire(timeout=timeout)
-        try:
-            if not result:
-                self.logger.error("Lock was *NOT* acquired.")
-                raise FractalSSHTimeoutError(
-                    f"Failed to acquire lock within {timeout} seconds"
-                )
-            self.logger.debug("Lock was acquired.")
-            yield result
-        finally:
-            if result:
-                self._lock.release()
-                self.logger.debug("Lock was released")
-
     @property
     def is_connected(self) -> bool:
         return self._connection.is_connected
@@ -91,36 +116,82 @@ class FractalSSH(object):
     def logger(self) -> logging.Logger:
         return get_logger(self.logger_name)
 
-    def put(
-        self, *args, lock_timeout: Optional[float] = None, **kwargs
+    def _put(
+        self,
+        *,
+        local: str,
+        remote: str,
+        label: str,
+        lock_timeout: Optional[float] = None,
     ) -> Result:
+        """
+        Transfer a local file to a remote path, via SFTP.
+        """
         actual_lock_timeout = self.default_lock_timeout
         if lock_timeout is not None:
             actual_lock_timeout = lock_timeout
-        with self.acquire_timeout(timeout=actual_lock_timeout):
-            return self._connection.put(*args, **kwargs)
+        with _acquire_lock_with_timeout(
+            lock=self._lock,
+            label=label,
+            timeout=actual_lock_timeout,
+        ):
+            return self._sftp_unsafe().put(local, remote)
 
-    def get(
-        self, *args, lock_timeout: Optional[float] = None, **kwargs
+    def _get(
+        self,
+        *,
+        local: str,
+        remote: str,
+        label: str,
+        lock_timeout: Optional[float] = None,
     ) -> Result:
         actual_lock_timeout = self.default_lock_timeout
         if lock_timeout is not None:
             actual_lock_timeout = lock_timeout
-        with self.acquire_timeout(timeout=actual_lock_timeout):
-            return self._connection.get(*args, **kwargs)
+        with _acquire_lock_with_timeout(
+            lock=self._lock,
+            label=label,
+            timeout=actual_lock_timeout,
+        ):
+            return self._sftp_unsafe().get(
+                remote,
+                local,
+                prefetch=self.sftp_get_prefetch,
+                max_concurrent_prefetch_requests=self.sftp_get_max_requests,
+            )
 
-    def run(
-        self, *args, lock_timeout: Optional[float] = None, **kwargs
+    def _run(
+        self, *args, label: str, lock_timeout: Optional[float] = None, **kwargs
     ) -> Any:
         actual_lock_timeout = self.default_lock_timeout
         if lock_timeout is not None:
             actual_lock_timeout = lock_timeout
-        with self.acquire_timeout(timeout=actual_lock_timeout):
+        with _acquire_lock_with_timeout(
+            lock=self._lock,
+            label=label,
+            timeout=actual_lock_timeout,
+        ):
             return self._connection.run(*args, **kwargs)
 
-    def sftp(self) -> paramiko.sftp_client.SFTPClient:
+    def _sftp_unsafe(self) -> paramiko.sftp_client.SFTPClient:
+        """
+        This is marked as unsafe because you should only use its methods
+        after acquiring a lock.
+        """
         return self._connection.sftp()
 
+    def read_remote_json_file(self, filepath: str) -> dict[str, Any]:
+        self.logger.info(f"START reading remote JSON file {filepath}.")
+        with _acquire_lock_with_timeout(
+            lock=self._lock,
+            label="read_remote_json_file",
+            timeout=self.default_lock_timeout,
+        ):
+            with self._sftp_unsafe().open(filepath, "r") as f:
+                data = json.load(f)
+        self.logger.info(f"END reading remote JSON file {filepath}.")
+        return data
+
     def check_connection(self) -> None:
         """
         Open the SSH connection and handle exceptions.
@@ -131,7 +202,12 @@ class FractalSSH(object):
         """
         if not self._connection.is_connected:
             try:
-                self._connection.open()
+                with _acquire_lock_with_timeout(
+                    lock=self._lock,
+                    label="_connection.open",
+                    timeout=self.default_lock_timeout,
+                ):
+                    self._connection.open()
             except Exception as e:
                 raise RuntimeError(
                     f"Cannot open SSH connection. Original error:\n{str(e)}"
@@ -146,8 +222,12 @@ class FractalSSH(object):
         because we observed cases where `is_connected=False` but the underlying
         `Transport` object was not closed.
         """
-
-        self._connection.close()
+        with _acquire_lock_with_timeout(
+            lock=self._lock,
+            label="_connection.close",
+            timeout=self.default_lock_timeout,
+        ):
+            self._connection.close()
 
         if self._connection.client is not None:
             self._connection.client.close()
@@ -197,8 +277,11 @@ class FractalSSH(object):
             self.logger.info(f"{prefix} START running '{cmd}' over SSH.")
             try:
                 # Case 1: Command runs successfully
-                res = self.run(
-                    cmd, lock_timeout=actual_lock_timeout, hide=True
+                res = self._run(
+                    cmd,
+                    label=f"run {cmd}",
+                    lock_timeout=actual_lock_timeout,
+                    hide=True,
                 )
                 t_1 = time.perf_counter()
                 self.logger.info(
@@ -250,7 +333,6 @@ class FractalSSH(object):
         *,
         local: str,
         remote: str,
-        logger_name: Optional[str] = None,
         lock_timeout: Optional[float] = None,
     ) -> None:
         """
@@ -261,28 +343,65 @@ class FractalSSH(object):
             remote: Target path on remote host
             fractal_ssh: FractalSSH connection object with custom lock
             logger_name: Name of the logger
-
         """
         try:
-            self.put(local=local, remote=remote, lock_timeout=lock_timeout)
+            prefix = "[send_file]"
+            self.logger.info(f"{prefix} START transfer of '{local}' over SSH.")
+            self._put(
+                local=local,
+                remote=remote,
+                lock_timeout=lock_timeout,
+                label=f"send_file {local=} {remote=}",
+            )
+            self.logger.info(f"{prefix} END transfer of '{local}' over SSH.")
         except Exception as e:
-            logger = get_logger(logger_name=logger_name)
-            logger.error(
+            self.logger.error(
                 f"Transferring {local=} to {remote=} over SSH failed.\n"
                 f"Original Error:\n{str(e)}."
             )
             raise e
 
+    def fetch_file(
+        self,
+        *,
+        local: str,
+        remote: str,
+        lock_timeout: Optional[float] = None,
+    ) -> None:
+        """
+        Transfer a file via SSH
+
+        Args:
+            local: Local path to file
+            remote: Target path on remote host
+            logger_name: Name of the logger
+            lock_timeout:
+        """
+        try:
+            prefix = "[fetch_file] "
+            self.logger.info(f"{prefix} START fetching '{remote}' over SSH.")
+            self._get(
+                local=local,
+                remote=remote,
+                lock_timeout=lock_timeout,
+                label=f"fetch_file {local=} {remote=}",
+            )
+            self.logger.info(f"{prefix} END fetching '{remote}' over SSH.")
+        except Exception as e:
+            self.logger.error(
+                f"Transferring {remote=} to {local=} over SSH failed.\n"
+                f"Original Error:\n{str(e)}."
+            )
+            raise e
+
     def mkdir(self, *, folder: str, parents: bool = True) -> None:
         """
         Create a folder remotely via SSH.
 
         Args:
             folder:
-            fractal_ssh:
             parents:
         """
-        # FIXME SSH: try using `mkdir` method of `paramiko.SFTPClient`
         if parents:
             cmd = f"mkdir -p {folder}"
         else:
@@ -339,12 +458,18 @@ class FractalSSH(object):
             contents: File contents
             lock_timeout:
         """
+        self.logger.info(f"START writing to remote file {path}.")
         actual_lock_timeout = self.default_lock_timeout
         if lock_timeout is not None:
             actual_lock_timeout = lock_timeout
-        with self.acquire_timeout(timeout=actual_lock_timeout):
-            with self.sftp().open(filename=path, mode="w") as f:
+        with _acquire_lock_with_timeout(
+            lock=self._lock,
+            label=f"write_remote_file {path=}",
+            timeout=actual_lock_timeout,
+        ):
+            with self._sftp_unsafe().open(filename=path, mode="w") as f:
                 f.write(content)
+        self.logger.info(f"END writing to remote file {path}.")
 
 
 class FractalSSHList(object):
@@ -425,7 +550,11 @@ class FractalSSHList(object):
                     "look_for_keys": False,
                 },
             )
-            with self.acquire_lock_with_timeout():
+            with _acquire_lock_with_timeout(
+                lock=self._lock,
+                label="FractalSSHList.get",
+                timeout=self._timeout,
+            ):
                 self._data[key] = FractalSSH(connection=connection)
                 return self._data[key]
 
@@ -465,7 +594,11 @@ class FractalSSHList(object):
             key_path:
         """
         key = (host, user, key_path)
-        with self.acquire_lock_with_timeout():
+        with _acquire_lock_with_timeout(
+            lock=self._lock,
+            timeout=self._timeout,
+            label="FractalSSHList.remove",
+        ):
             self.logger.info(
                 f"Removing FractalSSH object for {user}@{host} "
                 "from collection."
@@ -492,24 +625,4 @@ class FractalSSHList(object):
                 f"Closing FractalSSH object for {user}@{host} "
                 f"({fractal_ssh_obj.is_connected=})."
             )
-            with fractal_ssh_obj.acquire_timeout(timeout=timeout):
-                fractal_ssh_obj.close()
-
-    @contextmanager
-    def acquire_lock_with_timeout(self) -> Generator[Literal[True], Any, None]:
-        self.logger.debug(
-            f"Trying to acquire lock, with timeout {self._timeout} s"
-        )
-        result = self._lock.acquire(timeout=self._timeout)
-        try:
-            if not result:
-                self.logger.error("Lock was *NOT* acquired.")
-                raise FractalSSHListTimeoutError(
-                    f"Failed to acquire lock within {self._timeout} ss"
-                )
-            self.logger.debug("Lock was acquired.")
-            yield result
-        finally:
-            if result:
-                self._lock.release()
-                self.logger.debug("Lock was released")
+            fractal_ssh_obj.close()

fractal_server/tasks/v2/background_operations_ssh.py

@@ -1,4 +1,3 @@
-import json
 import os
 from pathlib import Path
 from tempfile import TemporaryDirectory
@@ -144,6 +143,13 @@ def background_collect_pip_ssh(
         for key, value in task_group.model_dump().items():
             logger.debug(f"task_group.{key}: {value}")
 
+        # `remove_venv_folder_upon_failure` is set to True only if
+        # script 1 goes through, which means that the remote folder
+        # `package_env_dir` did not already exist. If this remote
+        # folder already existed, then script 1 fails and the boolean
+        # flag `remove_venv_folder_upon_failure` remains false.
+        remove_venv_folder_upon_failure = False
+
         # Open a DB session soon, since it is needed for updating `state`
         with next(get_sync_db()) as db:
             try:
@@ -187,12 +193,11 @@ def background_collect_pip_ssh(
                 # long operations that do not use the db
                 db.close()
 
-                # `remove_venv_folder_upon_failure` is set to True only if
-                # script 1 goes through, which means that the remote folder
-                # `package_env_dir` did not already exist. If this remote
-                # folder already existed, then script 1 fails and the boolean
-                # flag `remove_venv_folder_upon_failure` remains false.
-                remove_venv_folder_upon_failure = False
+                # Create remote folder (note that because of `parents=True` we
+                # are in the `no error if existing, make parent directories as
+                # needed` scenario)
+                fractal_ssh.mkdir(folder=tasks_base_dir, parents=True)
+
                 stdout = _customize_and_run_template(
                     script_filename="_1_create_venv.sh",
                     **common_args,
@@ -263,8 +268,9 @@ def background_collect_pip_ssh(
                 ).as_posix()
 
                 # Read and validate remote manifest file
-                with fractal_ssh.sftp().open(manifest_path_remote, "r") as f:
-                    pkg_manifest_dict = json.load(f)
+                pkg_manifest_dict = fractal_ssh.read_remote_json_file(
+                    manifest_path_remote
+                )
                 logger.info(f"collecting - loaded {manifest_path_remote=}")
                 pkg_manifest = ManifestV2(**pkg_manifest_dict)
                 logger.info("collecting - manifest is a valid ManifestV2")

{fractal_server-2.7.0a9.dist-info → fractal_server-2.7.0a11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fractal-server
-Version: 2.7.0a9
+Version: 2.7.0a11
 Summary: Server component of the Fractal analytics platform
 Home-page: https://github.com/fractal-analytics-platform/fractal-server
 License: BSD-3-Clause

{fractal_server-2.7.0a9.dist-info → fractal_server-2.7.0a11.dist-info}/RECORD

@@ -1,4 +1,4 @@
-fractal_server/__init__.py,sha256=yjIvBx2CMOQ9DTXJNDwb8gCOMh5maKwXOpnecX3Wsog,24
+fractal_server/__init__.py,sha256=U6Vt70hce3I5-D3BcN_SgVx3kKrnUis8AS0m58E-7IY,25
 fractal_server/__main__.py,sha256=dEkCfzLLQrIlxsGC-HBfoR-RBMWnJDgNrxYTyzmE9c0,6146
 fractal_server/alembic.ini,sha256=MWwi7GzjzawI9cCAK1LW7NxIBQDUqD12-ptJoq5JpP0,3153
 fractal_server/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -55,7 +55,7 @@ fractal_server/app/routes/api/v2/task_collection.py,sha256=gCxOwigT_tfs8lCDNoE7n
 fractal_server/app/routes/api/v2/task_collection_custom.py,sha256=9T0U_4gqrQbJCy6uFDCMSZ-b1sfNIzyz_qm4P41W2Gs,6133
 fractal_server/app/routes/api/v2/task_group.py,sha256=P32EUYbtGThexSWe5zI9WUFrgoOMof035fJBILTNnfQ,5580
 fractal_server/app/routes/api/v2/workflow.py,sha256=PyvkrUHHzFGUGZE5X0VW5u3DPQA7wtXXNcEpG7-N66I,8687
-fractal_server/app/routes/api/v2/workflow_import.py,sha256=3qX3iHnLJb62TQFLQKakcC_mYh-tWXkYL-4PuyEkwRs,10895
+fractal_server/app/routes/api/v2/workflow_import.py,sha256=rD26vZ-ztjehvglrERixTeHtXuzepAtgAuPiKRNz84Q,10981
 fractal_server/app/routes/api/v2/workflowtask.py,sha256=ciHTwXXFiFnMF7ZpJ3Xs0q6YfuZrFvIjqndlzAEdZpo,6969
 fractal_server/app/routes/auth/__init__.py,sha256=fao6CS0WiAjHDTvBzgBVV_bSXFpEAeDBF6Z6q7rRkPc,1658
 fractal_server/app/routes/auth/_aux_auth.py,sha256=ifkNocTYatBSMYGwiR14qohmvR9SfMldceiEj6uJBrU,4783
@@ -84,7 +84,7 @@ fractal_server/app/runner/executors/slurm/remote.py,sha256=wLziIsGdSMiO-jIXM8x77
 fractal_server/app/runner/executors/slurm/ssh/__init__.py,sha256=Cjn1rYvljddi96tAwS-qqGkNfOcfPzjChdaEZEObCcM,65
 fractal_server/app/runner/executors/slurm/ssh/_executor_wait_thread.py,sha256=bKo5Ja0IGxJWpPWyh9dN0AG-PwzTDZzD5LyaEHB3YU4,3742
 fractal_server/app/runner/executors/slurm/ssh/_slurm_job.py,sha256=rwlqZzoGo4SAb4nSlFjsQJdaCgfM1J6YGcjb8yYxlqc,4506
-fractal_server/app/runner/executors/slurm/ssh/executor.py,sha256=K1lKsn40PqQb-GZKwJkzeJk2Q8OGqy6YR-IxcV0E0Pw,57705
+fractal_server/app/runner/executors/slurm/ssh/executor.py,sha256=si_RHAMnXwQorQ_gWeZ_hQ_cNQbbAuYPjg7nwFQoPVg,58709
 fractal_server/app/runner/executors/slurm/sudo/__init__.py,sha256=Cjn1rYvljddi96tAwS-qqGkNfOcfPzjChdaEZEObCcM,65
 fractal_server/app/runner/executors/slurm/sudo/_check_jobs_status.py,sha256=wAgwpVcr6JIslKHOuS0FhRa_6T1KCManyRJqA-fifzw,1909
 fractal_server/app/runner/executors/slurm/sudo/_executor_wait_thread.py,sha256=z5LlhaiqAb8pHsF1WwdzXN39C5anQmwjo1rSQgtRAYE,4422
@@ -197,7 +197,7 @@ fractal_server/migrations/versions/efa89c30e0a4_add_project_timestamp_created.py
 fractal_server/migrations/versions/f384e1c0cf5d_drop_task_default_args_columns.py,sha256=9BwqUS9Gf7UW_KjrzHbtViC880qhD452KAytkHWWZyk,746
 fractal_server/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fractal_server/ssh/__init__.py,sha256=sVUmzxf7_DuXG1xoLQ1_00fo5NPhi2LJipSmU5EAkPs,124
-fractal_server/ssh/_fabric.py,sha256=Fw6NZl8YnlShYTL-jmuPpbJ_Kn2pDRAIRY3bbDcnMA8,16214
+fractal_server/ssh/_fabric.py,sha256=Pha-gRVUImj1cMsxulrJzaQa6Z60CmMYRAS4o22FcP0,19506
 fractal_server/string_tools.py,sha256=Z4qcleqXSG6RCG4hqS1emm0U-Bvv0sgTm_T87ZdYn7M,2395
 fractal_server/syringe.py,sha256=3qSMW3YaMKKnLdgnooAINOPxnCOxP7y2jeAQYB21Gdo,2786
 fractal_server/tasks/__init__.py,sha256=kadmVUoIghl8s190_Tt-8f-WBqMi8u8oU4Pvw39NHE8,23
@@ -211,7 +211,7 @@ fractal_server/tasks/v1/utils.py,sha256=J9oKys-82OehBxOon5wWl3CxjVBgYWeVEEyWGVFn
 fractal_server/tasks/v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fractal_server/tasks/v2/_venv_pip.py,sha256=FOD20yKfTsW7sim3h7CsB6pgp85JBhELyYvbkpaiDKA,6390
 fractal_server/tasks/v2/background_operations.py,sha256=jWmg_XkBmcXQfPKJu_eu0wtDL4sMp1QP2bIZ9QtMj_Y,15411
-fractal_server/tasks/v2/background_operations_ssh.py,sha256=RcMshcrj04r70hnyCB9aANk-8x8gr55AeP3v0gS7-Ts,13410
+fractal_server/tasks/v2/background_operations_ssh.py,sha256=W-w_a7FgrSc9FAVSXoVBzvXSCmH9oQkaDJx2S9hQPlc,13616
 fractal_server/tasks/v2/database_operations.py,sha256=6r56yyFPnEBrXl6ncmO6D76znzISQCFZqCYcD-Ummd4,1213
 fractal_server/tasks/v2/endpoint_operations.py,sha256=MtUoI0XWHuPSousDeH2IC2WU--AUKQVup6Q6AbHiNUA,4102
 fractal_server/tasks/v2/templates/_1_create_venv.sh,sha256=7tt-B6n8KRN-pannZ0enE6XSxyq-hKRYRGY63CvtINI,1151
@@ -223,8 +223,8 @@ fractal_server/tasks/v2/utils.py,sha256=MnY6MhcxDRo4rPuXo2tQ252eWEPZF3OlCGe-p5Mr
 fractal_server/urls.py,sha256=5o_qq7PzKKbwq12NHSQZDmDitn5RAOeQ4xufu-2v9Zk,448
 fractal_server/utils.py,sha256=jrlCBPmC7F0ptBVcDac-EbZNsdYTLbHfX9oxkXthS5Q,2193
 fractal_server/zip_tools.py,sha256=xYpzBshysD2nmxkD5WLYqMzPYUcCRM3kYy-7n9bJL-U,4426
-fractal_server-2.7.0a9.dist-info/LICENSE,sha256=QKAharUuhxL58kSoLizKJeZE3mTCBnX6ucmz8W0lxlk,1576
-fractal_server-2.7.0a9.dist-info/METADATA,sha256=UjnXL9Eo_UGayOQRJ6uRCuEijXJegpsQ-g3kGj64484,4630
-fractal_server-2.7.0a9.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-fractal_server-2.7.0a9.dist-info/entry_points.txt,sha256=8tV2kynvFkjnhbtDnxAqImL6HMVKsopgGfew0DOp5UY,58
-fractal_server-2.7.0a9.dist-info/RECORD,,
+fractal_server-2.7.0a11.dist-info/LICENSE,sha256=QKAharUuhxL58kSoLizKJeZE3mTCBnX6ucmz8W0lxlk,1576
+fractal_server-2.7.0a11.dist-info/METADATA,sha256=7vwgLiEeN-_a0vB2gnMmsPX_WMaa9BH3azyswEIT128,4631
+fractal_server-2.7.0a11.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+fractal_server-2.7.0a11.dist-info/entry_points.txt,sha256=8tV2kynvFkjnhbtDnxAqImL6HMVKsopgGfew0DOp5UY,58
+fractal_server-2.7.0a11.dist-info/RECORD,,