fractal-server 2.16.3__py3-none-any.whl → 2.16.4a0__py3-none-any.whl

fractal_server/{app/runner → runner}/v2/submit_workflow.py

@@ -11,26 +11,26 @@ from pathlib import Path
 
 from sqlalchemy.orm import Session as DBSyncSession
 
-from ....config import get_settings
-from ....logger import get_logger
-from ....logger import reset_logger_handlers
-from ....logger import set_logger
-from ....ssh._fabric import FractalSSH
-from ....syringe import Inject
-from ....utils import get_timestamp
-from ....zip_tools import _zip_folder_to_file_and_remove
-from ...db import DB
-from ...models.v2 import DatasetV2
-from ...models.v2 import JobV2
-from ...models.v2 import WorkflowV2
-from ...schemas.v2 import JobStatusTypeV2
+from ...config import get_settings
+from ...logger import get_logger
+from ...logger import reset_logger_handlers
+from ...logger import set_logger
+from ...ssh._fabric import FractalSSH
+from ...syringe import Inject
+from ...utils import get_timestamp
+from ...zip_tools import _zip_folder_to_file_and_remove
 from ..exceptions import JobExecutionError
 from ..filenames import WORKFLOW_LOG_FILENAME
 from ._local import process_workflow as local_process_workflow
 from ._slurm_ssh import process_workflow as slurm_ssh_process_workflow
 from ._slurm_sudo import process_workflow as slurm_sudo_process_workflow
 from fractal_server import __VERSION__
+from fractal_server.app.db import DB
 from fractal_server.app.models import UserSettings
+from fractal_server.app.models.v2 import DatasetV2
+from fractal_server.app.models.v2 import JobV2
+from fractal_server.app.models.v2 import WorkflowV2
+from fractal_server.app.schemas.v2 import JobStatusTypeV2
 
 
 _backends = {}

fractal_server/{app/runner → runner}/v2/task_interface.py

@@ -5,8 +5,8 @@ from pydantic import ConfigDict
 from pydantic import Field
 from pydantic import ValidationError
 
-from ....images import SingleImageTaskOutput
-from fractal_server.app.runner.exceptions import TaskOutputValidationError
+from fractal_server.images import SingleImageTaskOutput
+from fractal_server.runner.exceptions import TaskOutputValidationError
 from fractal_server.types import ZarrUrlStr
 
 

fractal_server/ssh/_fabric.py

@@ -1,5 +1,6 @@
 import json
 import logging
+import os
 import time
 from collections.abc import Generator
 from contextlib import contextmanager
@@ -15,11 +16,15 @@ from invoke import UnexpectedExit
 from paramiko.ssh_exception import NoValidConnectionsError
 from pydantic import BaseModel
 
+from ..logger import close_logger
 from ..logger import get_logger
 from ..logger import set_logger
 from fractal_server.string_tools import validate_cmd
 
 
+SSH_MONITORING_LOGGER_NAME = "ssh-log"
+
+
 class FractalSSHTimeoutError(RuntimeError):
     pass
 
@@ -42,9 +47,6 @@ class SSHConfig(BaseModel):
     key_path: str
 
 
-logger = set_logger(__name__)
-
-
 def retry_if_socket_error(func):
     @wraps(func)
     def func_with_retry(*args, **kwargs):
@@ -76,7 +78,8 @@ def _acquire_lock_with_timeout(
     lock: Lock,
     label: str,
     timeout: float,
-    logger_name: str = __name__,
+    pid: int,
+    logger_name: str,
 ) -> Generator[Literal[True], Any, None]:
     """
     Given a `threading.Lock` object, try to acquire it within a given timeout.
@@ -88,8 +91,9 @@ def _acquire_lock_with_timeout(
         logger_name:
     """
     logger = get_logger(logger_name)
+    ssh_logger = get_logger(SSH_MONITORING_LOGGER_NAME)
     logger.info(f"Trying to acquire lock for '{label}', with {timeout=}")
-    t_start_lock_acquire = time.perf_counter()
+    t_lock_request = time.perf_counter()
     result = lock.acquire(timeout=timeout)
     try:
         if not result:
@@ -98,14 +102,25 @@ def _acquire_lock_with_timeout(
                 f"Failed to acquire lock for '{label}' within "
                 f"{timeout} seconds"
             )
-        t_end_lock_acquire = time.perf_counter()
-        elapsed = t_end_lock_acquire - t_start_lock_acquire
+        t_lock_acquisition = time.perf_counter()
+        elapsed = t_lock_acquisition - t_lock_request
         logger.info(f"Lock for '{label}' was acquired - {elapsed=:.4f} s")
         yield result
     finally:
         if result:
             lock.release()
             logger.info(f"Lock for '{label}' was released.")
+            t_lock_release = time.perf_counter()
+            lock_was_acquired = 1
+        else:
+            t_lock_release = time.perf_counter()
+            t_lock_acquisition = t_lock_release
+            lock_was_acquired = 0
+        lock_waiting_time = t_lock_acquisition - t_lock_request
+        lock_holding_time = t_lock_release - t_lock_acquisition
+        ssh_logger.info(
+            f"{pid} {lock_waiting_time:.6e} {lock_holding_time:.6e} {lock_was_acquired} {label.replace(' ', '_')}"  # noqa
+        )
 
 
 class FractalSSH:
@@ -130,11 +145,12 @@ class FractalSSH:
     sftp_get_prefetch: bool
     sftp_get_max_requests: int
     logger_name: str
+    _pid: int
 
     def __init__(
         self,
         connection: Connection,
-        default_timeout: float = 250,
+        default_timeout: float = 500.0,
         sftp_get_prefetch: bool = False,
         sftp_get_max_requests: int = 64,
         logger_name: str = __name__,
@@ -146,6 +162,8 @@ class FractalSSH:
         self.sftp_get_max_requests = sftp_get_max_requests
         self.logger_name = logger_name
         set_logger(self.logger_name)
+        set_logger(SSH_MONITORING_LOGGER_NAME)
+        self._pid = os.getpid()
 
     @property
     def is_connected(self) -> bool:
@@ -197,6 +215,8 @@ class FractalSSH:
             lock=self._lock,
             label=label,
             timeout=actual_lock_timeout,
+            pid=self._pid,
+            logger_name=self.logger_name,
         ):
             return self._connection.run(*args, **kwargs)
 
@@ -212,8 +232,10 @@ class FractalSSH:
         self.logger.info(f"START reading remote JSON file {filepath}.")
         with _acquire_lock_with_timeout(
             lock=self._lock,
-            label="read_remote_json_file",
             timeout=self.default_lock_timeout,
+            logger_name=self.logger_name,
+            pid=self._pid,
+            label=f"read_remote_json_file({filepath})",
         ):
             try:
                 with self._sftp_unsafe().open(filepath, "r") as f:
@@ -239,8 +261,10 @@ class FractalSSH:
         self.logger.info(f"START reading remote text file {filepath}.")
         with _acquire_lock_with_timeout(
             lock=self._lock,
-            label="read_remote_text_file",
+            label=f"read_remote_text_file({filepath})",
             timeout=self.default_lock_timeout,
+            pid=self._pid,
+            logger_name=self.logger_name,
         ):
             try:
                 with self._sftp_unsafe().open(filepath, "r") as f:
@@ -298,9 +322,10 @@ class FractalSSH:
             self.close()
             with _acquire_lock_with_timeout(
                 lock=self._lock,
-                label="_connection.open",
+                label="FractalSSH._connection.{open,open_sftp}()",
                 timeout=self.default_lock_timeout,
                 logger_name=self.logger_name,
+                pid=self._pid,
             ):
                 self._connection.open()
                 self._connection.client.open_sftp()
@@ -324,12 +349,16 @@ class FractalSSH:
         """
         with _acquire_lock_with_timeout(
             lock=self._lock,
-            label="_connection.close",
+            label="FractalSSH._connection.close()",
             timeout=self.default_lock_timeout,
+            pid=self._pid,
+            logger_name=self.logger_name,
         ):
             self._connection.close()
             if self._connection.client is not None:
                 self._connection.client.close()
+        close_logger(get_logger(self.logger_name))
+        close_logger(get_logger(SSH_MONITORING_LOGGER_NAME))
 
     @retry_if_socket_error
     def run_command(
@@ -364,14 +393,14 @@ class FractalSSH:
             # Case 1: Command runs successfully
             res = self._run(
                 cmd,
-                label=f"run {cmd}",
+                label=cmd,
                 lock_timeout=actual_lock_timeout,
                 hide=True,
                 in_stream=False,
             )
             t_1 = time.perf_counter()
             self.logger.info(
-                f"END   running '{cmd}' over SSH, " f"elapsed={t_1 - t_0:.3f}"
+                f"END   running '{cmd}' over SSH, elapsed={t_1 - t_0:.3f}"
             )
             self.logger.debug("STDOUT:")
             self.logger.debug(res.stdout)
@@ -423,8 +452,10 @@ class FractalSSH:
                 actual_lock_timeout = lock_timeout
             with _acquire_lock_with_timeout(
                 lock=self._lock,
-                label=f"send_file {local=} {remote=}",
+                label=f"send_file({local},{remote})",
                 timeout=actual_lock_timeout,
+                pid=self._pid,
+                logger_name=self.logger_name,
             ):
                 self._sftp_unsafe().put(local, remote)
             self.logger.info(
@@ -463,8 +494,10 @@ class FractalSSH:
                 actual_lock_timeout = lock_timeout
             with _acquire_lock_with_timeout(
                 lock=self._lock,
-                label=f"fetch_file {local=} {remote=}",
+                label=f"fetch_file({local},{remote})",
                 timeout=actual_lock_timeout,
+                pid=self._pid,
+                logger_name=self.logger_name,
             ):
                 self._sftp_unsafe().get(
                     remote,
@@ -554,8 +587,10 @@ class FractalSSH:
             actual_lock_timeout = lock_timeout
         with _acquire_lock_with_timeout(
             lock=self._lock,
-            label=f"write_remote_file {path=}",
+            label=f"write_remote_file({path})",
             timeout=actual_lock_timeout,
+            pid=self._pid,
+            logger_name=self.logger_name,
         ):
             try:
                 with self._sftp_unsafe().open(filename=path, mode="w") as f:
@@ -576,8 +611,10 @@ class FractalSSH:
         self.logger.info(f"START remote_file_exists {path}")
         with _acquire_lock_with_timeout(
             lock=self._lock,
-            label=f"remote_file_exists {path=}",
+            label=f"remote_file_exists({path})",
             timeout=self.default_lock_timeout,
+            pid=self._pid,
+            logger_name=self.logger_name,
         ):
             try:
                 self._sftp_unsafe().stat(path)
@@ -613,6 +650,7 @@ class FractalSSHList:
     _lock: Lock
     _timeout: float
     _logger_name: str
+    _pid: int
 
     def __init__(
         self,
@@ -625,6 +663,7 @@ class FractalSSHList:
         self._timeout = timeout
         self._logger_name = logger_name
         set_logger(self._logger_name)
+        self._pid = os.getpid()
 
     @property
     def logger(self) -> logging.Logger:
@@ -677,6 +716,8 @@ class FractalSSHList:
                 lock=self._lock,
                 label="FractalSSHList.get",
                 timeout=self._timeout,
+                pid=self._pid,
+                logger_name=self._logger_name,
             ):
                 self._data[key] = FractalSSH(connection=connection)
                 return self._data[key]
@@ -721,6 +762,8 @@ class FractalSSHList:
             lock=self._lock,
             timeout=self._timeout,
             label="FractalSSHList.remove",
+            pid=self._pid,
+            logger_name=self._logger_name,
         ):
             self.logger.info(
                 f"Removing FractalSSH object for {user}@{host} "

{fractal_server-2.16.3.dist-info → fractal_server-2.16.4a0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fractal-server
-Version: 2.16.3
+Version: 2.16.4a0
 Summary: Backend component of the Fractal analytics platform
 License: BSD-3-Clause
 Author: Tommaso Comparin
@@ -14,7 +14,7 @@ Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: alembic (>=1.13.1,<2.0.0)
 Requires-Dist: cryptography (>=46.0.0,<47.0.0)
 Requires-Dist: fabric (>=3.2.2,<3.3.0)
-Requires-Dist: fastapi (>=0.116.0,<0.117.0)
+Requires-Dist: fastapi (>=0.117.0,<0.118.0)
 Requires-Dist: fastapi-users[oauth] (>=14,<15)
 Requires-Dist: gunicorn (>=23.0,<24.0)
 Requires-Dist: packaging (>=25.0.0,<26.0.0)
@@ -23,10 +23,10 @@ Requires-Dist: pydantic (>=2.11.0,<2.12.0)
 Requires-Dist: pydantic-settings (>=2.7.0)
 Requires-Dist: python-dotenv (>=1.1.0,<1.2.0)
 Requires-Dist: sqlalchemy[asyncio] (>=2.0.23,<2.1)
-Requires-Dist: sqlmodel (==0.0.24)
+Requires-Dist: sqlmodel (==0.0.25)
 Requires-Dist: tomli_w (>=1.2.0,<1.3.0)
-Requires-Dist: uvicorn (>=0.29.0,<0.35.0)
-Requires-Dist: uvicorn-worker (==0.3.0)
+Requires-Dist: uvicorn (>=0.29.0,<=0.36.0)
+Requires-Dist: uvicorn-worker (==0.4.0)
 Project-URL: Documentation, https://fractal-analytics-platform.github.io/fractal-server
 Project-URL: Homepage, https://github.com/fractal-analytics-platform/fractal-server
 Project-URL: Repository, https://github.com/fractal-analytics-platform/fractal-server

{fractal_server-2.16.3.dist-info → fractal_server-2.16.4a0.dist-info}/RECORD

@@ -1,4 +1,4 @@
-fractal_server/__init__.py,sha256=XhckXI3NQgXzQFmbqjMIwxfxShIlVm8JhcoaOhWWvyY,23
+fractal_server/__init__.py,sha256=l0jh4TNuWUFSZSrc8IGGsg4R9J-yut5-a5mbjgOpznE,25
 fractal_server/__main__.py,sha256=rkM8xjY1KeS3l63irB8yCrlVobR-73uDapC4wvrIlxI,6957
 fractal_server/alembic.ini,sha256=MWwi7GzjzawI9cCAK1LW7NxIBQDUqD12-ptJoq5JpP0,3153
 fractal_server/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -23,91 +23,51 @@ fractal_server/app/routes/admin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm
 fractal_server/app/routes/admin/v2/__init__.py,sha256=_5lqb6-M8-fZqE1HRMep6pAFYRUKMxrvbZOKs-RXWkw,933
 fractal_server/app/routes/admin/v2/accounting.py,sha256=-eLzBirENvdhjhrCf6bs1pTriIui19DAr_k-zuJB7Y8,3593
 fractal_server/app/routes/admin/v2/impersonate.py,sha256=gc4lshfEPFR6W2asH7aKu6hqE6chzusdhAUVV9p51eU,1131
-fractal_server/app/routes/admin/v2/job.py,sha256=NJwAecyiQDRwLEqg1fKQsllxW1FdtbwnLJQHocBQinU,8597
+fractal_server/app/routes/admin/v2/job.py,sha256=6NG93I-yQTlJZ1AIhMPwuizQcQvFlvI7kGJ8jG94Igg,8595
 fractal_server/app/routes/admin/v2/project.py,sha256=MA_LdoEuSuisSGRO43TapMuJ080y5iaUGSAUgKuuKOg,1188
-fractal_server/app/routes/admin/v2/task.py,sha256=8njjq_zcvNW-Ewxn7WfsRQbs_aV5h-7pgcRIAegzTnc,4308
-fractal_server/app/routes/admin/v2/task_group.py,sha256=KYMBqpQ2ysjP2Nt0sLvSEpL21IzBpcliuuB48U4N8q4,6042
-fractal_server/app/routes/admin/v2/task_group_lifecycle.py,sha256=OOmSI_VfmLrsYrkmm-_z_OJHAnwThCUrG_gJt7ca5ZM,11591
+fractal_server/app/routes/admin/v2/task.py,sha256=e1UxsA6CWeYqvnHqxySE78ckRuE0EK60X02VMnkG2gg,4309
+fractal_server/app/routes/admin/v2/task_group.py,sha256=7-Axk5SG6Nw02p7pc6Q6_EtO9Ncy74pVXvt2GCG-Iuc,6043
+fractal_server/app/routes/admin/v2/task_group_lifecycle.py,sha256=frIW3aNQxK_xLRTCfet7249fxi3WF7uNjv6ApGrverY,11594
 fractal_server/app/routes/api/__init__.py,sha256=B8l6PSAhR10iZqHEiyTat-_0tkeKdrCigIE6DJGP5b8,638
 fractal_server/app/routes/api/v2/__init__.py,sha256=D3sRRsqkmZO6kBxUjg40q0aRDsnuXI4sOOfn0xF9JsM,2820
-fractal_server/app/routes/api/v2/_aux_functions.py,sha256=tfnxqbufKqd5dbFUzOFlRPc9EgURhpNa5V2V4agzndg,14259
-fractal_server/app/routes/api/v2/_aux_functions_history.py,sha256=ykTyiY3H-nGSgGfcVkb2xQqJd65YOY2aWVK9sfqTdIY,4439
-fractal_server/app/routes/api/v2/_aux_functions_task_lifecycle.py,sha256=bcDsc9ZBVkVkPwHyGpzmQknMAh1hNbdZe_rg6VWiFLE,7430
+fractal_server/app/routes/api/v2/_aux_functions.py,sha256=nPmSHRlFoTGDoP95J2KHnhtzxH5Ry2_h2fo9QHugo9c,14266
+fractal_server/app/routes/api/v2/_aux_functions_history.py,sha256=TOdUJNsHaAzG0FP0sCylyBUPtz5ml_68fEW4kR-cmkg,4441
+fractal_server/app/routes/api/v2/_aux_functions_task_lifecycle.py,sha256=bguVrV_8cleRGqmMBhaGlTZ2iKtwGrVSzN-6dDDf05g,8600
 fractal_server/app/routes/api/v2/_aux_functions_task_version_update.py,sha256=PKjV7r8YsPRXoNiVSnOK4KBYVV3l_Yb_ZPrqAkMkXrQ,1182
-fractal_server/app/routes/api/v2/_aux_functions_tasks.py,sha256=VuFaucsAcsb5M54mVlp4sqjt2REiMIDHn7bSZjedqrg,12495
+fractal_server/app/routes/api/v2/_aux_functions_tasks.py,sha256=QlFrW2RGoYU-cllyOu2H_5ZAjA6neIa8pWZnq7g1UNc,12503
 fractal_server/app/routes/api/v2/_aux_task_group_disambiguation.py,sha256=8x1_q9FyCzItnPmdSdLQuwUTy4B9xCsXscp97_lJcpM,4635
-fractal_server/app/routes/api/v2/dataset.py,sha256=6u4MFqJ3YZ0Zq6Xx8CRMrTPKW55ZaR63Uno21DqFr4Q,8889
-fractal_server/app/routes/api/v2/history.py,sha256=EUD_Gdqzj1GwcB6cKqwP2ULxmRer-TqcAu4sPylIx4E,17100
-fractal_server/app/routes/api/v2/images.py,sha256=amQz2uoaeSSuUeYfjfQ_zyEak6h_F7KSgI5RM81cqDI,7864
-fractal_server/app/routes/api/v2/job.py,sha256=8xRTwh_OCHmK9IfI_zUASa2ozewR0qu0zVBl_a4IvHw,6467
+fractal_server/app/routes/api/v2/dataset.py,sha256=7hSqqE3sFjQjJOZf_QvYbMersjXXbS3YM3a1WDgv1rM,8893
+fractal_server/app/routes/api/v2/history.py,sha256=kEhD-TKkORenUUsfyODNBi1jOsGc82h_EWbRKyRkD5k,17102
+fractal_server/app/routes/api/v2/images.py,sha256=foA5ZNjMzZWlcYMgYuGSZsm1I-po-8NSYBIMz4ZbkPc,7867
+fractal_server/app/routes/api/v2/job.py,sha256=FyK4Aa9XoIcfD0ZpKM-4Bkj3ZcIc93UzDc8jF85uHps,6478
 fractal_server/app/routes/api/v2/pre_submission_checks.py,sha256=AEN6w2X5kCkSpZydudvuTPhl_VZUQ4zUav6D3NiF6r8,4935
-fractal_server/app/routes/api/v2/project.py,sha256=ldMEyjtwGpX2teu85sCNWaubDFlw-En8U1SA7G1VaIw,4567
-fractal_server/app/routes/api/v2/status_legacy.py,sha256=ZckHeBy8y21cyQ_OLY-VmkapzMhd3g9ae-qg-r4-uVo,6317
-fractal_server/app/routes/api/v2/submit.py,sha256=C9cLecHoVl_VXQFAHs2Y2LVE6S_v2PHpa-w5XQIaT-M,9174
-fractal_server/app/routes/api/v2/task.py,sha256=WxkWRw-0fJkgmMt9fSDc6oikBFd6nSS-JdPwR0z24tg,7024
-fractal_server/app/routes/api/v2/task_collection.py,sha256=Phb_ORpJ9J8oa1c75qcThJjOWSpv66qfwoePpoUQ_no,12738
-fractal_server/app/routes/api/v2/task_collection_custom.py,sha256=j-vI1_FjoxU59VwkwUTYW5HhMVB5NObMkWLCjAKbers,6726
-fractal_server/app/routes/api/v2/task_collection_pixi.py,sha256=1rCBfKANT9zcKcOVMQTByVpP8GjdrzeaMzyQT05k0nY,7517
+fractal_server/app/routes/api/v2/project.py,sha256=j7SHe38IKMYBW5dTDUgnWn0alxDUeCFL1eJmgoCxvcc,4568
+fractal_server/app/routes/api/v2/status_legacy.py,sha256=GG-AUdAngFL1QpE8Alo3xPxyKC4csG7DbomYlIfqTEs,6318
+fractal_server/app/routes/api/v2/submit.py,sha256=xcoAiLTtns9yGQmobVyqT8GkvwFN_VEn5sSvvSJ9SZ4,9171
+fractal_server/app/routes/api/v2/task.py,sha256=CAXWU4Xo0GdyMbUr4NfTIZvvJ5wjeKbmEQDHLoTwc-c,7028
+fractal_server/app/routes/api/v2/task_collection.py,sha256=5l0O5AgGWAZicpt-gGoPOp-krXn2v_6R9MvwbT3RV1w,12743
+fractal_server/app/routes/api/v2/task_collection_custom.py,sha256=-jpAjyALdZrwzrWQV5-vqyjSCI1kXbbfnTMgzp9P_cs,6732
+fractal_server/app/routes/api/v2/task_collection_pixi.py,sha256=aHbfisZ8ulQ6p1OCerWY-nOERqGybTfRyQKPcj-R_FE,7522
 fractal_server/app/routes/api/v2/task_group.py,sha256=An8EAwZDNx53TVnGBhjgeW7RI_uk-bAlu-6NrqYuaWY,8068
-fractal_server/app/routes/api/v2/task_group_lifecycle.py,sha256=V_U0v1FH9Aljksthc-tf7CpRmlyu6f2U3GURPNgIi2k,12286
-fractal_server/app/routes/api/v2/task_version_update.py,sha256=RoeN21kYvBwc8xLcJZ8t0hZtl32P3FhMqiofntHyrjQ,8233
-fractal_server/app/routes/api/v2/workflow.py,sha256=n0sRsDHGrjDAoo_0klqoYW4AuZ1LajWtlzEd6gKmvps,10697
-fractal_server/app/routes/api/v2/workflow_import.py,sha256=kOGDaCj0jCGK1WSYGbnUjtUg2U1YxUY9UMH-2ilqJg4,9027
-fractal_server/app/routes/api/v2/workflowtask.py,sha256=5_SQAG8ztDnaaRXwKalcO69HVpSl-QbrhiI7fCP3YRI,7924
+fractal_server/app/routes/api/v2/task_group_lifecycle.py,sha256=DDwFy9TUchUkRTigPSlA6biZZY0wbb8NAZtdo_XYO7E,12289
+fractal_server/app/routes/api/v2/task_version_update.py,sha256=zBYlyqN0UvCfnkqcg945b5jNMRNjSUKJvBz6IPxC0qo,8236
+fractal_server/app/routes/api/v2/workflow.py,sha256=iTIasNP2I5ovhHafQ2P8JJuImAl41ucy5y64J7FiP3w,10701
+fractal_server/app/routes/api/v2/workflow_import.py,sha256=GUEl8HIFcDZ2XaA93t4W6PYC3ALX2XnJGBHH9Yr8q50,9028
+fractal_server/app/routes/api/v2/workflowtask.py,sha256=gwFvC-qU4yb-Q5ey30rOD4u3D0-4jlJzBTuq8udqpzc,7930
 fractal_server/app/routes/auth/__init__.py,sha256=fao6CS0WiAjHDTvBzgBVV_bSXFpEAeDBF6Z6q7rRkPc,1658
 fractal_server/app/routes/auth/_aux_auth.py,sha256=UZgauY0V6mSqjte_sYI1cBl2h8bcbLaeWzgpl1jdJlk,4883
 fractal_server/app/routes/auth/current_user.py,sha256=EjkwMxUA0l6FLbDJdertHGnuOoSS-HEysmm6l5FkAlY,5903
-fractal_server/app/routes/auth/group.py,sha256=VBeV5PbSYkyevvg2USrMPHR2sD1DUp8IsDyhC2Gb7vI,7888
+fractal_server/app/routes/auth/group.py,sha256=gSyB9iuwCqT6CMDHyO8hYIQ1J341gs8SDrRdHppOMT0,7890
 fractal_server/app/routes/auth/login.py,sha256=tSu6OBLOieoBtMZB4JkBAdEgH2Y8KqPGSbwy7NIypIo,566
 fractal_server/app/routes/auth/oauth.py,sha256=KCtJHSzemC4S8AfX9bLLdVhlF1nU4DOyox-sNQtcWew,1978
 fractal_server/app/routes/auth/register.py,sha256=DlHq79iOvGd_gt2v9uwtsqIKeO6i_GKaW59VIkllPqY,587
 fractal_server/app/routes/auth/router.py,sha256=tzJrygXFZlmV_uWelVqTOJMEH-3Fr7ydwlgx1LxRjxY,527
-fractal_server/app/routes/auth/users.py,sha256=Zr1Bsa7Hpricb_1uFwKPCtgt3PzGnP0TaMLMdpbQDNs,7825
+fractal_server/app/routes/auth/users.py,sha256=ZZLqkjuBISU3GoUg3WJtrHvDEy7xEG7B5IumM3dHUQ4,7826
 fractal_server/app/routes/aux/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fractal_server/app/routes/aux/_job.py,sha256=XWyWpOObcV55YyK7uzGRlaslmPDCBZy4hiSZBpoa_bg,616
-fractal_server/app/routes/aux/_runner.py,sha256=spNudutueHTBJPhm55RlOuYzb31DhyheSjl2rk6dloM,873
-fractal_server/app/routes/aux/validate_user_settings.py,sha256=FLVi__8YFcm_6c_K5uMQo7raWWXQLBcZtx8yaPO4jaE,2301
+fractal_server/app/routes/aux/_job.py,sha256=nqqdcW5B7fL_PbvHf57_TcifjUfcMgl04tKNvG2sV1U,628
+fractal_server/app/routes/aux/_runner.py,sha256=9wgW--pAitRwR0zEsOrADtI9zi_DYT7W-cvEfKjKYQE,875
+fractal_server/app/routes/aux/validate_user_settings.py,sha256=J6VjuC-qIzHsYiLextCi5lX8Q2z3g2_JOSpIFiuMJSc,2303
 fractal_server/app/routes/pagination.py,sha256=IGy8Ll5lYr6ENYE18h3huH5s0GMX1DCs5VdCi6j1cdk,1174
-fractal_server/app/runner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fractal_server/app/runner/components.py,sha256=-Ii5l8d_V6f5DFOd-Zsr8VYmOsyqw0Hox9fEFQiuqxY,66
-fractal_server/app/runner/exceptions.py,sha256=lzYefT6IpA2ajcQUzm9kT9pTxRl5UGIJKyLs6v1JczA,1731
-fractal_server/app/runner/executors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fractal_server/app/runner/executors/base_runner.py,sha256=bznvlTrRPk_-ER9DrG8iRQI6bqKM66_uMd7VV5mIUhg,6069
-fractal_server/app/runner/executors/call_command_wrapper.py,sha256=1BHl-zbXoX2oGUWGAFprVZMmg5QjutPH0-VZJSIC0II,1419
-fractal_server/app/runner/executors/local/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fractal_server/app/runner/executors/local/get_local_config.py,sha256=KBYOkcuwpSYl-ZIAwPBxpn59QSyFF8eJ-fLKVIhwwzA,3594
-fractal_server/app/runner/executors/local/runner.py,sha256=syJcUYkdKeGYbD33VzuX9Ysr7OjK34TkZjB7xvkzqXU,10838
-fractal_server/app/runner/executors/slurm_common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fractal_server/app/runner/executors/slurm_common/_batching.py,sha256=gbHZIxt90GjUwhB9_UInwVqpX-KdxRQMDeXzUagdL3U,8816
-fractal_server/app/runner/executors/slurm_common/_job_states.py,sha256=nuV-Zba38kDrRESOVB3gaGbrSPZc4q7YGichQaeqTW0,238
-fractal_server/app/runner/executors/slurm_common/_slurm_config.py,sha256=FI2p4yQ16571C_YM3Vs_tJnb71urVykwxfIXDXa79I8,15721
-fractal_server/app/runner/executors/slurm_common/base_slurm_runner.py,sha256=7pI46LpmnbBzNKxsK0YC0Uv5YfyNwVYWA3dMuAoa_Lg,40020
-fractal_server/app/runner/executors/slurm_common/get_slurm_config.py,sha256=KhQQJrGQZLX5u8fsMcIx7FN8jUKSGEeG68yO7Ay_LXg,7454
-fractal_server/app/runner/executors/slurm_common/remote.py,sha256=LHK2Ram8X8q6jNSCxnnwKUwmSJMsyQyRem_VjH53qdw,3811
-fractal_server/app/runner/executors/slurm_common/slurm_job_task_models.py,sha256=K4SdJOKsUWzDlnkb8Ug_UmTx6nBMsTqn9_oKqwE4XDI,3520
-fractal_server/app/runner/executors/slurm_ssh/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fractal_server/app/runner/executors/slurm_ssh/run_subprocess.py,sha256=SyW6t4egvbiARph2YkFjc88Hj94fCamZVi50L7ph8VM,996
-fractal_server/app/runner/executors/slurm_ssh/runner.py,sha256=3YEE3_CKQWqrgINA7LLn5Xnq510clskTP2KDSTc2orw,10098
-fractal_server/app/runner/executors/slurm_ssh/tar_commands.py,sha256=yqBjWTLmh_FzhCllt_KfbuCUGvRhsHLVWlPOZlRWLzY,1762
-fractal_server/app/runner/executors/slurm_sudo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fractal_server/app/runner/executors/slurm_sudo/_subprocess_run_as_user.py,sha256=W-FxnVcHxMGpv4zGgJVttVQoweyGgR4uBxO22QIZkp0,2576
-fractal_server/app/runner/executors/slurm_sudo/runner.py,sha256=pb2otlEQIHT9qaN-q0qlUq_BJG4-ozk-P2-C9drHbDU,5862
-fractal_server/app/runner/filenames.py,sha256=lPnxKHtdRizr6FqG3zOdjDPyWA7GoaJGTtiuJV0gA8E,70
-fractal_server/app/runner/set_start_and_last_task_index.py,sha256=NsioSzfEpGyo9ZKrV5KsbxeI7d5V3tE678Y3IAo5rHM,1218
-fractal_server/app/runner/shutdown.py,sha256=ViSNJyXWU_iWPSDOOMGNh_iQdUFrdPh_jvf8vVKLpAo,1950
-fractal_server/app/runner/task_files.py,sha256=V_7aZhu6-c6Y-0XUe-5cZVDrdnXEJhp8pQoUMtx6ko0,4041
-fractal_server/app/runner/v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fractal_server/app/runner/v2/_local.py,sha256=tTJgABK-zAZmmRzoie_MPNTXJx_zBAXiZiiWl1CC2qo,3035
-fractal_server/app/runner/v2/_slurm_ssh.py,sha256=JlDngsVSOUNqEubDl-5jkIxQQmV4mhTqbMbJVY_rL6M,2840
-fractal_server/app/runner/v2/_slurm_sudo.py,sha256=Gvsh4tUlc1_3KdF3B7zEqs-YIntC_joLtTGSNFbKKSs,2939
-fractal_server/app/runner/v2/db_tools.py,sha256=ozp4RFLB3LNI0rM0q0udi6ja8-5vooH_dVqrbmTPNDg,3323
-fractal_server/app/runner/v2/deduplicate_list.py,sha256=IVTE4abBU1bUprFTkxrTfYKnvkNTanWQ-KWh_etiT08,645
-fractal_server/app/runner/v2/merge_outputs.py,sha256=YOTKbGOM9s-uqY4KN2onoIxuHNm-v3hr5zv6Aa1KEtA,905
-fractal_server/app/runner/v2/runner.py,sha256=q-8StMNKqMyrE1FraI-t3zZhsqtJwByE9rJGR22JRlQ,19557
-fractal_server/app/runner/v2/runner_functions.py,sha256=xteoDSXKxStl3ABEXyjrggPiXdXAPy9sJPfT3B8CQ3Y,19050
-fractal_server/app/runner/v2/submit_workflow.py,sha256=P4zP2ISOcTxv4nYOTAgVpPyq506DPw9ebeSISkvuXVY,12276
-fractal_server/app/runner/v2/task_interface.py,sha256=BRSKpitGproY48JQdCbfrghbDonA-EqPP1yIopohpPo,2525
-fractal_server/app/runner/versions.py,sha256=4BW-8Et8RVgILgpFoUJLWkEnZz53pv8hv_2ucG480ns,398
 fractal_server/app/schemas/__init__.py,sha256=stURAU_t3AOBaH0HSUbV-GKhlPKngnnIMoqWc3orFyI,135
 fractal_server/app/schemas/user.py,sha256=t9nbyYjGCSOsxm9K97PDG3-9o27CsaFfhWb_L5nrjqA,1910
 fractal_server/app/schemas/user_group.py,sha256=x3-kqbo0q2wTP7QI0iZ7PU_9Dr957UYrFMKqS7BXLhE,1425
@@ -129,7 +89,7 @@ fractal_server/app/schemas/v2/workflowtask.py,sha256=6eweAMyziwaoMT-7R1fVJYunIeZ
 fractal_server/app/security/__init__.py,sha256=oJ8RVglpOvWPQY4RokiE2YA72Nqo42dZEjywWTt8xr8,14032
 fractal_server/app/security/signup_email.py,sha256=Xd6QYxcdmg0PHpDwmUE8XQmPcOj3Xjy5oROcIMhmltM,1472
 fractal_server/app/user_settings.py,sha256=OP1yiYKtPadxwM51_Q0hdPk3z90TCN4z1BLpQsXyWiU,1316
-fractal_server/config.py,sha256=q0f1DsSgXKE9gyzty7y4NGiQyttV6H13LUMPZsOeGH8,30896
+fractal_server/config.py,sha256=UxGXo0X0AdVRvWKGJbDkiuqC7yoAM6vF2dWp2GUkQBo,30888
 fractal_server/data_migrations/2_14_10.py,sha256=jzMg2c1zNO8C_Nho_9_EZJD6kR1-gkFNpNrMR5Hr8hM,1598
 fractal_server/data_migrations/README.md,sha256=_3AEFvDg9YkybDqCLlFPdDmGJvr6Tw7HRI14aZ3LOIw,398
 fractal_server/data_migrations/tools.py,sha256=LeMeASwYGtEqd-3wOLle6WARdTGAimoyMmRbbJl-hAM,572
@@ -140,7 +100,7 @@ fractal_server/images/models.py,sha256=6WchcIzLLLwdkLNRfg71Dl4Y-9UFLPyrrzh1lWgju
 fractal_server/images/status_tools.py,sha256=KICcThwHniHNkDftzJmK_n2gQK2IeTKlR_PFAesf204,4912
 fractal_server/images/tools.py,sha256=nhkp_7bBI6pmJpqbg1s7xjoZKW5S4aNInkym5ntvJYA,4169
 fractal_server/logger.py,sha256=QIeVn3QpZsiIL2jDdrKotr-MLyDcZYgiPiTluFU46lE,5317
-fractal_server/main.py,sha256=0lh7JQGt7e6wekLHzhiSWURKzkE8GeRz6lhlCWjF8Ho,4560
+fractal_server/main.py,sha256=bq-G4HDpLz-y97x4LCh9c0ZA8AQhFRpeJqwQiPfXQMM,4556
 fractal_server/migrations/env.py,sha256=nfyBpMIOT3kny6t-b-tUjyRjZ4k906bb1_wCQ7me1BI,1353
 fractal_server/migrations/naming_convention.py,sha256=htbKrVdetx3pklowb_9Cdo5RqeF0fJ740DNecY5de_M,265
 fractal_server/migrations/versions/034a469ec2eb_task_groups.py,sha256=vrPhC8hfFu1c4HmLHNZyCuqEfecFD8-bWc49bXMNes0,6199
@@ -187,8 +147,48 @@ fractal_server/migrations/versions/f37aceb45062_make_historyunit_logfile_require
 fractal_server/migrations/versions/f384e1c0cf5d_drop_task_default_args_columns.py,sha256=9BwqUS9Gf7UW_KjrzHbtViC880qhD452KAytkHWWZyk,746
 fractal_server/migrations/versions/fbce16ff4e47_new_history_items.py,sha256=TDWCaIoM0Q4SpRWmR9zr_rdp3lJXhCfBPTMhtrP5xYE,3950
 fractal_server/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fractal_server/runner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fractal_server/runner/components.py,sha256=-Ii5l8d_V6f5DFOd-Zsr8VYmOsyqw0Hox9fEFQiuqxY,66
+fractal_server/runner/exceptions.py,sha256=lzYefT6IpA2ajcQUzm9kT9pTxRl5UGIJKyLs6v1JczA,1731
+fractal_server/runner/executors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fractal_server/runner/executors/base_runner.py,sha256=8Fy5_EG-Y_DBZrRlulkk-3hCDYYSVNuEXYtb-VGZrMM,6065
+fractal_server/runner/executors/call_command_wrapper.py,sha256=n6c9haKyIudBlz9-d3GQb19rewBR54qLG8jlrma6okk,1415
+fractal_server/runner/executors/local/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fractal_server/runner/executors/local/get_local_config.py,sha256=KBYOkcuwpSYl-ZIAwPBxpn59QSyFF8eJ-fLKVIhwwzA,3594
+fractal_server/runner/executors/local/runner.py,sha256=G5zLGrw4KkNR__XIX0QGoBzLPHEF7PULeWpuWyg00UY,10810
+fractal_server/runner/executors/slurm_common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fractal_server/runner/executors/slurm_common/_batching.py,sha256=gbHZIxt90GjUwhB9_UInwVqpX-KdxRQMDeXzUagdL3U,8816
+fractal_server/runner/executors/slurm_common/_job_states.py,sha256=nuV-Zba38kDrRESOVB3gaGbrSPZc4q7YGichQaeqTW0,238
+fractal_server/runner/executors/slurm_common/_slurm_config.py,sha256=Su8uQB_DSG8kZbto4s1Tyc4uLCGY64wKb4RCoUEJ6Uc,15717
+fractal_server/runner/executors/slurm_common/base_slurm_runner.py,sha256=6aYiabn0r1J_FAhW-7id7LFby9REVaoVoMTNBdRcDac,39976
+fractal_server/runner/executors/slurm_common/get_slurm_config.py,sha256=KhQQJrGQZLX5u8fsMcIx7FN8jUKSGEeG68yO7Ay_LXg,7454
+fractal_server/runner/executors/slurm_common/remote.py,sha256=LHK2Ram8X8q6jNSCxnnwKUwmSJMsyQyRem_VjH53qdw,3811
+fractal_server/runner/executors/slurm_common/slurm_job_task_models.py,sha256=DWOspO6k5C5FUVdE8t_qeb2G-xgD7h2yyFcpasjrtmE,3516
+fractal_server/runner/executors/slurm_ssh/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fractal_server/runner/executors/slurm_ssh/run_subprocess.py,sha256=SyW6t4egvbiARph2YkFjc88Hj94fCamZVi50L7ph8VM,996
+fractal_server/runner/executors/slurm_ssh/runner.py,sha256=3YEE3_CKQWqrgINA7LLn5Xnq510clskTP2KDSTc2orw,10098
+fractal_server/runner/executors/slurm_ssh/tar_commands.py,sha256=yqBjWTLmh_FzhCllt_KfbuCUGvRhsHLVWlPOZlRWLzY,1762
+fractal_server/runner/executors/slurm_sudo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fractal_server/runner/executors/slurm_sudo/_subprocess_run_as_user.py,sha256=W-FxnVcHxMGpv4zGgJVttVQoweyGgR4uBxO22QIZkp0,2576
+fractal_server/runner/executors/slurm_sudo/runner.py,sha256=YDR4z5KuMTxjW0UcK6N5ntDqasrG8lVs3KVu3SGs6F8,5858
+fractal_server/runner/filenames.py,sha256=lPnxKHtdRizr6FqG3zOdjDPyWA7GoaJGTtiuJV0gA8E,70
+fractal_server/runner/set_start_and_last_task_index.py,sha256=NsioSzfEpGyo9ZKrV5KsbxeI7d5V3tE678Y3IAo5rHM,1218
+fractal_server/runner/shutdown.py,sha256=ViSNJyXWU_iWPSDOOMGNh_iQdUFrdPh_jvf8vVKLpAo,1950
+fractal_server/runner/task_files.py,sha256=MvOrklZrN8iLj3oPx7G-iaEFj2N_mkTOnxIBX-okH1E,4037
+fractal_server/runner/v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fractal_server/runner/v2/_local.py,sha256=F6gTF1r36O5MxXQ2tUkifYk9GHyPbeUB5N4TSu0tsBw,3067
+fractal_server/runner/v2/_slurm_ssh.py,sha256=Pu4WO9uBMUOmAo1XsG-rLm9_gyIy4o6lP_gpC660qJs,2871
+fractal_server/runner/v2/_slurm_sudo.py,sha256=B94KZIi_8cENUc97m0nN8GL_b3LWwO0287qZeFkNh8o,2971
+fractal_server/runner/v2/db_tools.py,sha256=ozp4RFLB3LNI0rM0q0udi6ja8-5vooH_dVqrbmTPNDg,3323
+fractal_server/runner/v2/deduplicate_list.py,sha256=vidkd7K6u3w0A4zVgsGZkc9mwoP6ihTYJZQUhBNorfE,667
+fractal_server/runner/v2/merge_outputs.py,sha256=0ahaSwdMFAoEhxVaEaO9nSJuKIcWg9pDZ356ktSHcC0,897
+fractal_server/runner/v2/runner.py,sha256=wGfvHhO61Rqik8ZNDxoSQC4b_5UnVBgQmwZmZoPmBvE,19545
+fractal_server/runner/v2/runner_functions.py,sha256=k5PIeaNsUTWGDpf23kZwvh4PPkTZvDJP9p-by2zTHpY,19018
+fractal_server/runner/v2/submit_workflow.py,sha256=I_Etm4_u5fmyb4gtajlVu4QS2RPehA2ZM5A77gUUNjU,12348
+fractal_server/runner/v2/task_interface.py,sha256=ftPPpOU16rbJD8q-QV7o_3ey8W7MQTFuWJiYUr4OmF4,2532
+fractal_server/runner/versions.py,sha256=4BW-8Et8RVgILgpFoUJLWkEnZz53pv8hv_2ucG480ns,398
 fractal_server/ssh/__init__.py,sha256=sVUmzxf7_DuXG1xoLQ1_00fo5NPhi2LJipSmU5EAkPs,124
-fractal_server/ssh/_fabric.py,sha256=I_94ha3xxVESoa8opuvidcO32E4cJZF1Tow4eXPPc8o,24450
+fractal_server/ssh/_fabric.py,sha256=320heQ14BBwj7QmNeopmiRWW83XqFOqn2TpmnkwDwvE,26113
 fractal_server/string_tools.py,sha256=UJFi8rhlI6QXxX5twycLjsvOQ6x4uG7L3JdxEVDhC5A,2592
 fractal_server/syringe.py,sha256=3YJeIALH-wibuJ9R5VMNYUWh7x1-MkWT0SqGcWG5MY8,2795
 fractal_server/tasks/__init__.py,sha256=kadmVUoIghl8s190_Tt-8f-WBqMi8u8oU4Pvw39NHE8,23
@@ -236,8 +236,8 @@ fractal_server/types/validators/_workflow_task_arguments_validators.py,sha256=HL
 fractal_server/urls.py,sha256=QjIKAC1a46bCdiPMu3AlpgFbcv6a4l3ABcd5xz190Og,471
 fractal_server/utils.py,sha256=Vn35lApt1T1J8nc09sAVqd10Cy0sa3dLipcljI-hkuk,2185
 fractal_server/zip_tools.py,sha256=H0w7wS5yE4ebj7hw1_77YQ959dl2c-L0WX6J_ro1TY4,4884
-fractal_server-2.16.3.dist-info/LICENSE,sha256=QKAharUuhxL58kSoLizKJeZE3mTCBnX6ucmz8W0lxlk,1576
-fractal_server-2.16.3.dist-info/METADATA,sha256=pw_mNLXMPEkIRXLZGpIG5a4T_S4ehtX1HAuqTkKZrXQ,4334
-fractal_server-2.16.3.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-fractal_server-2.16.3.dist-info/entry_points.txt,sha256=8tV2kynvFkjnhbtDnxAqImL6HMVKsopgGfew0DOp5UY,58
-fractal_server-2.16.3.dist-info/RECORD,,
+fractal_server-2.16.4a0.dist-info/LICENSE,sha256=QKAharUuhxL58kSoLizKJeZE3mTCBnX6ucmz8W0lxlk,1576
+fractal_server-2.16.4a0.dist-info/METADATA,sha256=7N02pLy4e0Uvw_07uMJStJy0UiLiJF0InjftHiq1lm4,4337
+fractal_server-2.16.4a0.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+fractal_server-2.16.4a0.dist-info/entry_points.txt,sha256=8tV2kynvFkjnhbtDnxAqImL6HMVKsopgGfew0DOp5UY,58
+fractal_server-2.16.4a0.dist-info/RECORD,,