fractal-server 2.15.1__py3-none-any.whl → 2.15.3__py3-none-any.whl

fractal_server/__init__.py

@@ -1 +1 @@
-__VERSION__ = "2.15.1"
+__VERSION__ = "2.15.3"

fractal_server/app/db/__init__.py

@@ -45,13 +45,11 @@ class DB:
         settings = Inject(get_settings)
         settings.check_db()
 
-        engine_kwargs_async = {"pool_pre_ping": True}
-
         cls._engine_async = create_async_engine(
             settings.DATABASE_ASYNC_URL,
             echo=settings.DB_ECHO,
             future=True,
-            **engine_kwargs_async,
+            pool_pre_ping=True,
         )
         cls._async_session_maker = sessionmaker(
             cls._engine_async,
@@ -65,13 +63,11 @@ class DB:
         settings = Inject(get_settings)
         settings.check_db()
 
-        engine_kwargs_sync = {}
-
         cls._engine_sync = create_engine(
             settings.DATABASE_SYNC_URL,
             echo=settings.DB_ECHO,
             future=True,
-            **engine_kwargs_sync,
+            pool_pre_ping=True,
         )
 
         cls._sync_session_maker = sessionmaker(

fractal_server/app/routes/admin/v2/job.py

@@ -156,8 +156,15 @@ async def update_job(
             detail=f"Cannot set job status to {job_update.status}",
         )
 
+    timestamp = get_timestamp()
     setattr(job, "status", job_update.status)
-    setattr(job, "end_timestamp", get_timestamp())
+    setattr(job, "end_timestamp", timestamp)
+    setattr(
+        job,
+        "log",
+        f"{job.log or ''}\nThis job was manually marked as "
+        f"'{JobStatusTypeV2.FAILED}' by an admin ({timestamp.isoformat()}).",
+    )
     await db.commit()
     await db.refresh(job)
     await db.close()

fractal_server/app/routes/api/v2/_aux_functions.py

@@ -6,6 +6,7 @@ from typing import Literal
 
 from fastapi import HTTPException
 from fastapi import status
+from sqlalchemy.exc import MultipleResultsFound
 from sqlalchemy.orm.attributes import flag_modified
 from sqlmodel import select
 from sqlmodel.sql.expression import SelectOfScalar
@@ -19,6 +20,9 @@ from ....models.v2 import TaskV2
 from ....models.v2 import WorkflowTaskV2
 from ....models.v2 import WorkflowV2
 from ....schemas.v2 import JobStatusTypeV2
+from fractal_server.logger import set_logger
+
+logger = set_logger(__name__)
 
 
 async def _get_project_check_owner(
@@ -325,6 +329,24 @@ def _get_submitted_jobs_statement() -> SelectOfScalar:
     return stm
 
 
+async def _workflow_has_submitted_job(
+    workflow_id: int,
+    db: AsyncSession,
+) -> bool:
+
+    res = await db.execute(
+        select(JobV2.id)
+        .where(JobV2.status == JobStatusTypeV2.SUBMITTED)
+        .where(JobV2.workflow_id == workflow_id)
+        .limit(1)
+    )
+    submitted_jobs = res.scalar_one_or_none()
+    if submitted_jobs is not None:
+        return True
+
+    return False
+
+
 async def _workflow_insert_task(
     *,
     workflow_id: int,
@@ -481,3 +503,39 @@ async def _get_workflowtask_or_404(
         )
     else:
         return wftask
+
+
+async def _get_submitted_job_or_none(
+    *,
+    dataset_id: int,
+    workflow_id: int,
+    db: AsyncSession,
+) -> JobV2 | None:
+    """
+    Get the submitted job for given dataset/workflow, if any.
+
+    This function also handles the invalid branch where more than one job
+    is found.
+
+    Args:
+        dataset_id:
+        workflow_id:
+        db:
+    """
+    res = await db.execute(
+        _get_submitted_jobs_statement()
+        .where(JobV2.dataset_id == dataset_id)
+        .where(JobV2.workflow_id == workflow_id)
+    )
+    try:
+        return res.scalars().one_or_none()
+    except MultipleResultsFound as e:
+        error_msg = (
+            "Multiple running jobs found for "
+            f"{dataset_id=} and {workflow_id=}."
+        )
+        logger.error(f"{error_msg} Original error: {str(e)}.")
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=error_msg,
+        )

fractal_server/app/routes/api/v2/history.py

@@ -1,5 +1,3 @@
-from copy import deepcopy
-
 from fastapi import APIRouter
 from fastapi import Depends
 from fastapi import HTTPException
@@ -9,6 +7,7 @@ from sqlmodel import func
 from sqlmodel import select
 
 from ._aux_functions import _get_dataset_check_owner
+from ._aux_functions import _get_submitted_job_or_none
 from ._aux_functions import _get_workflow_check_owner
 from ._aux_functions_history import _verify_workflow_and_dataset_access
 from ._aux_functions_history import get_history_run_or_404
@@ -72,6 +71,7 @@ async def get_workflow_tasks_statuses(
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
 ) -> JSONResponse:
+
     # Access control
     workflow = await _get_workflow_check_owner(
         project_id=project_id,
@@ -86,6 +86,19 @@ async def get_workflow_tasks_statuses(
         db=db,
     )
 
+    running_job = await _get_submitted_job_or_none(
+        db=db,
+        dataset_id=dataset_id,
+        workflow_id=workflow_id,
+    )
+    if running_job is not None:
+        running_wftasks = workflow.task_list[
+            running_job.first_task_index : running_job.last_task_index + 1
+        ]
+        running_wftask_ids = [wft.id for wft in running_wftasks]
+    else:
+        running_wftask_ids = []
+
     response: dict[int, dict[str, int | str] | None] = {}
     for wftask in workflow.task_list:
         res = await db.execute(
@@ -95,17 +108,37 @@ async def get_workflow_tasks_statuses(
             .order_by(HistoryRun.timestamp_started.desc())
             .limit(1)
         )
-        latest_history_run = res.scalar_one_or_none()
-        if latest_history_run is None:
-            logger.debug(
-                f"No HistoryRun found for {dataset_id=} and {wftask.id=}."
-            )
-            response[wftask.id] = None
+        latest_run = res.scalar_one_or_none()
+
+        if latest_run is None:
+            if wftask.id in running_wftask_ids:
+                logger.debug(f"A1: No HistoryRun for {wftask.id=}.")
+                response[wftask.id] = dict(status=HistoryUnitStatus.SUBMITTED)
+            else:
+                logger.debug(f"A2: No HistoryRun for {wftask.id=}.")
+                response[wftask.id] = None
             continue
-        response[wftask.id] = dict(
-            status=latest_history_run.status,
-            num_available_images=latest_history_run.num_available_images,
-        )
+        else:
+            if wftask.id in running_wftask_ids:
+                if latest_run.job_id == running_job.id:
+                    logger.debug(
+                        f"B1 for {wftask.id} and {latest_run.job_id=}."
+                    )
+                    response[wftask.id] = dict(status=latest_run.status)
+                else:
+                    logger.debug(
+                        f"B2 for {wftask.id} and {latest_run.job_id=}."
+                    )
+                    response[wftask.id] = dict(
+                        status=HistoryUnitStatus.SUBMITTED
+                    )
+            else:
+                logger.debug(f"C1: {wftask.id=} not in {running_wftask_ids=}.")
+                response[wftask.id] = dict(status=latest_run.status)
+
+        response[wftask.id][
+            "num_available_images"
+        ] = latest_run.num_available_images
 
         for target_status in HistoryUnitStatus:
             stm = (
@@ -122,18 +155,24 @@ async def get_workflow_tasks_statuses(
             num_images = res.scalar()
             response[wftask.id][f"num_{target_status}_images"] = num_images
 
-    new_response = deepcopy(response)
-    for key, value in response.items():
-        if value is not None:
-            num_total_images = sum(
-                value[f"num_{target_status}_images"]
-                for target_status in HistoryUnitStatus
-            )
-            if num_total_images > value["num_available_images"]:
-                value["num_available_images"] = None
-        new_response[key] = value
+    # Set `num_available_images=None` for cases where it would be
+    # smaller than `num_total_images`
+    values_to_skip = (None, {"status": HistoryUnitStatus.SUBMITTED})
+    response_update = {}
+    for wftask_id, status_value in response.items():
+        if status_value in values_to_skip:
+            # Skip cases where status has no image counters
+            continue
+        num_total_images = sum(
+            status_value[f"num_{target_status}_images"]
+            for target_status in HistoryUnitStatus
+        )
+        if num_total_images > status_value["num_available_images"]:
+            status_value["num_available_images"] = None
+            response_update[wftask_id] = status_value
+    response.update(response_update)
 
-    return JSONResponse(content=new_response, status_code=200)
+    return JSONResponse(content=response, status_code=200)
 
 
 @router.get("/project/{project_id}/status/run/")

fractal_server/app/routes/api/v2/workflow.py

@@ -22,6 +22,7 @@ from ._aux_functions import _check_workflow_exists
 from ._aux_functions import _get_project_check_owner
 from ._aux_functions import _get_submitted_jobs_statement
 from ._aux_functions import _get_workflow_check_owner
+from ._aux_functions import _workflow_has_submitted_job
 from ._aux_functions_tasks import _add_warnings_to_workflow_tasks
 from fractal_server.app.models import UserOAuth
 from fractal_server.app.models.v2 import TaskGroupV2
@@ -146,6 +147,18 @@ async def update_workflow(
 
     for key, value in patch.model_dump(exclude_unset=True).items():
         if key == "reordered_workflowtask_ids":
+
+            if await _workflow_has_submitted_job(
+                workflow_id=workflow_id, db=db
+            ):
+                raise HTTPException(
+                    status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+                    detail=(
+                        "Cannot re-order WorkflowTasks while a Job is running "
+                        "for this Workflow."
+                    ),
+                )
+
             current_workflowtask_ids = [
                 wftask.id for wftask in workflow.task_list
             ]

fractal_server/app/routes/api/v2/workflowtask.py

@@ -10,6 +10,7 @@ from ....db import AsyncSession
 from ....db import get_async_db
 from ._aux_functions import _get_workflow_check_owner
 from ._aux_functions import _get_workflow_task_check_owner
+from ._aux_functions import _workflow_has_submitted_job
 from ._aux_functions import _workflow_insert_task
 from ._aux_functions_tasks import _check_type_filters_compatibility
 from ._aux_functions_tasks import _get_task_read_access
@@ -224,6 +225,15 @@ async def delete_workflowtask(
         db=db,
     )
 
+    if await _workflow_has_submitted_job(workflow_id=workflow_id, db=db):
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=(
+                "Cannot delete a WorkflowTask while a Job is running for this "
+                "Workflow."
+            ),
+        )
+
     # Delete WorkflowTask
     await db.delete(db_workflow_task)
     await db.commit()

fractal_server/app/runner/executors/slurm_common/base_slurm_runner.py

@@ -137,6 +137,34 @@ class BaseSlurmRunner(BaseRunner):
     def run_squeue(self, *, job_ids: list[str], **kwargs) -> str:
         raise NotImplementedError("Implement in child class.")
 
+    def _is_squeue_error_recoverable(self, exception: BaseException) -> True:
+        """
+        Determine whether a `squeue` error is considered recoverable.
+
+        A _recoverable_ error is one which will disappear after some time,
+        without any specific action from the `fractal-server` side.
+
+        Note: if this function returns `True` for an error that does not
+        actually recover, this leads to an infinite loop  where
+        `fractal-server` keeps polling `squeue` information forever.
+
+        More info at
+        https://github.com/fractal-analytics-platform/fractal-server/issues/2682
+
+        Args:
+            exception: The exception raised by `self.run_squeue`.
+        Returns:
+            Whether the error is considered recoverable.
+        """
+        str_exception = str(exception)
+        if (
+            "slurm_load_jobs" in str_exception
+            and "Socket timed out on send/recv operation" in str_exception
+        ):
+            return True
+        else:
+            return False
+
     def _get_finished_jobs(self, job_ids: list[str]) -> set[str]:
         #  If there is no Slurm job to check, return right away
         if not job_ids:
@@ -161,12 +189,26 @@ class BaseSlurmRunner(BaseRunner):
                         {stdout.split()[0]: stdout.split()[1]}
                     )
                 except Exception as e:
-                    logger.warning(
-                        "[_get_finished_jobs] `squeue` failed for "
-                        f"{job_id=}, mark job as completed. "
+                    msg = (
+                        f"[_get_finished_jobs] `squeue` failed for {job_id=}. "
                         f"Original error: {str(e)}."
                     )
-                    slurm_statuses.update({str(job_id): "COMPLETED"})
+                    logger.warning(msg)
+                    if self._is_squeue_error_recoverable(e):
+                        logger.warning(
+                            "[_get_finished_jobs] Recoverable `squeue` "
+                            f"error - mark {job_id=} as FRACTAL_UNDEFINED and"
+                            " retry later."
+                        )
+                        slurm_statuses.update(
+                            {str(job_id): "FRACTAL_UNDEFINED"}
+                        )
+                    else:
+                        logger.warning(
+                            "[_get_finished_jobs] Non-recoverable `squeue`"
+                            f"error - mark {job_id=} as completed."
+                        )
+                        slurm_statuses.update({str(job_id): "COMPLETED"})
 
         # If a job is not in `squeue` output, mark it as completed.
         finished_jobs = {

fractal_server/app/runner/executors/slurm_common/remote.py

@@ -1,6 +1,5 @@
 import argparse
 import json
-import logging
 import os
 import sys
 
@@ -32,7 +31,6 @@ def worker(
     # Create output folder, if missing
     out_dir = os.path.dirname(out_fname)
     if not os.path.exists(out_dir):
-        logging.debug(f"_slurm.remote.worker: create {out_dir=}")
         os.mkdir(out_dir)
 
     # Execute the job and capture exceptions
@@ -40,10 +38,8 @@ def worker(
         with open(in_fname) as f:
             input_data = json.load(f)
 
-        server_python_version = input_data["python_version"]
-        server_fractal_server_version = input_data["fractal_server_version"]
-
         # Fractal-server version must be identical
+        server_fractal_server_version = input_data["fractal_server_version"]
         worker_fractal_server_version = __VERSION__
         if worker_fractal_server_version != server_fractal_server_version:
             raise FractalVersionMismatch(
@@ -51,11 +47,16 @@ def worker(
                 f"{worker_fractal_server_version=}"
             )
 
-        # Python version mismatch only raises a warning
-        worker_python_version = tuple(sys.version_info[:3])
+        # Get `worker_python_version` as a `list` since this is the type of
+        # `server_python_version` after a JSON dump/load round trip.
+        worker_python_version = list(sys.version_info[:3])
+
+        # Print a warning for Python version mismatch
+        server_python_version = input_data["python_version"]
         if worker_python_version != server_python_version:
             if worker_python_version[:2] != server_python_version[:2]:
-                logging.warning(
+                print(
+                    "WARNING: "
                     f"{server_python_version=} but {worker_python_version=}."
                 )
 
@@ -116,7 +117,6 @@ if __name__ == "__main__":
         required=True,
     )
     parsed_args = parser.parse_args()
-    logging.debug(f"{parsed_args=}")
 
     kwargs = dict(
         in_fname=parsed_args.input_file,

fractal_server/app/runner/v2/_slurm_ssh.py

@@ -20,7 +20,6 @@ from pathlib import Path
 from ....ssh._fabric import FractalSSH
 from ...models.v2 import DatasetV2
 from ...models.v2 import WorkflowV2
-from ..exceptions import JobExecutionError
 from ..executors.slurm_common.get_slurm_config import get_slurm_config
 from ..executors.slurm_ssh.runner import SlurmSSHRunner
 from ..set_start_and_last_task_index import set_start_and_last_task_index
@@ -64,18 +63,6 @@ def process_workflow(
     if isinstance(worker_init, str):
         worker_init = worker_init.split("\n")
 
-    # Create main remote folder
-    try:
-        fractal_ssh.mkdir(folder=str(workflow_dir_remote))
-        logger.info(f"Created {str(workflow_dir_remote)} via SSH.")
-    except Exception as e:
-        error_msg = (
-            f"Could not create {str(workflow_dir_remote)} via SSH.\n"
-            f"Original error: {str(e)}."
-        )
-        logger.error(error_msg)
-        raise JobExecutionError(info=error_msg)
-
     with SlurmSSHRunner(
         fractal_ssh=fractal_ssh,
         root_dir_local=workflow_dir_local,

fractal_server/ssh/_fabric.py

@@ -165,7 +165,11 @@ class FractalSSH:
         raise e
 
     def _run(
-        self, *args, label: str, lock_timeout: float | None = None, **kwargs
+        self,
+        *args,
+        label: str,
+        lock_timeout: float | None = None,
+        **kwargs,
     ) -> Any:
         actual_lock_timeout = self.default_lock_timeout
         if lock_timeout is not None:
@@ -353,6 +357,7 @@ class FractalSSH:
                     label=f"run {cmd}",
                     lock_timeout=actual_lock_timeout,
                     hide=True,
+                    in_stream=False,
                 )
                 t_1 = time.perf_counter()
                 self.logger.info(

fractal_server/tasks/v2/local/collect_pixi.py

@@ -161,7 +161,7 @@ def collect_local_pixi(
 
                 # Make task folder 755
                 source_dir = Path(task_group.path, SOURCE_DIR_NAME).as_posix()
-                command = f"chmod 755 {source_dir} -R"
+                command = f"chmod -R 755 {source_dir}"
                 execute_command_sync(
                     command=command,
                     logger_name=LOGGER_NAME,

fractal_server/tasks/v2/local/reactivate_pixi.py

@@ -145,7 +145,7 @@ def reactivate_local_pixi(
 
                 # Make task folder 755
                 source_dir = Path(task_group.path, SOURCE_DIR_NAME).as_posix()
-                command = f"chmod 755 {source_dir} -R"
+                command = f"chmod -R 755 {source_dir}"
                 execute_command_sync(
                     command=command,
                     logger_name=LOGGER_NAME,

fractal_server/tasks/v2/ssh/collect_pixi.py

@@ -221,7 +221,7 @@ def collect_ssh_pixi(
                     source_dir = Path(
                         task_group.path, SOURCE_DIR_NAME
                     ).as_posix()
-                    fractal_ssh.run_command(cmd=f"chmod 755 {source_dir} -R")
+                    fractal_ssh.run_command(cmd=f"chmod -R 755 {source_dir}")
 
                     # Read and validate remote manifest file
                     manifest_path_remote = (

fractal_server/tasks/v2/ssh/reactivate_pixi.py

@@ -196,7 +196,7 @@ def reactivate_ssh_pixi(
                     activity.log = get_current_log(log_file_path)
                     activity = add_commit_refresh(obj=activity, db=db)
 
-                    fractal_ssh.run_command(cmd=f"chmod 755 {source_dir} -R")
+                    fractal_ssh.run_command(cmd=f"chmod -R 755 {source_dir}")
 
                     # Finalize (write metadata to DB)
                     activity.status = TaskGroupActivityStatusV2.OK

{fractal_server-2.15.1.dist-info → fractal_server-2.15.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fractal-server
-Version: 2.15.1
+Version: 2.15.3
 Summary: Backend component of the Fractal analytics platform
 License: BSD-3-Clause
 Author: Tommaso Comparin

{fractal_server-2.15.1.dist-info → fractal_server-2.15.3.dist-info}/RECORD

@@ -1,8 +1,8 @@
-fractal_server/__init__.py,sha256=6A0nKt1r_WHhSOgKEGX5ueD-wRkklKFCyvI9s88a16s,23
+fractal_server/__init__.py,sha256=TohihjKSFnz5CZYMVH94PEv_l0OHohZckQFN39DhyqE,23
 fractal_server/__main__.py,sha256=rkM8xjY1KeS3l63irB8yCrlVobR-73uDapC4wvrIlxI,6957
 fractal_server/alembic.ini,sha256=MWwi7GzjzawI9cCAK1LW7NxIBQDUqD12-ptJoq5JpP0,3153
 fractal_server/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fractal_server/app/db/__init__.py,sha256=EFzcf6iKemWlOSRj4vtDT63hAE9HBYWh4abYOdDwzMo,2907
+fractal_server/app/db/__init__.py,sha256=U2gwpNyy79iMsK1lg43LRl9z-MW8wiOaICJ7GGdA4yo,2814
 fractal_server/app/models/__init__.py,sha256=xJWiGAwpXmCpnFMC4c_HTqoUCzMOXrakoGLUH_uMvdA,415
 fractal_server/app/models/linkusergroup.py,sha256=3KkkE4QIUAlTrBAZs_tVy0pGvAxUAq6yOEjflct_z2M,678
 fractal_server/app/models/linkuserproject.py,sha256=hvaxh3Lkiy2uUCwB8gvn8RorCpvxSSdzWdCS_U1GL7g,315
@@ -23,21 +23,21 @@ fractal_server/app/routes/admin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm
 fractal_server/app/routes/admin/v2/__init__.py,sha256=_5lqb6-M8-fZqE1HRMep6pAFYRUKMxrvbZOKs-RXWkw,933
 fractal_server/app/routes/admin/v2/accounting.py,sha256=YPWwCWylXrJpV4bq_dJ3t6Kn5uuveTrFx-5w1wzfETU,3594
 fractal_server/app/routes/admin/v2/impersonate.py,sha256=gc4lshfEPFR6W2asH7aKu6hqE6chzusdhAUVV9p51eU,1131
-fractal_server/app/routes/admin/v2/job.py,sha256=EOnW645RaacyNof55O_NV_4ONyb7ihM9ORTPb0v68xY,7373
+fractal_server/app/routes/admin/v2/job.py,sha256=VcyXHYjieOKnTAi1NsiO_bK3A6UufUwX2lmWCwa4sa0,7585
 fractal_server/app/routes/admin/v2/project.py,sha256=MA_LdoEuSuisSGRO43TapMuJ080y5iaUGSAUgKuuKOg,1188
 fractal_server/app/routes/admin/v2/task.py,sha256=93QIbWZNnqaBhG9R9-RStDX2mpqRNN3G7BIb0KM-jeE,4312
 fractal_server/app/routes/admin/v2/task_group.py,sha256=biibAvMPD2w-267eyTm3wH2s3mITjiS5gYzwCCwmLbI,7099
 fractal_server/app/routes/admin/v2/task_group_lifecycle.py,sha256=2J3M9VXWD_0j9jRTZ5APuUXl9E-aVv0qF8K02vvcO3s,9150
 fractal_server/app/routes/api/__init__.py,sha256=B8l6PSAhR10iZqHEiyTat-_0tkeKdrCigIE6DJGP5b8,638
 fractal_server/app/routes/api/v2/__init__.py,sha256=D3sRRsqkmZO6kBxUjg40q0aRDsnuXI4sOOfn0xF9JsM,2820
-fractal_server/app/routes/api/v2/_aux_functions.py,sha256=P5exwdiNm0ZxtoGw4wxvm_-u8e83gXz8iYEVFuUq_cU,12792
+fractal_server/app/routes/api/v2/_aux_functions.py,sha256=YU7yT9L6yc01VMWozXPnRcx0X0063rTylmeU6PKNyKM,14260
 fractal_server/app/routes/api/v2/_aux_functions_history.py,sha256=Z23xwvBaVEEQ5B-JsWZJpjj4_QqoXqHYONztnbAH6gw,4425
 fractal_server/app/routes/api/v2/_aux_functions_task_lifecycle.py,sha256=GpKfw9yj01LmOAuNMTOreU1PFkCKpjK5oCt7_wp35-A,6741
 fractal_server/app/routes/api/v2/_aux_functions_task_version_update.py,sha256=WLDOYCnb6fnS5avKflyx6yN24Vo1n5kJk5ZyiKbzb8Y,1175
 fractal_server/app/routes/api/v2/_aux_functions_tasks.py,sha256=MNty8CBnTMPSAKE5gMT7tCY8QWpCQyhft_shq12hHpA,12208
 fractal_server/app/routes/api/v2/_aux_task_group_disambiguation.py,sha256=8x1_q9FyCzItnPmdSdLQuwUTy4B9xCsXscp97_lJcpM,4635
 fractal_server/app/routes/api/v2/dataset.py,sha256=6u4MFqJ3YZ0Zq6Xx8CRMrTPKW55ZaR63Uno21DqFr4Q,8889
-fractal_server/app/routes/api/v2/history.py,sha256=BHBZYFSF5lw-YYOl0OVV5yEZPMxiqjH72_KwR66EtaE,15495
+fractal_server/app/routes/api/v2/history.py,sha256=ErLqkJbhx9XzHL4KQvMraVAtD9WOmDNh5tZi5wmNkL0,17114
 fractal_server/app/routes/api/v2/images.py,sha256=TS1ltUhP0_SaViupdHrSh3MLDi5OVk-lOhE1VCVyZj0,7869
 fractal_server/app/routes/api/v2/job.py,sha256=8xRTwh_OCHmK9IfI_zUASa2ozewR0qu0zVBl_a4IvHw,6467
 fractal_server/app/routes/api/v2/pre_submission_checks.py,sha256=2jaaM5WJBTGpOWhm6a42JViT8j4X5hixltxIY1p-188,4936
@@ -51,9 +51,9 @@ fractal_server/app/routes/api/v2/task_collection_pixi.py,sha256=LS5xOYRRvI25TyvP
 fractal_server/app/routes/api/v2/task_group.py,sha256=Wmp5Rt6NQm8_EbdJyi3XOkTXxJTTd4MNIy0ja6K-ifA,9205
 fractal_server/app/routes/api/v2/task_group_lifecycle.py,sha256=-uS_z8E3__t_twEqhZOzcEcAxZsgnpg-c7Ya9RF3_bs,9998
 fractal_server/app/routes/api/v2/task_version_update.py,sha256=o8W_C0I84X0u8gAMnCvi8ChiVAKrb5WzUBuJLSuujCA,8235
-fractal_server/app/routes/api/v2/workflow.py,sha256=gwMtpfUY_JiTv5_R_q1I9WNkp6nTqEVtYx8jWNJRxcU,10227
+fractal_server/app/routes/api/v2/workflow.py,sha256=SfjegoVO4DaGmDD7OPhWNLkcvZhJKwNX4DTQAcVKk9Q,10699
 fractal_server/app/routes/api/v2/workflow_import.py,sha256=kOGDaCj0jCGK1WSYGbnUjtUg2U1YxUY9UMH-2ilqJg4,9027
-fractal_server/app/routes/api/v2/workflowtask.py,sha256=KQU9rSQNhc6TRFdUYM09zty8Bu150sKvcLGz_tX4Fgo,7548
+fractal_server/app/routes/api/v2/workflowtask.py,sha256=5_SQAG8ztDnaaRXwKalcO69HVpSl-QbrhiI7fCP3YRI,7924
 fractal_server/app/routes/auth/__init__.py,sha256=fao6CS0WiAjHDTvBzgBVV_bSXFpEAeDBF6Z6q7rRkPc,1658
 fractal_server/app/routes/auth/_aux_auth.py,sha256=UZgauY0V6mSqjte_sYI1cBl2h8bcbLaeWzgpl1jdJlk,4883
 fractal_server/app/routes/auth/current_user.py,sha256=EjkwMxUA0l6FLbDJdertHGnuOoSS-HEysmm6l5FkAlY,5903
@@ -81,9 +81,9 @@ fractal_server/app/runner/executors/slurm_common/__init__.py,sha256=47DEQpj8HBSa
 fractal_server/app/runner/executors/slurm_common/_batching.py,sha256=gbHZIxt90GjUwhB9_UInwVqpX-KdxRQMDeXzUagdL3U,8816
 fractal_server/app/runner/executors/slurm_common/_job_states.py,sha256=nuV-Zba38kDrRESOVB3gaGbrSPZc4q7YGichQaeqTW0,238
 fractal_server/app/runner/executors/slurm_common/_slurm_config.py,sha256=U9BONnnwn8eDqDevwUtFSBcvIsxvNgDHirhcQGJ9t9E,15947
-fractal_server/app/runner/executors/slurm_common/base_slurm_runner.py,sha256=iXRlTuPvBqqq_I-WYF2IRocPEayMCBxCSJkn_eop6t4,36226
+fractal_server/app/runner/executors/slurm_common/base_slurm_runner.py,sha256=1Sh56lb7NERVtsBMvVs4K7nVHhMy_KDbwquPl1ub8vE,37937
 fractal_server/app/runner/executors/slurm_common/get_slurm_config.py,sha256=jhoFHauWJm55bIC_v7pFylbK8WgcRJemGu2OjUiRbpQ,7377
-fractal_server/app/runner/executors/slurm_common/remote.py,sha256=xWnI6WktHR_7cxUme72ztIeBb4osnbZNu6J2azWn9K8,3765
+fractal_server/app/runner/executors/slurm_common/remote.py,sha256=LHK2Ram8X8q6jNSCxnnwKUwmSJMsyQyRem_VjH53qdw,3811
 fractal_server/app/runner/executors/slurm_common/slurm_job_task_models.py,sha256=K4SdJOKsUWzDlnkb8Ug_UmTx6nBMsTqn9_oKqwE4XDI,3520
 fractal_server/app/runner/executors/slurm_ssh/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fractal_server/app/runner/executors/slurm_ssh/run_subprocess.py,sha256=SyW6t4egvbiARph2YkFjc88Hj94fCamZVi50L7ph8VM,996
@@ -98,7 +98,7 @@ fractal_server/app/runner/shutdown.py,sha256=ViSNJyXWU_iWPSDOOMGNh_iQdUFrdPh_jvf
 fractal_server/app/runner/task_files.py,sha256=V_7aZhu6-c6Y-0XUe-5cZVDrdnXEJhp8pQoUMtx6ko0,4041
 fractal_server/app/runner/v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fractal_server/app/runner/v2/_local.py,sha256=tTJgABK-zAZmmRzoie_MPNTXJx_zBAXiZiiWl1CC2qo,3035
-fractal_server/app/runner/v2/_slurm_ssh.py,sha256=VN89sFwqi139m9wpO1LmExIYIVhmYbEoMgtX7kLAMhE,3302
+fractal_server/app/runner/v2/_slurm_ssh.py,sha256=JlDngsVSOUNqEubDl-5jkIxQQmV4mhTqbMbJVY_rL6M,2840
 fractal_server/app/runner/v2/_slurm_sudo.py,sha256=Gvsh4tUlc1_3KdF3B7zEqs-YIntC_joLtTGSNFbKKSs,2939
 fractal_server/app/runner/v2/db_tools.py,sha256=du5dKhMMFMErQXbGIgu9JvO_vtMensodyPsyDeqz1yQ,3324
 fractal_server/app/runner/v2/deduplicate_list.py,sha256=IVTE4abBU1bUprFTkxrTfYKnvkNTanWQ-KWh_etiT08,645
@@ -185,7 +185,7 @@ fractal_server/migrations/versions/f384e1c0cf5d_drop_task_default_args_columns.p
 fractal_server/migrations/versions/fbce16ff4e47_new_history_items.py,sha256=TDWCaIoM0Q4SpRWmR9zr_rdp3lJXhCfBPTMhtrP5xYE,3950
 fractal_server/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fractal_server/ssh/__init__.py,sha256=sVUmzxf7_DuXG1xoLQ1_00fo5NPhi2LJipSmU5EAkPs,124
-fractal_server/ssh/_fabric.py,sha256=3Q0nlXctEXtRspQvg00wzCOyYK-0fAJlCowQuXEMk9I,25118
+fractal_server/ssh/_fabric.py,sha256=7fCxTYqkAOaTTm67trfYdYQenOsI4EfrRQoG6x3M5kk,25188
 fractal_server/string_tools.py,sha256=qLB5u6-4QxXPiZrUeWn_cEo47axj4OXFzDd47kNTIWw,1847
 fractal_server/syringe.py,sha256=3YJeIALH-wibuJ9R5VMNYUWh7x1-MkWT0SqGcWG5MY8,2795
 fractal_server/tasks/__init__.py,sha256=kadmVUoIghl8s190_Tt-8f-WBqMi8u8oU4Pvw39NHE8,23
@@ -194,19 +194,19 @@ fractal_server/tasks/v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG
 fractal_server/tasks/v2/local/__init__.py,sha256=S842wRersYKBKjc7xbmj0ov8b5i1YuCHa2f_yYuxcaI,312
 fractal_server/tasks/v2/local/_utils.py,sha256=p2KJ4BvEwJxahICpzbvzrc5-ciLCFnLXWPCwdNGi-3Q,2495
 fractal_server/tasks/v2/local/collect.py,sha256=MQncScKbWv3g9lrjF8WOhzuEoTEOOgS02RqOJno5csI,11897
-fractal_server/tasks/v2/local/collect_pixi.py,sha256=i24MS7yxV0_sHkZJ8rd148n8TGqCPo6Zob5LPks3odk,10753
+fractal_server/tasks/v2/local/collect_pixi.py,sha256=coV9SqOf5rz2dgUFG7uVisPFS0xvXEubFwU7rb3QHe8,10753
 fractal_server/tasks/v2/local/deactivate.py,sha256=LoEs2TUoHQOq3JfxufW6zroXD-Xx_b-hLtdigEBi1JU,9732
 fractal_server/tasks/v2/local/deactivate_pixi.py,sha256=_ycvnLIZ8zUFB3fZbCzDlNudh-SSetl4UkyFrClCcUU,3494
 fractal_server/tasks/v2/local/reactivate.py,sha256=Q43DOadNeFyyfgNP67lUqaXmZsS6onv67XwxH_-5ANA,5756
-fractal_server/tasks/v2/local/reactivate_pixi.py,sha256=wF_3gcMWO_8ArJFo4iYh-51LDZDF_1OuYYHrY9eUSL8,7320
+fractal_server/tasks/v2/local/reactivate_pixi.py,sha256=X1gdeuFGPtohjWEZ7OX2v8m6aI7Z93M-y64FqtYjApg,7320
 fractal_server/tasks/v2/ssh/__init__.py,sha256=vX5aIM9Hbn2T_cIP_LrZ5ekRqJzYm_GSfp-4Iv7kqeI,300
 fractal_server/tasks/v2/ssh/_utils.py,sha256=ktVH7psQSAhh353fVUe-BwiBZHzTdgXnR-Xv_vfuX0Y,3857
 fractal_server/tasks/v2/ssh/collect.py,sha256=M9gFD1h9Q1Z-BFQq65dI0vFs6HPCkKQzOkxaLddmChY,14334
-fractal_server/tasks/v2/ssh/collect_pixi.py,sha256=q68GSL9yvezqOEc-zt4Ko1d7ZhTlJugoGspRA8JPiJw,13801
+fractal_server/tasks/v2/ssh/collect_pixi.py,sha256=MYxHY5P69P7DdM4uC8FAsAoQBuqr8UJdDti0CPHAn_U,13801
 fractal_server/tasks/v2/ssh/deactivate.py,sha256=XAIy84cLT9MSTMiN67U-wfOjxMm5s7lmrGwhW0qp7BU,12439
 fractal_server/tasks/v2/ssh/deactivate_pixi.py,sha256=K0yK_NPUqhFMj6cp6G_0Kfn0Yo7oQux4kT5dFPulnos,4748
 fractal_server/tasks/v2/ssh/reactivate.py,sha256=NJIgMNFKaXMhbvK0iZOsMwMtsms6Boj9f8N4L01X9Bo,8271
-fractal_server/tasks/v2/ssh/reactivate_pixi.py,sha256=o86FNB8pX9Sv_ROrCOJQqxIfDUhJqzzmo3JKsSfCNlA,9784
+fractal_server/tasks/v2/ssh/reactivate_pixi.py,sha256=Vay6kfsrc5XKx2WJVTu_pLhgpuHZDdnrEB6Er8XchYo,9784
 fractal_server/tasks/v2/templates/1_create_venv.sh,sha256=PK0jdHKtQpda1zULebBaVPORt4t6V17wa4N1ohcj5ac,548
 fractal_server/tasks/v2/templates/2_pip_install.sh,sha256=jMJPQJXHKznO6fxOOXtFXKPdCmTf1VLLWj_JL_ZdKxo,1644
 fractal_server/tasks/v2/templates/3_pip_freeze.sh,sha256=JldREScEBI4cD_qjfX4UK7V4aI-FnX9ZvVNxgpSOBFc,168
@@ -230,8 +230,8 @@ fractal_server/types/validators/_workflow_task_arguments_validators.py,sha256=HL
 fractal_server/urls.py,sha256=QjIKAC1a46bCdiPMu3AlpgFbcv6a4l3ABcd5xz190Og,471
 fractal_server/utils.py,sha256=Vn35lApt1T1J8nc09sAVqd10Cy0sa3dLipcljI-hkuk,2185
 fractal_server/zip_tools.py,sha256=tqz_8f-vQ9OBRW-4OQfO6xxY-YInHTyHmZxU7U4PqZo,4885
-fractal_server-2.15.1.dist-info/LICENSE,sha256=QKAharUuhxL58kSoLizKJeZE3mTCBnX6ucmz8W0lxlk,1576
-fractal_server-2.15.1.dist-info/METADATA,sha256=kxk3S8JxO5PKXtG0HSpS2jLbnQ0JMF-WHZjrqHxPatY,4243
-fractal_server-2.15.1.dist-info/WHEEL,sha256=7dDg4QLnNKTvwIDR9Ac8jJaAmBC_owJrckbC0jjThyA,88
-fractal_server-2.15.1.dist-info/entry_points.txt,sha256=8tV2kynvFkjnhbtDnxAqImL6HMVKsopgGfew0DOp5UY,58
-fractal_server-2.15.1.dist-info/RECORD,,
+fractal_server-2.15.3.dist-info/LICENSE,sha256=QKAharUuhxL58kSoLizKJeZE3mTCBnX6ucmz8W0lxlk,1576
+fractal_server-2.15.3.dist-info/METADATA,sha256=-Z1TLhBsEtPcHcx4oq8Hij5f1mUfDyTiL3ttW1cBmTE,4243
+fractal_server-2.15.3.dist-info/WHEEL,sha256=7dDg4QLnNKTvwIDR9Ac8jJaAmBC_owJrckbC0jjThyA,88
+fractal_server-2.15.3.dist-info/entry_points.txt,sha256=8tV2kynvFkjnhbtDnxAqImL6HMVKsopgGfew0DOp5UY,58
+fractal_server-2.15.3.dist-info/RECORD,,