fractal-server 2.14.0a10__py3-none-any.whl → 2.14.0a11__py3-none-any.whl

fractal_server/app/runner/v2/runner_functions_low_level.py

@@ -2,16 +2,11 @@ import json
 import logging
 import shutil
 import subprocess  # nosec
-from pathlib import Path
-from shlex import split as shlex_split
+from shlex import split
 from typing import Any
-from typing import Optional
 
-from ..components import _COMPONENT_KEY_
-from ..exceptions import JobExecutionError
-from ..exceptions import TaskExecutionError
-from fractal_server.app.models.v2 import WorkflowTaskV2
-from fractal_server.app.runner.task_files import TaskFiles
+from fractal_server.app.runner.exceptions import JobExecutionError
+from fractal_server.app.runner.exceptions import TaskExecutionError
 from fractal_server.string_tools import validate_cmd
 
 
@@ -32,9 +27,9 @@ def _call_command_wrapper(cmd: str, log_path: str) -> None:
         raise TaskExecutionError(f"Invalid command. Original error: {str(e)}")
 
     # Verify that task command is executable
-    if shutil.which(shlex_split(cmd)[0]) is None:
+    if shutil.which(split(cmd)[0]) is None:
         msg = (
-            f'Command "{shlex_split(cmd)[0]}" is not valid. '
+            f'Command "{split(cmd)[0]}" is not valid. '
             "Hint: make sure that it is executable."
         )
         raise TaskExecutionError(msg)
@@ -42,7 +37,7 @@ def _call_command_wrapper(cmd: str, log_path: str) -> None:
     with open(log_path, "w") as fp_log:
         try:
             result = subprocess.run(  # nosec
-                shlex_split(cmd),
+                split(cmd),
                 stderr=fp_log,
                 stdout=fp_log,
             )
@@ -60,58 +55,61 @@ def _call_command_wrapper(cmd: str, log_path: str) -> None:
 
 
 def run_single_task(
-    parameters: dict[str, Any],
+    # COMMON to all parallel tasks
     command: str,
-    wftask: WorkflowTaskV2,
-    root_dir_local: Path,
-    root_dir_remote: Optional[Path] = None,
-    logger_name: Optional[str] = None,
+    workflow_task_order: int,
+    workflow_task_id: int,
+    task_name: str,
+    # SPECIAL for each parallel task
+    parameters: dict[str, Any],
+    remote_files: dict[str, str],
 ) -> dict[str, Any]:
     """
     Runs within an executor (AKA on the SLURM cluster).
     """
 
-    logger = logging.getLogger(logger_name)
-    logger.debug(f"Now start running {command=}")
-
-    if not root_dir_remote:
-        root_dir_remote = root_dir_local
-
-    task_name = wftask.task.name
+    try:
+        args_file_remote = remote_files["args_file_remote"]
+        metadiff_file_remote = remote_files["metadiff_file_remote"]
+        log_file_remote = remote_files["log_file_remote"]
+    except KeyError:
+        raise TaskExecutionError(
+            f"Invalid {remote_files=}",
+            workflow_task_order=workflow_task_order,
+            workflow_task_id=workflow_task_id,
+            task_name=task_name,
+        )
 
-    component = parameters.pop(_COMPONENT_KEY_)
-    task_files = TaskFiles(
-        root_dir_local=root_dir_local,
-        root_dir_remote=root_dir_remote,
-        task_name=task_name,
-        task_order=wftask.order,
-        component=component,
-    )
+    logger = logging.getLogger(None)
+    logger.debug(f"Now start running {command=}")
 
     # Write arguments to args.json file
-    with open(task_files.args_file_remote, "w") as f:
+    # FIXME: this could be done backend-side, with an additional
+    # file transfer if needed (e.g. on SSH)
+    with open(args_file_remote, "w") as f:
         json.dump(parameters, f, indent=2)
 
     # Assemble full command
+    # FIXME: this could be assembled backend-side
     full_command = (
         f"{command} "
-        f"--args-json {task_files.args_file_remote} "
-        f"--out-json {task_files.metadiff_file_remote}"
+        f"--args-json {args_file_remote} "
+        f"--out-json {metadiff_file_remote}"
     )
 
     try:
         _call_command_wrapper(
             full_command,
-            log_path=task_files.log_file_remote,
+            log_path=log_file_remote,
         )
     except TaskExecutionError as e:
-        e.workflow_task_order = wftask.order
-        e.workflow_task_id = wftask.id
-        e.task_name = wftask.task.name
+        e.workflow_task_order = workflow_task_order
+        e.workflow_task_id = workflow_task_id
+        e.task_name = task_name
         raise e
 
     try:
-        with open(task_files.metadiff_file_remote, "r") as f:
+        with open(metadiff_file_remote, "r") as f:
             out_meta = json.load(f)
     except FileNotFoundError as e:
         logger.debug(

fractal_server/app/runner/v2/submit_workflow.py

@@ -0,0 +1,366 @@
+"""
+Runner backend subsystem root V2
+
+This module is the single entry point to the runner backend subsystem V2.
+Other subsystems should only import this module and not its submodules or
+the individual backends.
+"""
+import os
+import traceback
+from pathlib import Path
+from typing import Optional
+
+from sqlalchemy.orm import Session as DBSyncSession
+
+from ....config import get_settings
+from ....logger import get_logger
+from ....logger import reset_logger_handlers
+from ....logger import set_logger
+from ....ssh._fabric import FractalSSH
+from ....syringe import Inject
+from ....utils import get_timestamp
+from ....zip_tools import _zip_folder_to_file_and_remove
+from ...db import DB
+from ...models.v2 import DatasetV2
+from ...models.v2 import JobV2
+from ...models.v2 import WorkflowV2
+from ...schemas.v2 import JobStatusTypeV2
+from ..exceptions import JobExecutionError
+from ..exceptions import TaskExecutionError
+from ..executors.slurm_sudo._subprocess_run_as_user import _mkdir_as_user
+from ..filenames import WORKFLOW_LOG_FILENAME
+from ._local import process_workflow as local_process_workflow
+from ._slurm_ssh import process_workflow as slurm_ssh_process_workflow
+from ._slurm_sudo import process_workflow as slurm_sudo_process_workflow
+from fractal_server import __VERSION__
+from fractal_server.app.models import UserSettings
+
+
+_backends = {}
+_backends["local"] = local_process_workflow
+_backends["slurm"] = slurm_sudo_process_workflow
+_backends["slurm_ssh"] = slurm_ssh_process_workflow
+
+
+def fail_job(
+    *,
+    db: DBSyncSession,
+    job: JobV2,
+    log_msg: str,
+    logger_name: str,
+    emit_log: bool = False,
+) -> None:
+    logger = get_logger(logger_name=logger_name)
+    if emit_log:
+        logger.error(log_msg)
+    reset_logger_handlers(logger)
+    job.status = JobStatusTypeV2.FAILED
+    job.end_timestamp = get_timestamp()
+    job.log = log_msg
+    db.merge(job)
+    db.commit()
+    db.close()
+    return
+
+
+def submit_workflow(
+    *,
+    workflow_id: int,
+    dataset_id: int,
+    job_id: int,
+    user_id: int,
+    user_settings: UserSettings,
+    worker_init: Optional[str] = None,
+    slurm_user: Optional[str] = None,
+    user_cache_dir: Optional[str] = None,
+    fractal_ssh: Optional[FractalSSH] = None,
+) -> None:
+    """
+    Prepares a workflow and applies it to a dataset
+
+    This function wraps the process_workflow one, which is different for each
+    backend (e.g. local or slurm backend).
+
+    Args:
+        workflow_id:
+            ID of the workflow being applied
+        dataset_id:
+            Dataset ID
+        job_id:
+            Id of the job record which stores the state for the current
+            workflow application.
+        user_id:
+            User ID.
+        worker_init:
+            Custom executor parameters that get parsed before the execution of
+            each task.
+        user_cache_dir:
+            Cache directory (namely a path where the user can write); for the
+            slurm backend, this is used as a base directory for
+            `job.working_dir_user`.
+        slurm_user:
+            The username to impersonate for the workflow execution, for the
+            slurm backend.
+    """
+    # Declare runner backend and set `process_workflow` function
+    settings = Inject(get_settings)
+    FRACTAL_RUNNER_BACKEND = settings.FRACTAL_RUNNER_BACKEND
+    logger_name = f"WF{workflow_id}_job{job_id}"
+    logger = set_logger(logger_name=logger_name)
+
+    with next(DB.get_sync_db()) as db_sync:
+        try:
+            job: Optional[JobV2] = db_sync.get(JobV2, job_id)
+            dataset: Optional[DatasetV2] = db_sync.get(DatasetV2, dataset_id)
+            workflow: Optional[WorkflowV2] = db_sync.get(
+                WorkflowV2, workflow_id
+            )
+        except Exception as e:
+            logger.error(
+                f"Error connecting to the database. Original error: {str(e)}"
+            )
+            reset_logger_handlers(logger)
+            return
+
+        if job is None:
+            logger.error(f"JobV2 {job_id} does not exist")
+            reset_logger_handlers(logger)
+            return
+        if dataset is None or workflow is None:
+            log_msg = ""
+            if not dataset:
+                log_msg += f"Cannot fetch dataset {dataset_id} from database\n"
+            if not workflow:
+                log_msg += (
+                    f"Cannot fetch workflow {workflow_id} from database\n"
+                )
+            fail_job(
+                db=db_sync, job=job, log_msg=log_msg, logger_name=logger_name
+            )
+            return
+
+        # Declare runner backend and set `process_workflow` function
+        settings = Inject(get_settings)
+        FRACTAL_RUNNER_BACKEND = settings.FRACTAL_RUNNER_BACKEND
+        try:
+            process_workflow = _backends[settings.FRACTAL_RUNNER_BACKEND]
+        except KeyError as e:
+            fail_job(
+                db=db_sync,
+                job=job,
+                log_msg=(
+                    f"Invalid {FRACTAL_RUNNER_BACKEND=}.\n"
+                    f"Original KeyError: {str(e)}"
+                ),
+                logger_name=logger_name,
+                emit_log=True,
+            )
+            return
+
+        # Define and create server-side working folder
+        WORKFLOW_DIR_LOCAL = Path(job.working_dir)
+        if WORKFLOW_DIR_LOCAL.exists():
+            fail_job(
+                db=db_sync,
+                job=job,
+                log_msg=f"Workflow dir {WORKFLOW_DIR_LOCAL} already exists.",
+                logger_name=logger_name,
+                emit_log=True,
+            )
+            return
+
+        try:
+            # Create WORKFLOW_DIR_LOCAL
+            if FRACTAL_RUNNER_BACKEND == "slurm":
+                original_umask = os.umask(0)
+                WORKFLOW_DIR_LOCAL.mkdir(parents=True, mode=0o755)
+                os.umask(original_umask)
+            else:
+                WORKFLOW_DIR_LOCAL.mkdir(parents=True)
+
+            # Define and create WORKFLOW_DIR_REMOTE
+            if FRACTAL_RUNNER_BACKEND == "local":
+                WORKFLOW_DIR_REMOTE = WORKFLOW_DIR_LOCAL
+            elif FRACTAL_RUNNER_BACKEND == "slurm":
+                WORKFLOW_DIR_REMOTE = (
+                    Path(user_cache_dir) / WORKFLOW_DIR_LOCAL.name
+                )
+                _mkdir_as_user(
+                    folder=str(WORKFLOW_DIR_REMOTE), user=slurm_user
+                )
+            elif FRACTAL_RUNNER_BACKEND == "slurm_ssh":
+                # Folder creation is deferred to _process_workflow
+                WORKFLOW_DIR_REMOTE = (
+                    Path(user_settings.ssh_jobs_dir) / WORKFLOW_DIR_LOCAL.name
+                )
+            else:
+                logger.error(
+                    "Invalid FRACTAL_RUNNER_BACKEND="
+                    f"{settings.FRACTAL_RUNNER_BACKEND}."
+                )
+
+        except Exception as e:
+            error_type = type(e).__name__
+            fail_job(
+                db=db_sync,
+                job=job,
+                log_msg=(
+                    f"{error_type} error occurred while creating job folder "
+                    f"and subfolders.\nOriginal error: {str(e)}"
+                ),
+                logger_name=logger_name,
+                emit_log=True,
+            )
+            return
+
+        # After Session.commit() is called, either explicitly or when using a
+        # context manager, all objects associated with the Session are expired.
+        # https://docs.sqlalchemy.org/en/14/orm/
+        #   session_basics.html#opening-and-closing-a-session
+        # https://docs.sqlalchemy.org/en/14/orm/
+        #   session_state_management.html#refreshing-expiring
+
+        # See issue #928:
+        # https://github.com/fractal-analytics-platform/
+        #   fractal-server/issues/928
+
+        db_sync.refresh(dataset)
+        db_sync.refresh(workflow)
+        for wftask in workflow.task_list:
+            db_sync.refresh(wftask)
+
+        # Write logs
+        log_file_path = WORKFLOW_DIR_LOCAL / WORKFLOW_LOG_FILENAME
+        logger = set_logger(
+            logger_name=logger_name,
+            log_file_path=log_file_path,
+        )
+        logger.info(
+            f'Start execution of workflow "{workflow.name}"; '
+            f"more logs at {str(log_file_path)}"
+        )
+        logger.debug(f"fractal_server.__VERSION__: {__VERSION__}")
+        logger.debug(f"FRACTAL_RUNNER_BACKEND: {FRACTAL_RUNNER_BACKEND}")
+        if FRACTAL_RUNNER_BACKEND == "slurm":
+            logger.debug(f"slurm_user: {slurm_user}")
+            logger.debug(f"slurm_account: {job.slurm_account}")
+            logger.debug(f"worker_init: {worker_init}")
+        elif FRACTAL_RUNNER_BACKEND == "slurm_ssh":
+            logger.debug(f"ssh_user: {user_settings.ssh_username}")
+            logger.debug(f"base dir: {user_settings.ssh_tasks_dir}")
+            logger.debug(f"worker_init: {worker_init}")
+        logger.debug(f"job.id: {job.id}")
+        logger.debug(f"job.working_dir: {job.working_dir}")
+        logger.debug(f"job.working_dir_user: {job.working_dir_user}")
+        logger.debug(f"job.first_task_index: {job.first_task_index}")
+        logger.debug(f"job.last_task_index: {job.last_task_index}")
+        logger.debug(f'START workflow "{workflow.name}"')
+
+    try:
+        if FRACTAL_RUNNER_BACKEND == "local":
+            process_workflow = local_process_workflow
+            backend_specific_kwargs = {}
+        elif FRACTAL_RUNNER_BACKEND == "slurm":
+            process_workflow = slurm_sudo_process_workflow
+            backend_specific_kwargs = dict(
+                slurm_user=slurm_user,
+                slurm_account=job.slurm_account,
+                user_cache_dir=user_cache_dir,
+            )
+        elif FRACTAL_RUNNER_BACKEND == "slurm_ssh":
+            process_workflow = slurm_ssh_process_workflow
+            backend_specific_kwargs = dict(fractal_ssh=fractal_ssh)
+        else:
+            raise RuntimeError(
+                f"Invalid runner backend {FRACTAL_RUNNER_BACKEND=}"
+            )
+
+        # "The Session.close() method does not prevent the Session from being
+        # used again. The Session itself does not actually have a distinct
+        # “closed” state; it merely means the Session will release all database
+        # connections and ORM objects."
+        # (https://docs.sqlalchemy.org/en/20/orm/session_api.html#sqlalchemy.orm.Session.close).
+        #
+        # We close the session before the (possibly long) process_workflow
+        # call, to make sure all DB connections are released. The reason why we
+        # are not using a context manager within the try block is that we also
+        # need access to db_sync in the except branches.
+        db_sync = next(DB.get_sync_db())
+        db_sync.close()
+
+        process_workflow(
+            workflow=workflow,
+            dataset=dataset,
+            user_id=user_id,
+            workflow_dir_local=WORKFLOW_DIR_LOCAL,
+            workflow_dir_remote=WORKFLOW_DIR_REMOTE,
+            logger_name=logger_name,
+            worker_init=worker_init,
+            first_task_index=job.first_task_index,
+            last_task_index=job.last_task_index,
+            job_attribute_filters=job.attribute_filters,
+            job_type_filters=job.type_filters,
+            **backend_specific_kwargs,
+        )
+
+        logger.info(
+            f'End execution of workflow "{workflow.name}"; '
+            f"more logs at {str(log_file_path)}"
+        )
+        logger.debug(f'END workflow "{workflow.name}"')
+
+        # Update job DB entry
+        job.status = JobStatusTypeV2.DONE
+        job.end_timestamp = get_timestamp()
+        with log_file_path.open("r") as f:
+            logs = f.read()
+        job.log = logs
+        db_sync.merge(job)
+        db_sync.commit()
+
+    except TaskExecutionError as e:
+        logger.debug(f'FAILED workflow "{workflow.name}", TaskExecutionError.')
+        logger.info(f'Workflow "{workflow.name}" failed (TaskExecutionError).')
+
+        exception_args_string = "\n".join(e.args)
+        log_msg = (
+            f"TASK ERROR: "
+            f"Task name: {e.task_name}, "
+            f"position in Workflow: {e.workflow_task_order}\n"
+            f"TRACEBACK:\n{exception_args_string}"
+        )
+        fail_job(db=db_sync, job=job, log_msg=log_msg, logger_name=logger_name)
+
+    except JobExecutionError as e:
+        logger.debug(f'FAILED workflow "{workflow.name}", JobExecutionError.')
+        logger.info(f'Workflow "{workflow.name}" failed (JobExecutionError).')
+
+        fail_job(
+            db=db_sync,
+            job=job,
+            log_msg=(
+                f"JOB ERROR in Fractal job {job.id}:\n"
+                f"TRACEBACK:\n{e.assemble_error()}"
+            ),
+            logger_name=logger_name,
+        )
+
+    except Exception:
+        logger.debug(f'FAILED workflow "{workflow.name}", unknown error.')
+        logger.info(f'Workflow "{workflow.name}" failed (unkwnon error).')
+
+        current_traceback = traceback.format_exc()
+        fail_job(
+            db=db_sync,
+            job=job,
+            log_msg=(
+                f"UNKNOWN ERROR in Fractal job {job.id}\n"
+                f"TRACEBACK:\n{current_traceback}"
+            ),
+            logger_name=logger_name,
+        )
+
+    finally:
+        reset_logger_handlers(logger)
+        db_sync.close()
+        _zip_folder_to_file_and_remove(folder=job.working_dir)

fractal_server/app/runner/v2/task_interface.py

@@ -1,11 +1,14 @@
 from typing import Any
+from typing import Optional
 
 from pydantic import BaseModel
 from pydantic import ConfigDict
 from pydantic import Field
 from pydantic import field_validator
+from pydantic import ValidationError
 
 from ....images import SingleImageTaskOutput
+from fractal_server.app.runner.exceptions import TaskOutputValidationError
 from fractal_server.urls import normalize_url
 
 
@@ -61,3 +64,31 @@ class InitTaskOutput(BaseModel):
     model_config = ConfigDict(extra="forbid")
 
     parallelization_list: list[InitArgsModel] = Field(default_factory=list)
+
+
+def _cast_and_validate_TaskOutput(
+    task_output: dict[str, Any]
+) -> Optional[TaskOutput]:
+    try:
+        validated_task_output = TaskOutput(**task_output)
+        return validated_task_output
+    except ValidationError as e:
+        raise TaskOutputValidationError(
+            "Validation of task output failed.\n"
+            f"Original error: {str(e)}\n"
+            f"Original data: {task_output}."
+        )
+
+
+def _cast_and_validate_InitTaskOutput(
+    init_task_output: dict[str, Any],
+) -> Optional[InitTaskOutput]:
+    try:
+        validated_init_task_output = InitTaskOutput(**init_task_output)
+        return validated_init_task_output
+    except ValidationError as e:
+        raise TaskOutputValidationError(
+            "Validation of init-task output failed.\n"
+            f"Original error: {str(e)}\n"
+            f"Original data: {init_task_output}."
+        )

{fractal_server-2.14.0a10.dist-info → fractal_server-2.14.0a11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fractal-server
-Version: 2.14.0a10
+Version: 2.14.0a11
 Summary: Backend component of the Fractal analytics platform
 License: BSD-3-Clause
 Author: Tommaso Comparin

{fractal_server-2.14.0a10.dist-info → fractal_server-2.14.0a11.dist-info}/RECORD

@@ -1,4 +1,4 @@
-fractal_server/__init__.py,sha256=UP2HICqxwL_8o6uR6anG2OkfjF9pm3Ayy-wgRZ37_Xo,26
+fractal_server/__init__.py,sha256=GtuXF0xAIDTX9AXSJY7o0G6jQtfGliAzZf-nkCVjhSg,26
 fractal_server/__main__.py,sha256=rkM8xjY1KeS3l63irB8yCrlVobR-73uDapC4wvrIlxI,6957
 fractal_server/alembic.ini,sha256=MWwi7GzjzawI9cCAK1LW7NxIBQDUqD12-ptJoq5JpP0,3153
 fractal_server/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -41,7 +41,7 @@ fractal_server/app/routes/api/v2/images.py,sha256=R0ccsHMeMyvZ3Sh23zGXyfqniMFOTV
 fractal_server/app/routes/api/v2/job.py,sha256=MU1sHIKk_89WrD0TD44d4ufzqnywot7On_W71KjyUbQ,6500
 fractal_server/app/routes/api/v2/project.py,sha256=hMvL9QLPUcAAiPGy6ta2LBLTVRozJsfvBPl5D06_MHg,6666
 fractal_server/app/routes/api/v2/status_legacy.py,sha256=Q5ZWQNfeZKL8Xgtou2Xr80iaF1uO-r4oSKgq5H42V_8,6349
-fractal_server/app/routes/api/v2/submit.py,sha256=BXoP83bwoahQTWC8NBh4SA-j9SWQAs_KAatgcdvGWIM,8806
+fractal_server/app/routes/api/v2/submit.py,sha256=hCwwC6bXP7EyhgGyVLv1ClybRH1YytDVoPunOzpsf0s,8822
 fractal_server/app/routes/api/v2/task.py,sha256=O7pquZhXIS4lRs5XqHvstiwe8BiCuS-B3ZKJI1g6EJU,6985
 fractal_server/app/routes/api/v2/task_collection.py,sha256=IDNF6sjDuU37HIQ0TuQA-TZIuf7nfHAQXUUNmkrlhLM,12706
 fractal_server/app/routes/api/v2/task_collection_custom.py,sha256=cctW61-C2QYF2KXluS15lLhZJS_kt30Ca6UGLFO32z0,6207
@@ -66,21 +66,19 @@ fractal_server/app/routes/aux/_runner.py,sha256=spNudutueHTBJPhm55RlOuYzb31Dhyhe
 fractal_server/app/routes/aux/validate_user_settings.py,sha256=FLVi__8YFcm_6c_K5uMQo7raWWXQLBcZtx8yaPO4jaE,2301
 fractal_server/app/routes/pagination.py,sha256=L8F5JqekF39qz-LpeScdlhb57MQnSRXjK4ZEtsZqYLk,1210
 fractal_server/app/runner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fractal_server/app/runner/components.py,sha256=ZF8ct_Ky5k8IAcrmpYOZ-bc6OBgdELEighYVqFDEbZg,119
+fractal_server/app/runner/components.py,sha256=-Ii5l8d_V6f5DFOd-Zsr8VYmOsyqw0Hox9fEFQiuqxY,66
 fractal_server/app/runner/compress_folder.py,sha256=HSc1tv7x2DBjBoXwugZlC79rm9GNBIWtQKK9yWn5ZBI,3991
-fractal_server/app/runner/exceptions.py,sha256=_qZ_t8O4umAdJ1ikockiF5rDJuxnEskrGrLjZcnQl7A,4159
+fractal_server/app/runner/exceptions.py,sha256=JC5ufHyeA1hYD_rkZUscI30DD8D903ncag7Z3AArmUY,4215
 fractal_server/app/runner/executors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fractal_server/app/runner/executors/base_runner.py,sha256=v-uUHEvJgRnipTEvU9AmIMkYbkkazqkjm4iAF1GXHEM,4562
+fractal_server/app/runner/executors/base_runner.py,sha256=WhS7MLOkSHy8vjrGZGRHkounziJoYsqUlPkAdUQIs44,4511
 fractal_server/app/runner/executors/local/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fractal_server/app/runner/executors/local/_local_config.py,sha256=8dyg2Gh8L2FlG_jJRYLMkcMgVHGEY2w7DME9aaKXFFo,3688
-fractal_server/app/runner/executors/local/_submit_setup.py,sha256=pDc9Q6axXL8_5JAV0byXzGOLOB0bZF88_L9LZykOgwM,1220
-fractal_server/app/runner/executors/local/runner.py,sha256=LNql8q6M-Cn_hEV4IMkNP57XFPQJ6eaVd0YIDKJLk60,5621
+fractal_server/app/runner/executors/local/get_local_config.py,sha256=wbrIYuGOvABOStrE7jNrC4ULPhtBQ5Q7Y3aKm_icomg,3508
+fractal_server/app/runner/executors/local/runner.py,sha256=KHmG6tw1TCb1HLtUhUC4FYcK_OdJk-SDdXtLkX_G4-4,7939
 fractal_server/app/runner/executors/slurm_common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fractal_server/app/runner/executors/slurm_common/_batching.py,sha256=ZY020JZlDS5mfpgpWTChQkyHU7iLE5kx2HVd57_C6XA,8850
-fractal_server/app/runner/executors/slurm_common/_check_jobs_status.py,sha256=eZd9lxbObsqc1M3B96IGMJ-1oC0jo8lBOX4Nto97VvE,2036
+fractal_server/app/runner/executors/slurm_common/_check_jobs_status.py,sha256=bzpqx8mxT-2xRIGQK6YLMIvMN-SM6OeYWUs97W0iRno,2103
 fractal_server/app/runner/executors/slurm_common/_job_states.py,sha256=nuV-Zba38kDrRESOVB3gaGbrSPZc4q7YGichQaeqTW0,238
 fractal_server/app/runner/executors/slurm_common/_slurm_config.py,sha256=fZaFUUXqDH0p3DndCFUpFqTqyD2tMVCuSYgYLAycpVw,15897
-fractal_server/app/runner/executors/slurm_common/_submit_setup.py,sha256=crbfAAvXbxe_9PaokXkkVdPV65lSCFbInZ0RlT6uyHI,2746
 fractal_server/app/runner/executors/slurm_common/get_slurm_config.py,sha256=-fAX1DZMB5RZnyYanIJD72mWOJAPkh21jd4loDXKJw4,5994
 fractal_server/app/runner/executors/slurm_common/remote.py,sha256=iXLu4d-bWzn7qmDaOjKFkcuaSHLjPESAMSLcg6c99fc,5852
 fractal_server/app/runner/executors/slurm_common/utils_executors.py,sha256=naPyJI0I3lD-sYHbSXbMFGUBK4h_SggA5V91Z1Ch1Xg,1416
@@ -88,28 +86,29 @@ fractal_server/app/runner/executors/slurm_ssh/__init__.py,sha256=47DEQpj8HBSa-_T
 fractal_server/app/runner/executors/slurm_ssh/_check_job_status_ssh.py,sha256=81e4kYpnNlONUw0a3o0o5kTu35Dshelc61y6Gf63m2M,1899
 fractal_server/app/runner/executors/slurm_ssh/_executor_wait_thread.py,sha256=lnW8dNNPqqbpQvojVBQaNJm4wN3Qkw02RWBZ1w68Hyw,3755
 fractal_server/app/runner/executors/slurm_ssh/_slurm_job.py,sha256=IL1C52dezEiincVX2yKryNiPHi4YOMURNLdQO_QPdGw,4406
-fractal_server/app/runner/executors/slurm_ssh/executor.py,sha256=VXMDaQRijYaeKyxly4RkBkRg2inLBZ75rfuODg0Mgr8,53602
-fractal_server/app/runner/executors/slurm_ssh/runner.py,sha256=5T9ITPSr-Cq52XEuDkPhZl2LgdnK_HNyL2h_FP3WZLg,24623
+fractal_server/app/runner/executors/slurm_ssh/executor.py,sha256=StEX6vN9jY79nTxqRDb5OEhkTVd3jYhT4X0_luZSqd4,53678
+fractal_server/app/runner/executors/slurm_ssh/runner.py,sha256=LON7H3RGPYwqmDj0gEcPg7CtBGQzClU8o0fubsXk-Rw,24482
 fractal_server/app/runner/executors/slurm_sudo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fractal_server/app/runner/executors/slurm_sudo/_subprocess_run_as_user.py,sha256=O1bNg1DiSDJmQE0RmOk2Ii47DagiXp5ryd0R6KxO2OM,3177
-fractal_server/app/runner/executors/slurm_sudo/runner.py,sha256=zygSjPCkin3jzABAaHsRunav-4IV2TysUVrwpNqyGKU,23410
+fractal_server/app/runner/executors/slurm_sudo/runner.py,sha256=hjHn09Gvw1xZ-mmSRRaye-0-JInA-YhM45ohCZ_8WYY,27911
 fractal_server/app/runner/extract_archive.py,sha256=tLpjDrX47OjTNhhoWvm6iNukg8KoieWyTb7ZfvE9eWU,2483
 fractal_server/app/runner/filenames.py,sha256=lPnxKHtdRizr6FqG3zOdjDPyWA7GoaJGTtiuJV0gA8E,70
 fractal_server/app/runner/run_subprocess.py,sha256=c3JbYXq3hX2aaflQU19qJ5Xs6J6oXGNvnTEoAfv2bxc,959
 fractal_server/app/runner/set_start_and_last_task_index.py,sha256=-q4zVybAj8ek2XlbENKlfOAJ39hT_zoJoZkqzDqiAMY,1254
 fractal_server/app/runner/shutdown.py,sha256=9pfSKHDNdIcm0eY-opgRTi7y0HmvfPmYiu9JR6Idark,2082
-fractal_server/app/runner/task_files.py,sha256=5enzBqiQct1AUGwrGX-rxFCxnhW3SPYIUylMYwyVfrE,2482
-fractal_server/app/runner/v2/__init__.py,sha256=PTLXNT_ZoEd9GTZ5C4Ai0imOFJOfY4E9Yg1rB-gsl7I,13340
-fractal_server/app/runner/v2/_db_tools.py,sha256=_8Mt4WbItew8EEtRwHdZidfIm4MvfM49FsMPf2JYol4,1361
-fractal_server/app/runner/v2/_local.py,sha256=dcXmsl7zBnrGACMLztfhpnuTjQLQy-tf9SYEHCayCoY,3031
-fractal_server/app/runner/v2/_slurm_ssh.py,sha256=hKjfQLTNINAXZ0jQzYHMdim-NSntPhSnBUPGiymmHuc,3313
-fractal_server/app/runner/v2/_slurm_sudo.py,sha256=oBnTt8RBIKjMDJ4tZuQl0iTO1qQSWfyPvRojViOsoT0,3000
+fractal_server/app/runner/task_files.py,sha256=pGqaFsSXg-3kywf_FGqgjx_8pPLwIn0MpjQ3009fkus,2741
+fractal_server/app/runner/v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fractal_server/app/runner/v2/_local.py,sha256=DK8yagbvd6HHjcDVhUzTy0f7MURlTkQha-NM6OZKgJc,3044
+fractal_server/app/runner/v2/_slurm_ssh.py,sha256=RhfV1quXGGU4bYioRF7UKAdrzOI_stgsFdXdcTegrv0,3310
+fractal_server/app/runner/v2/_slurm_sudo.py,sha256=vjvOA-6_0vSKp704wM1O0ubEDpKZZ0vJG5ZfUi9-wSA,2997
+fractal_server/app/runner/v2/db_tools.py,sha256=yPF2KgkrEdvAc380YL-wux68O0Ej9cY-usTSd-m_evE,2400
 fractal_server/app/runner/v2/deduplicate_list.py,sha256=IVTE4abBU1bUprFTkxrTfYKnvkNTanWQ-KWh_etiT08,645
 fractal_server/app/runner/v2/merge_outputs.py,sha256=D1L4Taieq9i71SPQyNc1kMokgHh-sV_MqF3bv7QMDBc,907
-fractal_server/app/runner/v2/runner.py,sha256=CTm_1vEez5jENcGgCeitfjmMw5m3jiyfg_l4zIDhHFg,15420
-fractal_server/app/runner/v2/runner_functions.py,sha256=DKeijU0XCOoCSL6DRrqmUrmm7MixaWHGN1iLWZBzdb0,22283
-fractal_server/app/runner/v2/runner_functions_low_level.py,sha256=dvvRK7od8iQ8vdPf80uGUxs3i5i0buGjCodBxSjZ7PQ,3671
-fractal_server/app/runner/v2/task_interface.py,sha256=e1GGQSYd0MyBj1EZvEVzqv-HpVE4YffXOq82WLrCaOc,1866
+fractal_server/app/runner/v2/runner.py,sha256=tCBGtUjExhlAplFLw3AlxEr7NZmpAj9QIwSh2xstIO8,15153
+fractal_server/app/runner/v2/runner_functions.py,sha256=xnc1rpuECwUEDuD4oLF7PpyenHUe3kTVD41xjZa2r_0,16490
+fractal_server/app/runner/v2/runner_functions_low_level.py,sha256=9t1CHN3EyfsGRWfG257YPY5WjQ6zuztsw_KZrpEAFPo,3703
+fractal_server/app/runner/v2/submit_workflow.py,sha256=PTLXNT_ZoEd9GTZ5C4Ai0imOFJOfY4E9Yg1rB-gsl7I,13340
+fractal_server/app/runner/v2/task_interface.py,sha256=IXdQTI8rXFgXv1Ez0js4CjKFf3QwO2GCHRTuwiFtiTQ,2891
 fractal_server/app/runner/versions.py,sha256=dSaPRWqmFPHjg20kTCHmi_dmGNcCETflDtDLronNanU,852
 fractal_server/app/schemas/__init__.py,sha256=stURAU_t3AOBaH0HSUbV-GKhlPKngnnIMoqWc3orFyI,135
 fractal_server/app/schemas/_filter_validators.py,sha256=Gkf2USrkuxZx1TWeeMRmhgfmG60AAIDQfbaWslLsvJQ,1572
@@ -210,8 +209,8 @@ fractal_server/tasks/v2/utils_templates.py,sha256=Kc_nSzdlV6KIsO0CQSPs1w70zLyENP
 fractal_server/urls.py,sha256=QjIKAC1a46bCdiPMu3AlpgFbcv6a4l3ABcd5xz190Og,471
 fractal_server/utils.py,sha256=PMwrxWFxRTQRl1b9h-NRIbFGPKqpH_hXnkAT3NfZdpY,3571
 fractal_server/zip_tools.py,sha256=GjDgo_sf6V_DDg6wWeBlZu5zypIxycn_l257p_YVKGc,4876
-fractal_server-2.14.0a10.dist-info/LICENSE,sha256=QKAharUuhxL58kSoLizKJeZE3mTCBnX6ucmz8W0lxlk,1576
-fractal_server-2.14.0a10.dist-info/METADATA,sha256=m0gXC3fObgrIZEYTBQIFmvhRnF29LW9WI2Gcw_X-j48,4563
-fractal_server-2.14.0a10.dist-info/WHEEL,sha256=7dDg4QLnNKTvwIDR9Ac8jJaAmBC_owJrckbC0jjThyA,88
-fractal_server-2.14.0a10.dist-info/entry_points.txt,sha256=8tV2kynvFkjnhbtDnxAqImL6HMVKsopgGfew0DOp5UY,58
-fractal_server-2.14.0a10.dist-info/RECORD,,
+fractal_server-2.14.0a11.dist-info/LICENSE,sha256=QKAharUuhxL58kSoLizKJeZE3mTCBnX6ucmz8W0lxlk,1576
+fractal_server-2.14.0a11.dist-info/METADATA,sha256=_yK06nJ_yK0gkC42b3nVAtSD0_TQTlD7VJwUl9rlzf0,4563
+fractal_server-2.14.0a11.dist-info/WHEEL,sha256=7dDg4QLnNKTvwIDR9Ac8jJaAmBC_owJrckbC0jjThyA,88
+fractal_server-2.14.0a11.dist-info/entry_points.txt,sha256=8tV2kynvFkjnhbtDnxAqImL6HMVKsopgGfew0DOp5UY,58
+fractal_server-2.14.0a11.dist-info/RECORD,,