fractal-server 2.2.0a0__py3-none-any.whl → 2.3.0a0__py3-none-any.whl

fractal_server/app/runner/v2/__init__.py

@@ -5,14 +5,18 @@ This module is the single entry point to the runner backend subsystem V2.
 Other subystems should only import this module and not its submodules or
 the individual backends.
 """
+import logging
 import os
 import traceback
 from pathlib import Path
 from typing import Optional
 
+from fabric import Connection  # FIXME SSH: try/except import
+from sqlalchemy.orm import Session as DBSyncSession
 from sqlalchemy.orm.attributes import flag_modified
 
 from ....config import get_settings
+from ....logger import get_logger
 from ....logger import reset_logger_handlers
 from ....logger import set_logger
 from ....syringe import Inject
@@ -25,14 +29,15 @@ from ...models.v2 import WorkflowV2
 from ...schemas.v2 import JobStatusTypeV2
 from ..exceptions import JobExecutionError
 from ..exceptions import TaskExecutionError
-from ..executors.slurm._subprocess_run_as_user import _mkdir_as_user
+from ..executors.slurm.sudo._subprocess_run_as_user import _mkdir_as_user
 from ..filenames import WORKFLOW_LOG_FILENAME
 from ..task_files import task_subfolder_name
 from ._local import process_workflow as local_process_workflow
 from ._local_experimental import (
     process_workflow as local_experimental_process_workflow,
 )
-from ._slurm import process_workflow as slurm_process_workflow
+from ._slurm import process_workflow as slurm_sudo_process_workflow
+from ._slurm_ssh import process_workflow as slurm_ssh_process_workflow
 from .handle_failed_job import assemble_filters_failed_job
 from .handle_failed_job import assemble_history_failed_job
 from .handle_failed_job import assemble_images_failed_job
@@ -40,8 +45,30 @@ from fractal_server import __VERSION__
 
 _backends = {}
 _backends["local"] = local_process_workflow
+_backends["slurm"] = slurm_sudo_process_workflow
+_backends["slurm_ssh"] = slurm_ssh_process_workflow
 _backends["local_experimental"] = local_experimental_process_workflow
-_backends["slurm"] = slurm_process_workflow
+
+
+def fail_job(
+    *,
+    db: DBSyncSession,
+    job: JobV2,
+    log_msg: str,
+    logger_name: str,
+    emit_log: bool = False,
+) -> None:
+    logger = get_logger(logger_name=logger_name)
+    if emit_log:
+        logger.error(log_msg)
+    reset_logger_handlers(logger)
+    job.status = JobStatusTypeV2.FAILED
+    job.end_timestamp = get_timestamp()
+    job.log = log_msg
+    db.merge(job)
+    db.commit()
+    db.close()
+    return
 
 
 async def submit_workflow(
@@ -52,6 +79,7 @@ async def submit_workflow(
     worker_init: Optional[str] = None,
     slurm_user: Optional[str] = None,
     user_cache_dir: Optional[str] = None,
+    connection: Optional[Connection] = None,
 ) -> None:
     """
     Prepares a workflow and applies it to a dataset
@@ -78,24 +106,36 @@ async def submit_workflow(
             The username to impersonate for the workflow execution, for the
             slurm backend.
     """
-
     # Declare runner backend and set `process_workflow` function
     settings = Inject(get_settings)
     FRACTAL_RUNNER_BACKEND = settings.FRACTAL_RUNNER_BACKEND
-    if FRACTAL_RUNNER_BACKEND == "local":
-        process_workflow = local_process_workflow
-    elif FRACTAL_RUNNER_BACKEND == "local_experimental":
-        process_workflow = local_experimental_process_workflow
-    elif FRACTAL_RUNNER_BACKEND == "slurm":
-        process_workflow = slurm_process_workflow
-    else:
-        raise RuntimeError(f"Invalid runner backend {FRACTAL_RUNNER_BACKEND=}")
+    logger_name = f"WF{workflow_id}_job{job_id}"
+    logger = set_logger(logger_name=logger_name)
 
     with next(DB.get_sync_db()) as db_sync:
 
         job: JobV2 = db_sync.get(JobV2, job_id)
         if not job:
-            raise ValueError(f"Cannot fetch job {job_id} from database")
+            logger.error(f"JobV2 {job_id} does not exist")
+            return
+
+        # Declare runner backend and set `process_workflow` function
+        settings = Inject(get_settings)
+        FRACTAL_RUNNER_BACKEND = settings.FRACTAL_RUNNER_BACKEND
+        try:
+            process_workflow = _backends[settings.FRACTAL_RUNNER_BACKEND]
+        except KeyError as e:
+            fail_job(
+                db=db_sync,
+                job=job,
+                log_msg=(
+                    f"Invalid {FRACTAL_RUNNER_BACKEND=}.\n"
+                    f"Original KeyError: {str(e)}"
+                ),
+                logger_name=logger_name,
+                emit_log=True,
+            )
+            return
 
         dataset: DatasetV2 = db_sync.get(DatasetV2, dataset_id)
         workflow: WorkflowV2 = db_sync.get(WorkflowV2, workflow_id)
@@ -107,61 +147,96 @@ async def submit_workflow(
                 log_msg += (
                     f"Cannot fetch workflow {workflow_id} from database\n"
                 )
-            job.status = JobStatusTypeV2.FAILED
-            job.end_timestamp = get_timestamp()
-            job.log = log_msg
-            db_sync.merge(job)
-            db_sync.commit()
-            db_sync.close()
+            fail_job(
+                db=db_sync, job=job, log_msg=log_msg, logger_name=logger_name
+            )
             return
 
         # Define and create server-side working folder
         WORKFLOW_DIR_LOCAL = Path(job.working_dir)
         if WORKFLOW_DIR_LOCAL.exists():
-            job.status = JobStatusTypeV2.FAILED
-            job.end_timestamp = get_timestamp()
-            job.log = f"Workflow dir {WORKFLOW_DIR_LOCAL} already exists."
-            db_sync.merge(job)
-            db_sync.commit()
-            db_sync.close()
+            fail_job(
+                db=db_sync,
+                job=job,
+                log_msg=f"Workflow dir {WORKFLOW_DIR_LOCAL} already exists.",
+                logger_name=logger_name,
+                emit_log=True,
+            )
             return
 
-        # Create WORKFLOW_DIR
-        original_umask = os.umask(0)
-        WORKFLOW_DIR_LOCAL.mkdir(parents=True, mode=0o755)
-        os.umask(original_umask)
-
-        # Define and create WORKFLOW_DIR_REMOTE
-        if FRACTAL_RUNNER_BACKEND == "local":
-            WORKFLOW_DIR_REMOTE = WORKFLOW_DIR_LOCAL
-        elif FRACTAL_RUNNER_BACKEND == "local_experimental":
-            WORKFLOW_DIR_REMOTE = WORKFLOW_DIR_LOCAL
-        elif FRACTAL_RUNNER_BACKEND == "slurm":
-            WORKFLOW_DIR_REMOTE = (
-                Path(user_cache_dir) / WORKFLOW_DIR_LOCAL.name
-            )
-            _mkdir_as_user(folder=str(WORKFLOW_DIR_REMOTE), user=slurm_user)
+        try:
 
-        # Create all tasks subfolders
-        for order in range(job.first_task_index, job.last_task_index + 1):
-            this_wftask = workflow.task_list[order]
-            if this_wftask.is_legacy_task:
-                task_name = this_wftask.task_legacy.name
-            else:
-                task_name = this_wftask.task.name
-            subfolder_name = task_subfolder_name(
-                order=order,
-                task_name=task_name,
-            )
+            # Create WORKFLOW_DIR_LOCAL
             original_umask = os.umask(0)
-            (WORKFLOW_DIR_LOCAL / subfolder_name).mkdir(mode=0o755)
+            WORKFLOW_DIR_LOCAL.mkdir(parents=True, mode=0o755)
             os.umask(original_umask)
-            if FRACTAL_RUNNER_BACKEND == "slurm":
+
+            # Define and create WORKFLOW_DIR_REMOTE
+            if FRACTAL_RUNNER_BACKEND == "local":
+                WORKFLOW_DIR_REMOTE = WORKFLOW_DIR_LOCAL
+            elif FRACTAL_RUNNER_BACKEND == "local_experimental":
+                WORKFLOW_DIR_REMOTE = WORKFLOW_DIR_LOCAL
+            elif FRACTAL_RUNNER_BACKEND == "slurm":
+                WORKFLOW_DIR_REMOTE = (
+                    Path(user_cache_dir) / WORKFLOW_DIR_LOCAL.name
+                )
                 _mkdir_as_user(
-                    folder=str(WORKFLOW_DIR_REMOTE / subfolder_name),
-                    user=slurm_user,
+                    folder=str(WORKFLOW_DIR_REMOTE), user=slurm_user
+                )
+            elif FRACTAL_RUNNER_BACKEND == "slurm_ssh":
+                WORKFLOW_DIR_REMOTE = (
+                    Path(settings.FRACTAL_SLURM_SSH_WORKING_BASE_DIR)
+                    / WORKFLOW_DIR_LOCAL.name
+                )
+                # FIXME SSH: move mkdir to executor, likely within handshake
+
+                from ....ssh._fabric import _mkdir_over_ssh
+
+                _mkdir_over_ssh(
+                    folder=str(WORKFLOW_DIR_REMOTE), connection=connection
+                )
+                logging.info(f"Created {str(WORKFLOW_DIR_REMOTE)} via SSH.")
+            else:
+                logging.error(
+                    "Invalid FRACTAL_RUNNER_BACKEND="
+                    f"{settings.FRACTAL_RUNNER_BACKEND}."
                 )
 
+            # Create all tasks subfolders
+            for order in range(job.first_task_index, job.last_task_index + 1):
+                this_wftask = workflow.task_list[order]
+                if this_wftask.is_legacy_task:
+                    task_name = this_wftask.task_legacy.name
+                else:
+                    task_name = this_wftask.task.name
+                subfolder_name = task_subfolder_name(
+                    order=order,
+                    task_name=task_name,
+                )
+                original_umask = os.umask(0)
+                (WORKFLOW_DIR_LOCAL / subfolder_name).mkdir(mode=0o755)
+                os.umask(original_umask)
+                if FRACTAL_RUNNER_BACKEND == "slurm":
+                    _mkdir_as_user(
+                        folder=str(WORKFLOW_DIR_REMOTE / subfolder_name),
+                        user=slurm_user,
+                    )
+                else:
+                    logging.info("Skip remote-subfolder creation")
+        except Exception as e:
+            error_type = type(e).__name__
+            fail_job(
+                db=db_sync,
+                job=job,
+                log_msg=(
+                    f"{error_type} error occurred while creating job folder "
+                    f"and subfolders.\nOriginal error: {str(e)}"
+                ),
+                logger_name=logger_name,
+                emit_log=True,
+            )
+            return
+
         # After Session.commit() is called, either explicitly or when using a
         # context manager, all objects associated with the Session are expired.
         # https://docs.sqlalchemy.org/en/14/orm/
@@ -179,7 +254,6 @@ async def submit_workflow(
             db_sync.refresh(wftask)
 
         # Write logs
-        logger_name = f"WF{workflow_id}_job{job_id}"
         log_file_path = WORKFLOW_DIR_LOCAL / WORKFLOW_LOG_FILENAME
         logger = set_logger(
             logger_name=logger_name,
@@ -191,9 +265,17 @@ async def submit_workflow(
         )
         logger.debug(f"fractal_server.__VERSION__: {__VERSION__}")
         logger.debug(f"FRACTAL_RUNNER_BACKEND: {FRACTAL_RUNNER_BACKEND}")
-        logger.debug(f"slurm_user: {slurm_user}")
-        logger.debug(f"slurm_account: {job.slurm_account}")
-        logger.debug(f"worker_init: {worker_init}")
+        if FRACTAL_RUNNER_BACKEND == "slurm":
+            logger.debug(f"slurm_user: {slurm_user}")
+            logger.debug(f"slurm_account: {job.slurm_account}")
+            logger.debug(f"worker_init: {worker_init}")
+        elif FRACTAL_RUNNER_BACKEND == "slurm_ssh":
+            logger.debug(f"ssh_host: {settings.FRACTAL_SLURM_SSH_HOST}")
+            logger.debug(f"ssh_user: {settings.FRACTAL_SLURM_SSH_USER}")
+            logger.debug(
+                f"base dir: {settings.FRACTAL_SLURM_SSH_WORKING_BASE_DIR}"
+            )
+            logger.debug(f"worker_init: {worker_init}")
         logger.debug(f"job.id: {job.id}")
         logger.debug(f"job.working_dir: {job.working_dir}")
         logger.debug(f"job.working_dir_user: {job.working_dir_user}")
@@ -202,6 +284,27 @@ async def submit_workflow(
         logger.debug(f'START workflow "{workflow.name}"')
 
     try:
+        if FRACTAL_RUNNER_BACKEND == "local":
+            process_workflow = local_process_workflow
+            backend_specific_kwargs = {}
+        elif FRACTAL_RUNNER_BACKEND == "local_experimental":
+            process_workflow = local_experimental_process_workflow
+            backend_specific_kwargs = {}
+        elif FRACTAL_RUNNER_BACKEND == "slurm":
+            process_workflow = slurm_sudo_process_workflow
+            backend_specific_kwargs = dict(
+                slurm_user=slurm_user,
+                slurm_account=job.slurm_account,
+                user_cache_dir=user_cache_dir,
+            )
+        elif FRACTAL_RUNNER_BACKEND == "slurm_ssh":
+            process_workflow = slurm_ssh_process_workflow
+            backend_specific_kwargs = dict(connection=connection)
+        else:
+            raise RuntimeError(
+                f"Invalid runner backend {FRACTAL_RUNNER_BACKEND=}"
+            )
+
         # "The Session.close() method does not prevent the Session from being
         # used again. The Session itself does not actually have a distinct
         # “closed” state; it merely means the Session will release all database
@@ -218,15 +321,13 @@ async def submit_workflow(
         new_dataset_attributes = await process_workflow(
             workflow=workflow,
             dataset=dataset,
-            slurm_user=slurm_user,
-            slurm_account=job.slurm_account,
-            user_cache_dir=user_cache_dir,
             workflow_dir_local=WORKFLOW_DIR_LOCAL,
             workflow_dir_remote=WORKFLOW_DIR_REMOTE,
             logger_name=logger_name,
             worker_init=worker_init,
             first_task_index=job.first_task_index,
             last_task_index=job.last_task_index,
+            **backend_specific_kwargs,
         )
 
         logger.info(
@@ -275,18 +376,14 @@ async def submit_workflow(
             dataset.images = latest_images
         db_sync.merge(dataset)
 
-        job.status = JobStatusTypeV2.FAILED
-        job.end_timestamp = get_timestamp()
-
         exception_args_string = "\n".join(e.args)
-        job.log = (
+        log_msg = (
             f"TASK ERROR: "
             f"Task name: {e.task_name}, "
             f"position in Workflow: {e.workflow_task_order}\n"
             f"TRACEBACK:\n{exception_args_string}"
         )
-        db_sync.merge(job)
-        db_sync.commit()
+        fail_job(db=db_sync, job=job, log_msg=log_msg, logger_name=logger_name)
 
     except JobExecutionError as e:
 
@@ -309,12 +406,15 @@ async def submit_workflow(
             dataset.images = latest_images
         db_sync.merge(dataset)
 
-        job.status = JobStatusTypeV2.FAILED
-        job.end_timestamp = get_timestamp()
-        error = e.assemble_error()
-        job.log = f"JOB ERROR in Fractal job {job.id}:\nTRACEBACK:\n{error}"
-        db_sync.merge(job)
-        db_sync.commit()
+        fail_job(
+            db=db_sync,
+            job=job,
+            log_msg=(
+                f"JOB ERROR in Fractal job {job.id}:\n"
+                f"TRACEBACK:\n{e.assemble_error()}"
+            ),
+            logger_name=logger_name,
+        )
 
     except Exception:
 
@@ -338,15 +438,16 @@ async def submit_workflow(
         if latest_images is not None:
             dataset.images = latest_images
         db_sync.merge(dataset)
-
-        job.status = JobStatusTypeV2.FAILED
-        job.end_timestamp = get_timestamp()
-        job.log = (
-            f"UNKNOWN ERROR in Fractal job {job.id}\n"
-            f"TRACEBACK:\n{current_traceback}"
+        fail_job(
+            db=db_sync,
+            job=job,
+            log_msg=(
+                f"UNKNOWN ERROR in Fractal job {job.id}\n"
+                f"TRACEBACK:\n{current_traceback}"
+            ),
+            logger_name=logger_name,
         )
-        db_sync.merge(job)
-        db_sync.commit()
+
     finally:
         reset_logger_handlers(logger)
         db_sync.close()

fractal_server/app/runner/v2/_local_experimental/__init__.py

@@ -1,9 +1,11 @@
+from concurrent.futures.process import BrokenProcessPool
 from pathlib import Path
 from typing import Optional
 
 from ....models.v2 import DatasetV2
 from ....models.v2 import WorkflowV2
 from ...async_wrap import async_wrap
+from ...exceptions import JobExecutionError
 from ...filenames import SHUTDOWN_FILENAME
 from ...set_start_and_last_task_index import set_start_and_last_task_index
 from ..runner import execute_tasks_v2
@@ -29,21 +31,29 @@ def _process_workflow(
     [process_workflow][fractal_server.app.runner.v2._local_experimental.process_workflow]
     for the call signature.
     """
-
     with FractalProcessPoolExecutor(
         shutdown_file=workflow_dir_local / SHUTDOWN_FILENAME
     ) as executor:
-        new_dataset_attributes = execute_tasks_v2(
-            wf_task_list=workflow.task_list[
-                first_task_index : (last_task_index + 1)  # noqa
-            ],  # noqa
-            dataset=dataset,
-            executor=executor,
-            workflow_dir_local=workflow_dir_local,
-            workflow_dir_remote=workflow_dir_local,
-            logger_name=logger_name,
-            submit_setup_call=_local_submit_setup,
-        )
+        try:
+            new_dataset_attributes = execute_tasks_v2(
+                wf_task_list=workflow.task_list[
+                    first_task_index : (last_task_index + 1)  # noqa
+                ],
+                dataset=dataset,
+                executor=executor,
+                workflow_dir_local=workflow_dir_local,
+                workflow_dir_remote=workflow_dir_local,
+                logger_name=logger_name,
+                submit_setup_call=_local_submit_setup,
+            )
+        except BrokenProcessPool as e:
+            raise JobExecutionError(
+                info=(
+                    "Job failed with BrokenProcessPool error, likely due to "
+                    f"an executor shutdown.\nOriginal error:\n{e.args[0]}"
+                )
+            )
+
     return new_dataset_attributes
 
 

fractal_server/app/runner/v2/_local_experimental/executor.py

@@ -2,8 +2,6 @@
 Custom version of Python
 [ProcessPoolExecutor](https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ProcessPoolExecutor)).
 """
-import os
-import signal
 import threading
 import time
 from concurrent.futures import ProcessPoolExecutor
@@ -14,13 +12,14 @@ from typing import Iterable
 from typing import Optional
 from typing import Sequence
 
+import psutil
+
 from ._local_config import get_default_local_backend_config
 from ._local_config import LocalBackendConfig
 from fractal_server.app.runner.exceptions import JobExecutionError
-from fractal_server.logger import get_logger
-
+from fractal_server.logger import set_logger
 
-logger = get_logger("FractalProcessPoolExecutor")
+logger = set_logger("FractalProcessPoolExecutor")
 
 
 class FractalProcessPoolExecutor(ProcessPoolExecutor):
@@ -66,12 +65,17 @@ class FractalProcessPoolExecutor(ProcessPoolExecutor):
         """
         Running on '_shutdown_file_thread'.
         """
+
         logger.info("Start terminating FractalProcessPoolExecutor processes.")
+        # We use 'psutil' in order to easily access the PIDs of the children.
         if self._processes is not None:
             for pid in self._processes.keys():
-                logger.debug(f"Sending SIGTERM to process {pid}")
-                os.kill(pid, signal.SIGTERM)
-                logger.debug(f"Process {pid} terminated.")
+                parent = psutil.Process(pid)
+                children = parent.children(recursive=True)
+                for child in children:
+                    child.kill()
+                parent.kill()
+                logger.info(f"Process {pid} and its children terminated.")
         logger.info("FractalProcessPoolExecutor processes terminated.")
 
     def shutdown(self, *args, **kwargs) -> None:

fractal_server/app/runner/v2/_slurm/__init__.py

@@ -24,16 +24,11 @@ from typing import Union
 from ....models.v2 import DatasetV2
 from ....models.v2 import WorkflowV2
 from ...async_wrap import async_wrap
-from ...executors.slurm.executor import FractalSlurmExecutor
+from ...executors.slurm.sudo.executor import FractalSlurmExecutor
 from ...set_start_and_last_task_index import set_start_and_last_task_index
 from ..runner import execute_tasks_v2
 from ._submit_setup import _slurm_submit_setup
 
-# from .._common import execute_tasks
-# from ..common import async_wrap
-# from ..common import set_start_and_last_task_index
-# from ..common import TaskParameters
-
 
 def _process_workflow(
     *,

fractal_server/app/runner/v2/_slurm_ssh/__init__.py

@@ -0,0 +1,126 @@
+# Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
+# University of Zurich
+#
+# Original authors:
+# Jacopo Nespolo <jacopo.nespolo@exact-lab.it>
+# Tommaso Comparin <tommaso.comparin@exact-lab.it>
+# Marco Franzon <marco.franzon@exact-lab.it>
+#
+# This file is part of Fractal and was originally developed by eXact lab S.r.l.
+# <exact-lab.it> under contract with Liberali Lab from the Friedrich Miescher
+# Institute for Biomedical Research and Pelkmans Lab from the University of
+# Zurich.
+"""
+Slurm Bakend
+
+This backend runs fractal workflows in a SLURM cluster using Clusterfutures
+Executor objects.
+"""
+from pathlib import Path
+from typing import Any
+from typing import Optional
+from typing import Union
+
+from fabric import Connection
+
+from ....models.v2 import DatasetV2
+from ....models.v2 import WorkflowV2
+from ...async_wrap import async_wrap
+from ...executors.slurm.ssh.executor import FractalSlurmSSHExecutor
+from ...set_start_and_last_task_index import set_start_and_last_task_index
+from ..runner import execute_tasks_v2
+from ._submit_setup import _slurm_submit_setup
+
+
+def _process_workflow(
+    *,
+    workflow: WorkflowV2,
+    dataset: DatasetV2,
+    logger_name: str,
+    workflow_dir_local: Path,
+    workflow_dir_remote: Path,
+    first_task_index: int,
+    last_task_index: int,
+    connection: Connection,
+    worker_init: Optional[Union[str, list[str]]] = None,
+) -> dict[str, Any]:
+    """
+    Internal processing routine for the SLURM backend
+
+    This function initialises the a FractalSlurmExecutor, setting logging,
+    workflow working dir and user to impersonate. It then schedules the
+    workflow tasks and returns the new dataset attributes
+
+    Cf.
+    [process_workflow][fractal_server.app.runner.v2._local.process_workflow]
+
+    Returns:
+        new_dataset_attributes:
+    """
+
+    if isinstance(worker_init, str):
+        worker_init = worker_init.split("\n")
+
+    with FractalSlurmSSHExecutor(
+        connection=connection,
+        workflow_dir_local=workflow_dir_local,
+        workflow_dir_remote=workflow_dir_remote,
+        common_script_lines=worker_init,
+    ) as executor:
+        new_dataset_attributes = execute_tasks_v2(
+            wf_task_list=workflow.task_list[
+                first_task_index : (last_task_index + 1)  # noqa
+            ],  # noqa
+            dataset=dataset,
+            executor=executor,
+            workflow_dir_local=workflow_dir_local,
+            workflow_dir_remote=workflow_dir_remote,
+            logger_name=logger_name,
+            submit_setup_call=_slurm_submit_setup,
+        )
+    return new_dataset_attributes
+
+
+async def process_workflow(
+    *,
+    workflow: WorkflowV2,
+    dataset: DatasetV2,
+    workflow_dir_local: Path,
+    workflow_dir_remote: Optional[Path] = None,
+    first_task_index: Optional[int] = None,
+    last_task_index: Optional[int] = None,
+    logger_name: str,
+    # Not used
+    connection: Connection,
+    user_cache_dir: Optional[str] = None,
+    slurm_user: Optional[str] = None,
+    slurm_account: Optional[str] = None,
+    worker_init: Optional[str] = None,
+) -> dict:
+    """
+    Process workflow (SLURM backend public interface)
+
+    Cf.
+    [process_workflow][fractal_server.app.runner.v2._local.process_workflow]
+    """
+
+    # Set values of first_task_index and last_task_index
+    num_tasks = len(workflow.task_list)
+    first_task_index, last_task_index = set_start_and_last_task_index(
+        num_tasks,
+        first_task_index=first_task_index,
+        last_task_index=last_task_index,
+    )
+
+    new_dataset_attributes = await async_wrap(_process_workflow)(
+        workflow=workflow,
+        dataset=dataset,
+        logger_name=logger_name,
+        workflow_dir_local=workflow_dir_local,
+        workflow_dir_remote=workflow_dir_remote,
+        first_task_index=first_task_index,
+        last_task_index=last_task_index,
+        worker_init=worker_init,
+        connection=connection,
+    )
+    return new_dataset_attributes