fractal-server 2.0.6__py3-none-any.whl → 2.2.0__py3-none-any.whl

fractal_server/app/runner/v2/__init__.py

@@ -10,9 +10,11 @@ import traceback
 from pathlib import Path
 from typing import Optional
 
+from sqlalchemy.orm import Session as DBSyncSession
 from sqlalchemy.orm.attributes import flag_modified
 
 from ....config import get_settings
+from ....logger import get_logger
 from ....logger import reset_logger_handlers
 from ....logger import set_logger
 from ....syringe import Inject
@@ -25,8 +27,13 @@ from ...models.v2 import WorkflowV2
 from ...schemas.v2 import JobStatusTypeV2
 from ..exceptions import JobExecutionError
 from ..exceptions import TaskExecutionError
+from ..executors.slurm._subprocess_run_as_user import _mkdir_as_user
 from ..filenames import WORKFLOW_LOG_FILENAME
+from ..task_files import task_subfolder_name
 from ._local import process_workflow as local_process_workflow
+from ._local_experimental import (
+    process_workflow as local_experimental_process_workflow,
+)
 from ._slurm import process_workflow as slurm_process_workflow
 from .handle_failed_job import assemble_filters_failed_job
 from .handle_failed_job import assemble_history_failed_job
@@ -35,9 +42,31 @@ from fractal_server import __VERSION__
 
 _backends = {}
 _backends["local"] = local_process_workflow
+_backends["local_experimental"] = local_experimental_process_workflow
 _backends["slurm"] = slurm_process_workflow
 
 
+def fail_job(
+    *,
+    db: DBSyncSession,
+    job: JobV2,
+    log_msg: str,
+    logger_name: str,
+    emit_log: bool = False,
+) -> None:
+    logger = get_logger(logger_name=logger_name)
+    if emit_log:
+        logger.error(log_msg)
+    reset_logger_handlers(logger)
+    job.status = JobStatusTypeV2.FAILED
+    job.end_timestamp = get_timestamp()
+    job.log = log_msg
+    db.merge(job)
+    db.commit()
+    db.close()
+    return
+
+
 async def submit_workflow(
     *,
     workflow_id: int,
@@ -72,22 +101,34 @@ async def submit_workflow(
             The username to impersonate for the workflow execution, for the
             slurm backend.
     """
-
-    # Declare runner backend and set `process_workflow` function
-    settings = Inject(get_settings)
-    FRACTAL_RUNNER_BACKEND = settings.FRACTAL_RUNNER_BACKEND
-    if FRACTAL_RUNNER_BACKEND == "local":
-        process_workflow = local_process_workflow
-    elif FRACTAL_RUNNER_BACKEND == "slurm":
-        process_workflow = slurm_process_workflow
-    else:
-        raise RuntimeError(f"Invalid runner backend {FRACTAL_RUNNER_BACKEND=}")
+    logger_name = f"WF{workflow_id}_job{job_id}"
+    logger = set_logger(logger_name=logger_name)
 
     with next(DB.get_sync_db()) as db_sync:
 
         job: JobV2 = db_sync.get(JobV2, job_id)
         if not job:
-            raise ValueError(f"Cannot fetch job {job_id} from database")
+            logger.error(f"JobV2 {job_id} does not exist")
+            return
+
+        # Declare runner backend and set `process_workflow` function
+        settings = Inject(get_settings)
+        FRACTAL_RUNNER_BACKEND = settings.FRACTAL_RUNNER_BACKEND
+        if FRACTAL_RUNNER_BACKEND == "local":
+            process_workflow = local_process_workflow
+        elif FRACTAL_RUNNER_BACKEND == "local_experimental":
+            process_workflow = local_experimental_process_workflow
+        elif FRACTAL_RUNNER_BACKEND == "slurm":
+            process_workflow = slurm_process_workflow
+        else:
+            fail_job(
+                db=db_sync,
+                job=job,
+                log_msg=f"Invalid {FRACTAL_RUNNER_BACKEND=}",
+                logger_name=logger_name,
+                emit_log=True,
+            )
+            return
 
         dataset: DatasetV2 = db_sync.get(DatasetV2, dataset_id)
         workflow: WorkflowV2 = db_sync.get(WorkflowV2, workflow_id)
@@ -99,38 +140,75 @@ async def submit_workflow(
                 log_msg += (
                     f"Cannot fetch workflow {workflow_id} from database\n"
                 )
-            job.status = JobStatusTypeV2.FAILED
-            job.end_timestamp = get_timestamp()
-            job.log = log_msg
-            db_sync.merge(job)
-            db_sync.commit()
-            db_sync.close()
+            fail_job(
+                db=db_sync, job=job, log_msg=log_msg, logger_name=logger_name
+            )
             return
 
         # Define and create server-side working folder
-        WORKFLOW_DIR = Path(job.working_dir)
-        if WORKFLOW_DIR.exists():
-            job.status = JobStatusTypeV2.FAILED
-            job.end_timestamp = get_timestamp()
-            job.log = f"Workflow dir {WORKFLOW_DIR} already exists."
-            db_sync.merge(job)
-            db_sync.commit()
-            db_sync.close()
+        WORKFLOW_DIR_LOCAL = Path(job.working_dir)
+        if WORKFLOW_DIR_LOCAL.exists():
+            fail_job(
+                db=db_sync,
+                job=job,
+                log_msg=f"Workflow dir {WORKFLOW_DIR_LOCAL} already exists.",
+                logger_name=logger_name,
+                emit_log=True,
+            )
             return
 
-        # Create WORKFLOW_DIR with 755 permissions
-        original_umask = os.umask(0)
-        WORKFLOW_DIR.mkdir(parents=True, mode=0o755)
-        os.umask(original_umask)
+        try:
 
-        # Define and create user-side working folder, if needed
-        WORKFLOW_DIR_USER = Path(job.working_dir_user)
-        if FRACTAL_RUNNER_BACKEND == "slurm":
-            from ..executors.slurm._subprocess_run_as_user import (
-                _mkdir_as_user,
-            )
+            # Create WORKFLOW_DIR
+            original_umask = os.umask(0)
+            WORKFLOW_DIR_LOCAL.mkdir(parents=True, mode=0o755)
+
+            os.umask(original_umask)
 
-            _mkdir_as_user(folder=str(WORKFLOW_DIR_USER), user=slurm_user)
+            # Define and create WORKFLOW_DIR_REMOTE
+            if FRACTAL_RUNNER_BACKEND == "local":
+                WORKFLOW_DIR_REMOTE = WORKFLOW_DIR_LOCAL
+            elif FRACTAL_RUNNER_BACKEND == "local_experimental":
+                WORKFLOW_DIR_REMOTE = WORKFLOW_DIR_LOCAL
+            elif FRACTAL_RUNNER_BACKEND == "slurm":
+                WORKFLOW_DIR_REMOTE = (
+                    Path(user_cache_dir) / WORKFLOW_DIR_LOCAL.name
+                )
+                _mkdir_as_user(
+                    folder=str(WORKFLOW_DIR_REMOTE), user=slurm_user
+                )
+
+            # Create all tasks subfolders
+            for order in range(job.first_task_index, job.last_task_index + 1):
+                this_wftask = workflow.task_list[order]
+                if this_wftask.is_legacy_task:
+                    task_name = this_wftask.task_legacy.name
+                else:
+                    task_name = this_wftask.task.name
+                subfolder_name = task_subfolder_name(
+                    order=order,
+                    task_name=task_name,
+                )
+                original_umask = os.umask(0)
+                (WORKFLOW_DIR_LOCAL / subfolder_name).mkdir(mode=0o755)
+                os.umask(original_umask)
+                if FRACTAL_RUNNER_BACKEND == "slurm":
+                    _mkdir_as_user(
+                        folder=str(WORKFLOW_DIR_REMOTE / subfolder_name),
+                        user=slurm_user,
+                    )
+        except Exception as e:
+            fail_job(
+                db=db_sync,
+                job=job,
+                log_msg=(
+                    "An error occurred while creating job folder and "
+                    f"subfolders.\nOriginal error: {str(e)}"
+                ),
+                logger_name=logger_name,
+                emit_log=True,
+            )
+            return
 
         # After Session.commit() is called, either explicitly or when using a
         # context manager, all objects associated with the Session are expired.
@@ -145,10 +223,11 @@ async def submit_workflow(
 
         db_sync.refresh(dataset)
         db_sync.refresh(workflow)
+        for wftask in workflow.task_list:
+            db_sync.refresh(wftask)
 
         # Write logs
-        logger_name = f"WF{workflow_id}_job{job_id}"
-        log_file_path = WORKFLOW_DIR / WORKFLOW_LOG_FILENAME
+        log_file_path = WORKFLOW_DIR_LOCAL / WORKFLOW_LOG_FILENAME
         logger = set_logger(
             logger_name=logger_name,
             log_file_path=log_file_path,
@@ -189,8 +268,8 @@ async def submit_workflow(
             slurm_user=slurm_user,
             slurm_account=job.slurm_account,
             user_cache_dir=user_cache_dir,
-            workflow_dir=WORKFLOW_DIR,
-            workflow_dir_user=WORKFLOW_DIR_USER,
+            workflow_dir_local=WORKFLOW_DIR_LOCAL,
+            workflow_dir_remote=WORKFLOW_DIR_REMOTE,
             logger_name=logger_name,
             worker_init=worker_init,
             first_task_index=job.first_task_index,
@@ -243,18 +322,14 @@ async def submit_workflow(
             dataset.images = latest_images
         db_sync.merge(dataset)
 
-        job.status = JobStatusTypeV2.FAILED
-        job.end_timestamp = get_timestamp()
-
         exception_args_string = "\n".join(e.args)
-        job.log = (
+        log_msg = (
             f"TASK ERROR: "
             f"Task name: {e.task_name}, "
             f"position in Workflow: {e.workflow_task_order}\n"
             f"TRACEBACK:\n{exception_args_string}"
         )
-        db_sync.merge(job)
-        db_sync.commit()
+        fail_job(db=db_sync, job=job, log_msg=log_msg, logger_name=logger_name)
 
     except JobExecutionError as e:
 
@@ -277,12 +352,15 @@ async def submit_workflow(
             dataset.images = latest_images
         db_sync.merge(dataset)
 
-        job.status = JobStatusTypeV2.FAILED
-        job.end_timestamp = get_timestamp()
-        error = e.assemble_error()
-        job.log = f"JOB ERROR in Fractal job {job.id}:\nTRACEBACK:\n{error}"
-        db_sync.merge(job)
-        db_sync.commit()
+        fail_job(
+            db=db_sync,
+            job=job,
+            log_msg=(
+                f"JOB ERROR in Fractal job {job.id}:\n"
+                f"TRACEBACK:\n{e.assemble_error()}"
+            ),
+            logger_name=logger_name,
+        )
 
     except Exception:
 
@@ -306,15 +384,16 @@ async def submit_workflow(
         if latest_images is not None:
             dataset.images = latest_images
         db_sync.merge(dataset)
-
-        job.status = JobStatusTypeV2.FAILED
-        job.end_timestamp = get_timestamp()
-        job.log = (
-            f"UNKNOWN ERROR in Fractal job {job.id}\n"
-            f"TRACEBACK:\n{current_traceback}"
+        fail_job(
+            db=db_sync,
+            job=job,
+            log_msg=(
+                f"UNKNOWN ERROR in Fractal job {job.id}\n"
+                f"TRACEBACK:\n{current_traceback}"
+            ),
+            logger_name=logger_name,
         )
-        db_sync.merge(job)
-        db_sync.commit()
+
     finally:
         reset_logger_handlers(logger)
         db_sync.close()

fractal_server/app/runner/v2/_local/__init__.py

@@ -36,7 +36,7 @@ def _process_workflow(
     workflow: WorkflowV2,
     dataset: DatasetV2,
     logger_name: str,
-    workflow_dir: Path,
+    workflow_dir_local: Path,
     first_task_index: int,
     last_task_index: int,
 ) -> dict:
@@ -57,8 +57,8 @@ def _process_workflow(
             ],  # noqa
             dataset=dataset,
             executor=executor,
-            workflow_dir=workflow_dir,
-            workflow_dir_user=workflow_dir,
+            workflow_dir_local=workflow_dir_local,
+            workflow_dir_remote=workflow_dir_local,
             logger_name=logger_name,
             submit_setup_call=_local_submit_setup,
         )
@@ -69,8 +69,8 @@ async def process_workflow(
     *,
     workflow: WorkflowV2,
     dataset: DatasetV2,
-    workflow_dir: Path,
-    workflow_dir_user: Optional[Path] = None,
+    workflow_dir_local: Path,
+    workflow_dir_remote: Optional[Path] = None,
     first_task_index: Optional[int] = None,
     last_task_index: Optional[int] = None,
     logger_name: str,
@@ -94,12 +94,13 @@ async def process_workflow(
             The workflow to be run
         dataset:
             Initial dataset.
-        workflow_dir:
+        workflow_dir_local:
             Working directory for this run.
-        workflow_dir_user:
+        workflow_dir_remote:
             Working directory for this run, on the user side. This argument is
             present for compatibility with the standard backend interface, but
-            for the `local` backend it cannot be different from `workflow_dir`.
+            for the `local` backend it cannot be different from
+            `workflow_dir_local`.
         first_task_index:
             Positional index of the first task to execute; if `None`, start
             from `0`.
@@ -137,10 +138,10 @@ async def process_workflow(
             of the workflow
     """
 
-    if workflow_dir_user and (workflow_dir_user != workflow_dir):
+    if workflow_dir_remote and (workflow_dir_remote != workflow_dir_local):
         raise NotImplementedError(
             "Local backend does not support different directories "
-            f"{workflow_dir=} and {workflow_dir_user=}"
+            f"{workflow_dir_local=} and {workflow_dir_remote=}"
         )
 
     # Set values of first_task_index and last_task_index
@@ -155,7 +156,7 @@ async def process_workflow(
         workflow=workflow,
         dataset=dataset,
         logger_name=logger_name,
-        workflow_dir=workflow_dir,
+        workflow_dir_local=workflow_dir_local,
         first_task_index=first_task_index,
         last_task_index=last_task_index,
     )

fractal_server/app/runner/v2/_local/_local_config.py

@@ -96,7 +96,7 @@ def get_local_backend_config(
         )
 
     if wftask_meta and key in wftask_meta:
-        parallel_tasks_per_job = wftask.meta[key]
+        parallel_tasks_per_job = wftask_meta[key]
     else:
         if not config_path:
             settings = Inject(get_settings)

fractal_server/app/runner/v2/_local/_submit_setup.py

@@ -22,8 +22,8 @@ from ._local_config import get_local_backend_config
 def _local_submit_setup(
     *,
     wftask: WorkflowTaskV2,
-    workflow_dir: Optional[Path] = None,
-    workflow_dir_user: Optional[Path] = None,
+    workflow_dir_local: Optional[Path] = None,
+    workflow_dir_remote: Optional[Path] = None,
     which_type: Literal["non_parallel", "parallel"],
 ) -> dict[str, object]:
     """
@@ -33,9 +33,9 @@ def _local_submit_setup(
     Arguments:
         wftask:
             WorkflowTask for which the configuration is to be assembled
-        workflow_dir:
+        workflow_dir_local:
             Not used in this function.
-        workflow_dir_user:
+        workflow_dir_remote:
             Not used in this function.
 
     Returns:

fractal_server/app/runner/v2/_local_experimental/__init__.py

@@ -0,0 +1,155 @@
+from concurrent.futures.process import BrokenProcessPool
+from pathlib import Path
+from typing import Optional
+
+from ....models.v2 import DatasetV2
+from ....models.v2 import WorkflowV2
+from ...async_wrap import async_wrap
+from ...exceptions import JobExecutionError
+from ...filenames import SHUTDOWN_FILENAME
+from ...set_start_and_last_task_index import set_start_and_last_task_index
+from ..runner import execute_tasks_v2
+from ._submit_setup import _local_submit_setup
+from .executor import FractalProcessPoolExecutor
+
+
+def _process_workflow(
+    *,
+    workflow: WorkflowV2,
+    dataset: DatasetV2,
+    logger_name: str,
+    workflow_dir_local: Path,
+    first_task_index: int,
+    last_task_index: int,
+) -> dict:
+    """
+    Internal processing routine
+
+    Schedules the workflow using a `FractalProcessPoolExecutor`.
+
+    Cf.
+    [process_workflow][fractal_server.app.runner.v2._local_experimental.process_workflow]
+    for the call signature.
+    """
+    with FractalProcessPoolExecutor(
+        shutdown_file=workflow_dir_local / SHUTDOWN_FILENAME
+    ) as executor:
+        try:
+            new_dataset_attributes = execute_tasks_v2(
+                wf_task_list=workflow.task_list[
+                    first_task_index : (last_task_index + 1)  # noqa
+                ],
+                dataset=dataset,
+                executor=executor,
+                workflow_dir_local=workflow_dir_local,
+                workflow_dir_remote=workflow_dir_local,
+                logger_name=logger_name,
+                submit_setup_call=_local_submit_setup,
+            )
+        except BrokenProcessPool as e:
+            raise JobExecutionError(
+                info=(
+                    "Job failed with BrokenProcessPool error, likely due to "
+                    f"an executor shutdown.\nOriginal error:\n{e.args[0]}"
+                )
+            )
+
+    return new_dataset_attributes
+
+
+async def process_workflow(
+    *,
+    workflow: WorkflowV2,
+    dataset: DatasetV2,
+    workflow_dir_local: Path,
+    workflow_dir_remote: Optional[Path] = None,
+    first_task_index: Optional[int] = None,
+    last_task_index: Optional[int] = None,
+    logger_name: str,
+    # Slurm-specific
+    user_cache_dir: Optional[str] = None,
+    slurm_user: Optional[str] = None,
+    slurm_account: Optional[str] = None,
+    worker_init: Optional[str] = None,
+) -> dict:
+    """
+    Run a workflow
+
+    This function is responsible for running a workflow on some input data,
+    saving the output and taking care of any exception raised during the run.
+
+    NOTE: This is the `local_experimental` backend's public interface,
+    which also works as a reference implementation for other backends.
+
+    Args:
+        workflow:
+            The workflow to be run
+        dataset:
+            Initial dataset.
+        workflow_dir_local:
+            Working directory for this run.
+        workflow_dir_remote:
+            Working directory for this run, on the user side. This argument is
+            present for compatibility with the standard backend interface, but
+            for the `local` backend it cannot be different from
+            `workflow_dir_local`.
+        first_task_index:
+            Positional index of the first task to execute; if `None`, start
+            from `0`.
+        last_task_index:
+            Positional index of the last task to execute; if `None`, proceed
+            until the last task.
+        logger_name: Logger name
+        slurm_user:
+            Username to impersonate to run the workflow. This argument is
+            present for compatibility with the standard backend interface, but
+            is ignored in the `local` backend.
+        slurm_account:
+            SLURM account to use when running the workflow. This argument is
+            present for compatibility with the standard backend interface, but
+            is ignored in the `local` backend.
+        user_cache_dir:
+            Cache directory of the user who will run the workflow. This
+            argument is present for compatibility with the standard backend
+            interface, but is ignored in the `local` backend.
+        worker_init:
+            Any additional, usually backend specific, information to be passed
+            to the backend executor. This argument is present for compatibility
+            with the standard backend interface, but is ignored in the `local`
+            backend.
+
+    Raises:
+        TaskExecutionError: wrapper for errors raised during tasks' execution
+                            (positive exit codes).
+        JobExecutionError: wrapper for errors raised by the tasks' executors
+                           (negative exit codes).
+
+    Returns:
+        output_dataset_metadata:
+            The updated metadata for the dataset, as returned by the last task
+            of the workflow
+    """
+
+    if workflow_dir_remote and (workflow_dir_remote != workflow_dir_local):
+        raise NotImplementedError(
+            "LocalExperimental backend does not support different directories "
+            f"{workflow_dir_local=} and {workflow_dir_remote=}"
+        )
+
+    # Set values of first_task_index and last_task_index
+    num_tasks = len(workflow.task_list)
+    first_task_index, last_task_index = set_start_and_last_task_index(
+        num_tasks,
+        first_task_index=first_task_index,
+        last_task_index=last_task_index,
+    )
+
+    new_dataset_attributes = await async_wrap(_process_workflow)(
+        workflow=workflow,
+        dataset=dataset,
+        logger_name=logger_name,
+        workflow_dir_local=workflow_dir_local,
+        first_task_index=first_task_index,
+        last_task_index=last_task_index,
+    )
+    return new_dataset_attributes

fractal_server/app/runner/v2/_local_experimental/_local_config.py

@@ -0,0 +1,108 @@
+"""
+Submodule to handle the local-backend configuration for a WorkflowTask
+"""
+import json
+from pathlib import Path
+from typing import Literal
+from typing import Optional
+
+from pydantic import BaseModel
+from pydantic import Extra
+from pydantic.error_wrappers import ValidationError
+
+from .....config import get_settings
+from .....syringe import Inject
+from ....models.v2 import WorkflowTaskV2
+
+
+class LocalBackendConfigError(ValueError):
+    """
+    Local-backend configuration error
+    """
+
+    pass
+
+
+class LocalBackendConfig(BaseModel, extra=Extra.forbid):
+    """
+    Specifications of the local-backend configuration
+
+    Attributes:
+        parallel_tasks_per_job:
+            Maximum number of tasks to be run in parallel as part of a call to
+            `FractalProcessPoolExecutor.map`; if `None`, then all tasks will
+            start at the same time.
+    """
+
+    parallel_tasks_per_job: Optional[int]
+
+
+def get_default_local_backend_config():
+    """
+    Return a default `LocalBackendConfig` configuration object
+    """
+    return LocalBackendConfig(parallel_tasks_per_job=None)
+
+
+def get_local_backend_config(
+    wftask: WorkflowTaskV2,
+    which_type: Literal["non_parallel", "parallel"],
+    config_path: Optional[Path] = None,
+) -> LocalBackendConfig:
+    """
+    Prepare a `LocalBackendConfig` configuration object
+
+    The sources for `parallel_tasks_per_job` attributes, starting from the
+    highest-priority one, are
+
+    1. Properties in `wftask.meta_parallel` or `wftask.meta_non_parallel`
+       (depending on `which_type`);
+    2. The general content of the local-backend configuration file;
+    3. The default value (`None`).
+
+    Arguments:
+        wftask:
+            WorkflowTaskV2 for which the backend configuration should
+            be prepared.
+        config_path:
+            Path of local-backend configuration file; if `None`, use
+            `FRACTAL_LOCAL_CONFIG_FILE` variable from settings.
+
+    Returns:
+        A local-backend configuration object
+    """
+
+    key = "parallel_tasks_per_job"
+    default_value = None
+
+    if which_type == "non_parallel":
+        wftask_meta = wftask.meta_non_parallel
+    elif which_type == "parallel":
+        wftask_meta = wftask.meta_parallel
+    else:
+        raise ValueError(
+            "`get_local_backend_config` received an invalid argument"
+            f" {which_type=}."
+        )
+
+    if wftask_meta and key in wftask_meta:
+        parallel_tasks_per_job = wftask_meta[key]
+    else:
+        if not config_path:
+            settings = Inject(get_settings)
+            config_path = settings.FRACTAL_LOCAL_CONFIG_FILE
+        if config_path is None:
+            parallel_tasks_per_job = default_value
+        else:
+            with config_path.open("r") as f:
+                env = json.load(f)
+            try:
+                _ = LocalBackendConfig(**env)
+            except ValidationError as e:
+                raise LocalBackendConfigError(
+                    f"Error while loading {config_path=}. "
+                    f"Original error:\n{str(e)}"
+                )
+
+            parallel_tasks_per_job = env.get(key, default_value)
+    return LocalBackendConfig(parallel_tasks_per_job=parallel_tasks_per_job)