fractal-server 1.4.10__py3-none-any.whl → 2.0.0a0__py3-none-any.whl

fractal_server/app/runner/v2/__init__.py

@@ -0,0 +1,337 @@
+"""
+Runner backend subsystem root V2
+
+This module is the single entry point to the runner backend subsystem V2.
+Other subystems should only import this module and not its submodules or
+the individual backends.
+"""
+import os
+import traceback
+from pathlib import Path
+from typing import Optional
+
+from sqlalchemy.orm.attributes import flag_modified
+
+from ....config import get_settings
+from ....logger import close_logger
+from ....logger import set_logger
+from ....syringe import Inject
+from ....utils import get_timestamp
+from ...db import DB
+from ...models.v2 import DatasetV2
+from ...models.v2 import JobV2
+from ...models.v2 import WorkflowTaskV2
+from ...models.v2 import WorkflowV2
+from ...schemas.v2 import JobStatusTypeV2
+from ..exceptions import JobExecutionError
+from ..exceptions import TaskExecutionError
+from ..filenames import WORKFLOW_LOG_FILENAME
+from ._local import process_workflow as local_process_workflow
+from ._slurm import process_workflow as slurm_process_workflow
+from .handle_failed_job import assemble_filters_failed_job
+from .handle_failed_job import assemble_history_failed_job
+from .handle_failed_job import assemble_images_failed_job
+from .runner import execute_tasks_v2  # noqa
+from fractal_server import __VERSION__
+
+_backends = {}
+_backends["local"] = local_process_workflow
+_backends["slurm"] = slurm_process_workflow
+
+
+async def submit_workflow(
+    *,
+    workflow_id: int,
+    dataset_id: int,
+    job_id: int,
+    worker_init: Optional[str] = None,
+    slurm_user: Optional[str] = None,
+    user_cache_dir: Optional[str] = None,
+) -> None:
+    """
+    Prepares a workflow and applies it to a dataset
+
+    This function wraps the process_workflow one, which is different for each
+    backend (e.g. local or slurm backend).
+
+    Args:
+        workflow_id:
+            ID of the workflow being applied
+        dataset_id:
+            Dataset ID
+        job_id:
+            Id of the job record which stores the state for the current
+            workflow application.
+        worker_init:
+            Custom executor parameters that get parsed before the execution of
+            each task.
+        user_cache_dir:
+            Cache directory (namely a path where the user can write); for the
+            slurm backend, this is used as a base directory for
+            `job.working_dir_user`.
+        slurm_user:
+            The username to impersonate for the workflow execution, for the
+            slurm backend.
+    """
+
+    # Declare runner backend and set `process_workflow` function
+    settings = Inject(get_settings)
+    FRACTAL_RUNNER_BACKEND = settings.FRACTAL_RUNNER_BACKEND
+    if FRACTAL_RUNNER_BACKEND == "local":
+        process_workflow = local_process_workflow
+    elif FRACTAL_RUNNER_BACKEND == "slurm":
+        process_workflow = slurm_process_workflow
+    else:
+        raise RuntimeError(f"Invalid runner backend {FRACTAL_RUNNER_BACKEND=}")
+
+    with next(DB.get_sync_db()) as db_sync:
+
+        job: JobV2 = db_sync.get(JobV2, job_id)
+        if not job:
+            raise ValueError(f"Cannot fetch job {job_id} from database")
+
+        dataset: DatasetV2 = db_sync.get(DatasetV2, dataset_id)
+        workflow: WorkflowV2 = db_sync.get(WorkflowV2, workflow_id)
+        if not (dataset and workflow):
+            log_msg = ""
+            if not dataset:
+                log_msg += f"Cannot fetch dataset {dataset_id} from database\n"
+            if not workflow:
+                log_msg += (
+                    f"Cannot fetch workflow {workflow_id} from database\n"
+                )
+            job.status = JobStatusTypeV2.FAILED
+            job.end_timestamp = get_timestamp()
+            job.log = log_msg
+            db_sync.merge(job)
+            db_sync.commit()
+            db_sync.close()
+            return
+
+        # Define and create server-side working folder
+        project_id = workflow.project_id
+        timestamp_string = get_timestamp().strftime("%Y%m%d_%H%M%S")
+        WORKFLOW_DIR = (
+            settings.FRACTAL_RUNNER_WORKING_BASE_DIR
+            / (
+                f"proj_{project_id:07d}_wf_{workflow_id:07d}_job_{job_id:07d}"
+                f"_{timestamp_string}"
+            )
+        ).resolve()
+
+        if WORKFLOW_DIR.exists():
+            raise RuntimeError(f"Workflow dir {WORKFLOW_DIR} already exists.")
+
+        # Create WORKFLOW_DIR with 755 permissions
+        original_umask = os.umask(0)
+        WORKFLOW_DIR.mkdir(parents=True, mode=0o755)
+        os.umask(original_umask)
+
+        # Define and create user-side working folder, if needed
+        if FRACTAL_RUNNER_BACKEND == "local":
+            WORKFLOW_DIR_USER = WORKFLOW_DIR
+        elif FRACTAL_RUNNER_BACKEND == "slurm":
+
+            from ..executors.slurm._subprocess_run_as_user import (
+                _mkdir_as_user,
+            )
+
+            WORKFLOW_DIR_USER = (
+                Path(user_cache_dir) / f"{WORKFLOW_DIR.name}"
+            ).resolve()
+            _mkdir_as_user(folder=str(WORKFLOW_DIR_USER), user=slurm_user)
+        else:
+            raise ValueError(f"{FRACTAL_RUNNER_BACKEND=} not supported")
+
+        # Update db
+        job.working_dir = WORKFLOW_DIR.as_posix()
+        job.working_dir_user = WORKFLOW_DIR_USER.as_posix()
+        db_sync.merge(job)
+        db_sync.commit()
+
+        # After Session.commit() is called, either explicitly or when using a
+        # context manager, all objects associated with the Session are expired.
+        # https://docs.sqlalchemy.org/en/14/orm/
+        #   session_basics.html#opening-and-closing-a-session
+        # https://docs.sqlalchemy.org/en/14/orm/
+        #   session_state_management.html#refreshing-expiring
+
+        # See issue #928:
+        # https://github.com/fractal-analytics-platform/
+        #   fractal-server/issues/928
+
+        db_sync.refresh(dataset)
+        db_sync.refresh(workflow)
+
+        # Write logs
+        logger_name = f"WF{workflow_id}_job{job_id}"
+        log_file_path = WORKFLOW_DIR / WORKFLOW_LOG_FILENAME
+        logger = set_logger(
+            logger_name=logger_name,
+            log_file_path=log_file_path,
+        )
+        logger.info(
+            f'Start execution of workflow "{workflow.name}"; '
+            f"more logs at {str(log_file_path)}"
+        )
+        logger.debug(f"fractal_server.__VERSION__: {__VERSION__}")
+        logger.debug(f"FRACTAL_RUNNER_BACKEND: {FRACTAL_RUNNER_BACKEND}")
+        logger.debug(f"slurm_user: {slurm_user}")
+        logger.debug(f"slurm_account: {job.slurm_account}")
+        logger.debug(f"worker_init: {worker_init}")
+        logger.debug(f"job.id: {job.id}")
+        logger.debug(f"job.working_dir: {job.working_dir}")
+        logger.debug(f"job.working_dir_user: {job.working_dir_user}")
+        logger.debug(f"job.first_task_index: {job.first_task_index}")
+        logger.debug(f"job.last_task_index: {job.last_task_index}")
+        logger.debug(f'START workflow "{workflow.name}"')
+
+    try:
+        # "The Session.close() method does not prevent the Session from being
+        # used again. The Session itself does not actually have a distinct
+        # “closed” state; it merely means the Session will release all database
+        # connections and ORM objects."
+        # (https://docs.sqlalchemy.org/en/20/orm/session_api.html#sqlalchemy.orm.Session.close).
+        #
+        # We close the session before the (possibly long) process_workflow
+        # call, to make sure all DB connections are released. The reason why we
+        # are not using a context manager within the try block is that we also
+        # need access to db_sync in the except branches.
+        db_sync = next(DB.get_sync_db())
+        db_sync.close()
+
+        new_dataset_attributes = await process_workflow(
+            workflow=workflow,
+            dataset=dataset,
+            slurm_user=slurm_user,
+            slurm_account=job.slurm_account,
+            user_cache_dir=user_cache_dir,
+            workflow_dir=WORKFLOW_DIR,
+            workflow_dir_user=WORKFLOW_DIR_USER,
+            logger_name=logger_name,
+            worker_init=worker_init,
+            first_task_index=job.first_task_index,
+            last_task_index=job.last_task_index,
+        )
+
+        logger.info(
+            f'End execution of workflow "{workflow.name}"; '
+            f"more logs at {str(log_file_path)}"
+        )
+        logger.debug(f'END workflow "{workflow.name}"')
+
+        # Update dataset attributes, in case of successful execution
+        dataset.history.extend(new_dataset_attributes["history"])
+        dataset.filters = new_dataset_attributes["filters"]
+        dataset.images = new_dataset_attributes["images"]
+        for attribute_name in ["filters", "history", "images"]:
+            flag_modified(dataset, attribute_name)
+        db_sync.merge(dataset)
+
+        # Update job DB entry
+        job.status = JobStatusTypeV2.DONE
+        job.end_timestamp = get_timestamp()
+        with log_file_path.open("r") as f:
+            logs = f.read()
+        job.log = logs
+        db_sync.merge(job)
+        db_sync.commit()
+
+    except TaskExecutionError as e:
+
+        logger.debug(f'FAILED workflow "{workflow.name}", TaskExecutionError.')
+        logger.info(f'Workflow "{workflow.name}" failed (TaskExecutionError).')
+
+        # Read dataset attributes produced by the last successful task, and
+        # update the DB dataset accordingly
+        failed_wftask = db_sync.get(WorkflowTaskV2, e.workflow_task_id)
+        dataset.history = assemble_history_failed_job(
+            job,
+            dataset,
+            workflow,
+            logger,
+            failed_wftask=failed_wftask,
+        )
+        latest_filters = assemble_filters_failed_job(job)
+        if latest_filters is not None:
+            dataset.filters = latest_filters
+        latest_images = assemble_images_failed_job(job)
+        if latest_images is not None:
+            dataset.images = latest_images
+        db_sync.merge(dataset)
+
+        job.status = JobStatusTypeV2.FAILED
+        job.end_timestamp = get_timestamp()
+
+        exception_args_string = "\n".join(e.args)
+        job.log = (
+            f"TASK ERROR: "
+            f"Task name: {e.task_name}, "
+            f"position in Workflow: {e.workflow_task_order}\n"
+            f"TRACEBACK:\n{exception_args_string}"
+        )
+        db_sync.merge(job)
+        db_sync.commit()
+
+    except JobExecutionError as e:
+
+        logger.debug(f'FAILED workflow "{workflow.name}", JobExecutionError.')
+        logger.info(f'Workflow "{workflow.name}" failed (JobExecutionError).')
+
+        # Read dataset attributes produced by the last successful task, and
+        # update the DB dataset accordingly
+        dataset.history = assemble_history_failed_job(
+            job,
+            dataset,
+            workflow,
+            logger,
+        )
+        latest_filters = assemble_filters_failed_job(job)
+        if latest_filters is not None:
+            dataset.filters = latest_filters
+        latest_images = assemble_images_failed_job(job)
+        if latest_images is not None:
+            dataset.images = latest_images
+        db_sync.merge(dataset)
+
+        job.status = JobStatusTypeV2.FAILED
+        job.end_timestamp = get_timestamp()
+        error = e.assemble_error()
+        job.log = f"JOB ERROR in Fractal job {job.id}:\nTRACEBACK:\n{error}"
+        db_sync.merge(job)
+        db_sync.commit()
+
+    except Exception:
+
+        logger.debug(f'FAILED workflow "{workflow.name}", unknown error.')
+        logger.info(f'Workflow "{workflow.name}" failed (unkwnon error).')
+
+        current_traceback = traceback.format_exc()
+
+        # Read dataset attributes produced by the last successful task, and
+        # update the DB dataset accordingly
+        dataset.history = assemble_history_failed_job(
+            job,
+            dataset,
+            workflow,
+            logger,
+        )
+        latest_filters = assemble_filters_failed_job(job)
+        if latest_filters is not None:
+            dataset.filters = latest_filters
+        latest_images = assemble_images_failed_job(job)
+        if latest_images is not None:
+            dataset.images = latest_images
+        db_sync.merge(dataset)
+
+        job.status = JobStatusTypeV2.FAILED
+        job.end_timestamp = get_timestamp()
+        job.log = (
+            f"UNKNOWN ERROR in Fractal job {job.id}\n"
+            f"TRACEBACK:\n{current_traceback}"
+        )
+        db_sync.merge(job)
+        db_sync.commit()
+    finally:
+        close_logger(logger)
+        db_sync.close()

fractal_server/app/runner/v2/_local/__init__.py

@@ -0,0 +1,169 @@
+# Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
+# University of Zurich
+#
+# Original authors:
+# Jacopo Nespolo <jacopo.nespolo@exact-lab.it>
+# Tommaso Comparin <tommaso.comparin@exact-lab.it>
+# Marco Franzon <marco.franzon@exact-lab.it>
+#
+# This file is part of Fractal and was originally developed by eXact lab S.r.l.
+# <exact-lab.it> under contract with Liberali Lab from the Friedrich Miescher
+# Institute for Biomedical Research and Pelkmans Lab from the University of
+# Zurich.
+"""
+Local Bakend
+
+This backend runs Fractal workflows using `FractalThreadPoolExecutor` (a custom
+version of Python
+[ThreadPoolExecutor](https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor))
+to run tasks in several threads.
+Incidentally, it also represents the reference implementation for a backend.
+"""
+from pathlib import Path
+from typing import Optional
+
+from ....models.v2 import DatasetV2
+from ....models.v2 import WorkflowV2
+from ...async_wrap import async_wrap
+from ...executors.local.executor import FractalThreadPoolExecutor
+from ...set_start_and_last_task_index import set_start_and_last_task_index
+from ..runner import execute_tasks_v2
+from ._submit_setup import _local_submit_setup
+
+# from typing import Any
+
+
+def _process_workflow(
+    *,
+    workflow: WorkflowV2,
+    dataset: DatasetV2,
+    logger_name: str,
+    workflow_dir: Path,
+    first_task_index: int,
+    last_task_index: int,
+) -> dict:
+    """
+    Internal processing routine
+
+    Schedules the workflow using a `FractalThreadPoolExecutor`.
+
+    Cf. [process_workflow][fractal_server.app.runner._local.process_workflow]
+    for the call signature.
+    """
+
+    with FractalThreadPoolExecutor() as executor:
+        new_dataset_attributes = execute_tasks_v2(
+            wf_task_list=workflow.task_list[
+                first_task_index : (last_task_index + 1)  # noqa
+            ],  # noqa
+            dataset=dataset,
+            executor=executor,
+            workflow_dir=workflow_dir,
+            workflow_dir_user=workflow_dir,
+            logger_name=logger_name,
+            submit_setup_call=_local_submit_setup,
+        )
+    return new_dataset_attributes
+
+
+async def process_workflow(
+    *,
+    workflow: WorkflowV2,
+    dataset: DatasetV2,
+    workflow_dir: Path,
+    workflow_dir_user: Optional[Path] = None,
+    first_task_index: Optional[int] = None,
+    last_task_index: Optional[int] = None,
+    logger_name: str,
+    # Slurm-specific
+    user_cache_dir: Optional[str] = None,
+    slurm_user: Optional[str] = None,
+    slurm_account: Optional[str] = None,
+    worker_init: Optional[str] = None,
+) -> dict:
+    """
+    Run a workflow
+
+    This function is responsible for running a workflow on some input data,
+    saving the output and taking care of any exception raised during the run.
+
+    NOTE: This is the `local` backend's public interface, which also works as
+    a reference implementation for other backends.
+
+    Args:
+        workflow:
+            The workflow to be run
+        input_paths:
+            The paths to the input files to pass to the first task of the
+            workflow
+        output_path:
+            The destination path for the last task of the workflow
+        input_metadata:
+            Initial metadata, passed to the first task
+        logger_name:
+            Name of the logger to log information on the run to
+        workflow_dir:
+            Working directory for this run.
+        workflow_dir_user:
+            Working directory for this run, on the user side. This argument is
+            present for compatibility with the standard backend interface, but
+            for the `local` backend it cannot be different from `workflow_dir`.
+        slurm_user:
+            Username to impersonate to run the workflow. This argument is
+            present for compatibility with the standard backend interface, but
+            is ignored in the `local` backend.
+        slurm_account:
+            SLURM account to use when running the workflow. This argument is
+            present for compatibility with the standard backend interface, but
+            is ignored in the `local` backend.
+        user_cache_dir:
+            Cache directory of the user who will run the workflow. This
+            argument is present for compatibility with the standard backend
+            interface, but is ignored in the `local` backend.
+        worker_init:
+            Any additional, usually backend specific, information to be passed
+            to the backend executor. This argument is present for compatibility
+            with the standard backend interface, but is ignored in the `local`
+            backend.
+        first_task_index:
+            Positional index of the first task to execute; if `None`, start
+            from `0`.
+        last_task_index:
+            Positional index of the last task to execute; if `None`, proceed
+            until the last task.
+
+    Raises:
+        TaskExecutionError: wrapper for errors raised during tasks' execution
+                            (positive exit codes).
+        JobExecutionError: wrapper for errors raised by the tasks' executors
+                           (negative exit codes).
+
+    Returns:
+        output_dataset_metadata:
+            The updated metadata for the dataset, as returned by the last task
+            of the workflow
+    """
+
+    if workflow_dir_user and (workflow_dir_user != workflow_dir):
+        raise NotImplementedError(
+            "Local backend does not support different directories "
+            f"{workflow_dir=} and {workflow_dir_user=}"
+        )
+
+    # Set values of first_task_index and last_task_index
+    num_tasks = len(workflow.task_list)
+    first_task_index, last_task_index = set_start_and_last_task_index(
+        num_tasks,
+        first_task_index=first_task_index,
+        last_task_index=last_task_index,
+    )
+
+    new_dataset_attributes = await async_wrap(_process_workflow)(
+        workflow=workflow,
+        dataset=dataset,
+        logger_name=logger_name,
+        workflow_dir=workflow_dir,
+        first_task_index=first_task_index,
+        last_task_index=last_task_index,
+    )
+    return new_dataset_attributes

fractal_server/app/runner/v2/_local/_local_config.py

@@ -0,0 +1,118 @@
+# Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
+# University of Zurich
+#
+# Original authors:
+# Tommaso Comparin <tommaso.comparin@exact-lab.it>
+#
+# This file is part of Fractal and was originally developed by eXact lab S.r.l.
+# <exact-lab.it> under contract with Liberali Lab from the Friedrich Miescher
+# Institute for Biomedical Research and Pelkmans Lab from the University of
+# Zurich.
+"""
+Submodule to handle the local-backend configuration for a WorkflowTask
+"""
+import json
+from pathlib import Path
+from typing import Literal
+from typing import Optional
+
+from pydantic import BaseModel
+from pydantic import Extra
+from pydantic.error_wrappers import ValidationError
+
+from .....config import get_settings
+from .....syringe import Inject
+from ....models.v2 import WorkflowTaskV2
+
+
+class LocalBackendConfigError(ValueError):
+    """
+    Local-backend configuration error
+    """
+
+    pass
+
+
+class LocalBackendConfig(BaseModel, extra=Extra.forbid):
+    """
+    Specifications of the local-backend configuration
+
+    Attributes:
+        parallel_tasks_per_job:
+            Maximum number of tasks to be run in parallel as part of a call to
+            `FractalThreadPoolExecutor.map`; if `None`, then all tasks will
+            start at the same time.
+    """
+
+    parallel_tasks_per_job: Optional[int]
+
+
+def get_default_local_backend_config():
+    """
+    Return a default `LocalBackendConfig` configuration object
+    """
+    return LocalBackendConfig(parallel_tasks_per_job=None)
+
+
+def get_local_backend_config(
+    wftask: WorkflowTaskV2,
+    which_type: Literal["non_parallel", "parallel"],
+    config_path: Optional[Path] = None,
+) -> LocalBackendConfig:
+    """
+    Prepare a `LocalBackendConfig` configuration object
+
+    The sources for `parallel_tasks_per_job` attributes, starting from the
+    highest-priority one, are
+
+    1. Properties in `wftask.meta_parallel` or `wftask.meta_non_parallel`
+       (depending on `which_type`);
+    2. The general content of the local-backend configuration file;
+    3. The default value (`None`).
+
+    Arguments:
+        wftask:
+            WorkflowTaskV2 for which the backend configuration should
+            be prepared.
+        config_path:
+            Path of local-backend configuration file; if `None`, use
+            `FRACTAL_LOCAL_CONFIG_FILE` variable from settings.
+
+    Returns:
+        A local-backend configuration object
+    """
+
+    key = "parallel_tasks_per_job"
+    default_value = None
+
+    if which_type == "non_parallel":
+        wftask_meta = wftask.meta_non_parallel
+    elif which_type == "parallel":
+        wftask_meta = wftask.meta_parallel
+    else:
+        raise ValueError(
+            "`get_local_backend_config` received an invalid argument"
+            f" {which_type=}."
+        )
+
+    if wftask_meta and key in wftask_meta:
+        parallel_tasks_per_job = wftask.meta[key]
+    else:
+        if not config_path:
+            settings = Inject(get_settings)
+            config_path = settings.FRACTAL_LOCAL_CONFIG_FILE
+        if config_path is None:
+            parallel_tasks_per_job = default_value
+        else:
+            with config_path.open("r") as f:
+                env = json.load(f)
+            try:
+                _ = LocalBackendConfig(**env)
+            except ValidationError as e:
+                raise LocalBackendConfigError(
+                    f"Error while loading {config_path=}. "
+                    f"Original error:\n{str(e)}"
+                )
+
+            parallel_tasks_per_job = env.get(key, default_value)
+    return LocalBackendConfig(parallel_tasks_per_job=parallel_tasks_per_job)

fractal_server/app/runner/v2/_local/_submit_setup.py

@@ -0,0 +1,52 @@
+# Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
+# University of Zurich
+#
+# Original authors:
+# Tommaso Comparin <tommaso.comparin@exact-lab.it>
+#
+# This file is part of Fractal and was originally developed by eXact lab S.r.l.
+# <exact-lab.it> under contract with Liberali Lab from the Friedrich Miescher
+# Institute for Biomedical Research and Pelkmans Lab from the University of
+# Zurich.
+"""
+Submodule to define _local_submit_setup
+"""
+from pathlib import Path
+from typing import Literal
+from typing import Optional
+
+from ....models.v2 import WorkflowTaskV2
+from ._local_config import get_local_backend_config
+
+
+def _local_submit_setup(
+    *,
+    wftask: WorkflowTaskV2,
+    workflow_dir: Optional[Path] = None,
+    workflow_dir_user: Optional[Path] = None,
+    which_type: Literal["non_parallel", "parallel"],
+) -> dict[str, object]:
+    """
+    Collect WorfklowTask-specific configuration parameters from different
+    sources, and inject them for execution.
+
+    Arguments:
+        wftask:
+            WorkflowTask for which the configuration is to be assembled
+        workflow_dir:
+            Not used in this function.
+        workflow_dir_user:
+            Not used in this function.
+
+    Returns:
+        submit_setup_dict:
+            A dictionary that will be passed on to
+            `FractalThreadPoolExecutor.submit` and
+            `FractalThreadPoolExecutor.map`, so as to set extra options.
+    """
+
+    local_backend_config = get_local_backend_config(
+        wftask=wftask, which_type=which_type
+    )
+
+    return dict(local_backend_config=local_backend_config)