fractal-server 1.4.9__py3-none-any.whl → 2.0.0a0__py3-none-any.whl

fractal_server/app/runner/v2/_local/__init__.py

@@ -0,0 +1,169 @@
+# Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
+# University of Zurich
+#
+# Original authors:
+# Jacopo Nespolo <jacopo.nespolo@exact-lab.it>
+# Tommaso Comparin <tommaso.comparin@exact-lab.it>
+# Marco Franzon <marco.franzon@exact-lab.it>
+#
+# This file is part of Fractal and was originally developed by eXact lab S.r.l.
+# <exact-lab.it> under contract with Liberali Lab from the Friedrich Miescher
+# Institute for Biomedical Research and Pelkmans Lab from the University of
+# Zurich.
+"""
+Local Bakend
+
+This backend runs Fractal workflows using `FractalThreadPoolExecutor` (a custom
+version of Python
+[ThreadPoolExecutor](https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor))
+to run tasks in several threads.
+Incidentally, it also represents the reference implementation for a backend.
+"""
+from pathlib import Path
+from typing import Optional
+
+from ....models.v2 import DatasetV2
+from ....models.v2 import WorkflowV2
+from ...async_wrap import async_wrap
+from ...executors.local.executor import FractalThreadPoolExecutor
+from ...set_start_and_last_task_index import set_start_and_last_task_index
+from ..runner import execute_tasks_v2
+from ._submit_setup import _local_submit_setup
+
+# from typing import Any
+
+
+def _process_workflow(
+    *,
+    workflow: WorkflowV2,
+    dataset: DatasetV2,
+    logger_name: str,
+    workflow_dir: Path,
+    first_task_index: int,
+    last_task_index: int,
+) -> dict:
+    """
+    Internal processing routine
+
+    Schedules the workflow using a `FractalThreadPoolExecutor`.
+
+    Cf. [process_workflow][fractal_server.app.runner._local.process_workflow]
+    for the call signature.
+    """
+
+    with FractalThreadPoolExecutor() as executor:
+        new_dataset_attributes = execute_tasks_v2(
+            wf_task_list=workflow.task_list[
+                first_task_index : (last_task_index + 1)  # noqa
+            ],  # noqa
+            dataset=dataset,
+            executor=executor,
+            workflow_dir=workflow_dir,
+            workflow_dir_user=workflow_dir,
+            logger_name=logger_name,
+            submit_setup_call=_local_submit_setup,
+        )
+    return new_dataset_attributes
+
+
+async def process_workflow(
+    *,
+    workflow: WorkflowV2,
+    dataset: DatasetV2,
+    workflow_dir: Path,
+    workflow_dir_user: Optional[Path] = None,
+    first_task_index: Optional[int] = None,
+    last_task_index: Optional[int] = None,
+    logger_name: str,
+    # Slurm-specific
+    user_cache_dir: Optional[str] = None,
+    slurm_user: Optional[str] = None,
+    slurm_account: Optional[str] = None,
+    worker_init: Optional[str] = None,
+) -> dict:
+    """
+    Run a workflow
+
+    This function is responsible for running a workflow on some input data,
+    saving the output and taking care of any exception raised during the run.
+
+    NOTE: This is the `local` backend's public interface, which also works as
+    a reference implementation for other backends.
+
+    Args:
+        workflow:
+            The workflow to be run
+        input_paths:
+            The paths to the input files to pass to the first task of the
+            workflow
+        output_path:
+            The destination path for the last task of the workflow
+        input_metadata:
+            Initial metadata, passed to the first task
+        logger_name:
+            Name of the logger to log information on the run to
+        workflow_dir:
+            Working directory for this run.
+        workflow_dir_user:
+            Working directory for this run, on the user side. This argument is
+            present for compatibility with the standard backend interface, but
+            for the `local` backend it cannot be different from `workflow_dir`.
+        slurm_user:
+            Username to impersonate to run the workflow. This argument is
+            present for compatibility with the standard backend interface, but
+            is ignored in the `local` backend.
+        slurm_account:
+            SLURM account to use when running the workflow. This argument is
+            present for compatibility with the standard backend interface, but
+            is ignored in the `local` backend.
+        user_cache_dir:
+            Cache directory of the user who will run the workflow. This
+            argument is present for compatibility with the standard backend
+            interface, but is ignored in the `local` backend.
+        worker_init:
+            Any additional, usually backend specific, information to be passed
+            to the backend executor. This argument is present for compatibility
+            with the standard backend interface, but is ignored in the `local`
+            backend.
+        first_task_index:
+            Positional index of the first task to execute; if `None`, start
+            from `0`.
+        last_task_index:
+            Positional index of the last task to execute; if `None`, proceed
+            until the last task.
+
+    Raises:
+        TaskExecutionError: wrapper for errors raised during tasks' execution
+                            (positive exit codes).
+        JobExecutionError: wrapper for errors raised by the tasks' executors
+                           (negative exit codes).
+
+    Returns:
+        output_dataset_metadata:
+            The updated metadata for the dataset, as returned by the last task
+            of the workflow
+    """
+
+    if workflow_dir_user and (workflow_dir_user != workflow_dir):
+        raise NotImplementedError(
+            "Local backend does not support different directories "
+            f"{workflow_dir=} and {workflow_dir_user=}"
+        )
+
+    # Set values of first_task_index and last_task_index
+    num_tasks = len(workflow.task_list)
+    first_task_index, last_task_index = set_start_and_last_task_index(
+        num_tasks,
+        first_task_index=first_task_index,
+        last_task_index=last_task_index,
+    )
+
+    new_dataset_attributes = await async_wrap(_process_workflow)(
+        workflow=workflow,
+        dataset=dataset,
+        logger_name=logger_name,
+        workflow_dir=workflow_dir,
+        first_task_index=first_task_index,
+        last_task_index=last_task_index,
+    )
+    return new_dataset_attributes

fractal_server/app/runner/v2/_local/_local_config.py

@@ -0,0 +1,118 @@
+# Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
+# University of Zurich
+#
+# Original authors:
+# Tommaso Comparin <tommaso.comparin@exact-lab.it>
+#
+# This file is part of Fractal and was originally developed by eXact lab S.r.l.
+# <exact-lab.it> under contract with Liberali Lab from the Friedrich Miescher
+# Institute for Biomedical Research and Pelkmans Lab from the University of
+# Zurich.
+"""
+Submodule to handle the local-backend configuration for a WorkflowTask
+"""
+import json
+from pathlib import Path
+from typing import Literal
+from typing import Optional
+
+from pydantic import BaseModel
+from pydantic import Extra
+from pydantic.error_wrappers import ValidationError
+
+from .....config import get_settings
+from .....syringe import Inject
+from ....models.v2 import WorkflowTaskV2
+
+
+class LocalBackendConfigError(ValueError):
+    """
+    Local-backend configuration error
+    """
+
+    pass
+
+
+class LocalBackendConfig(BaseModel, extra=Extra.forbid):
+    """
+    Specifications of the local-backend configuration
+
+    Attributes:
+        parallel_tasks_per_job:
+            Maximum number of tasks to be run in parallel as part of a call to
+            `FractalThreadPoolExecutor.map`; if `None`, then all tasks will
+            start at the same time.
+    """
+
+    parallel_tasks_per_job: Optional[int]
+
+
+def get_default_local_backend_config():
+    """
+    Return a default `LocalBackendConfig` configuration object
+    """
+    return LocalBackendConfig(parallel_tasks_per_job=None)
+
+
+def get_local_backend_config(
+    wftask: WorkflowTaskV2,
+    which_type: Literal["non_parallel", "parallel"],
+    config_path: Optional[Path] = None,
+) -> LocalBackendConfig:
+    """
+    Prepare a `LocalBackendConfig` configuration object
+
+    The sources for `parallel_tasks_per_job` attributes, starting from the
+    highest-priority one, are
+
+    1. Properties in `wftask.meta_parallel` or `wftask.meta_non_parallel`
+       (depending on `which_type`);
+    2. The general content of the local-backend configuration file;
+    3. The default value (`None`).
+
+    Arguments:
+        wftask:
+            WorkflowTaskV2 for which the backend configuration should
+            be prepared.
+        config_path:
+            Path of local-backend configuration file; if `None`, use
+            `FRACTAL_LOCAL_CONFIG_FILE` variable from settings.
+
+    Returns:
+        A local-backend configuration object
+    """
+
+    key = "parallel_tasks_per_job"
+    default_value = None
+
+    if which_type == "non_parallel":
+        wftask_meta = wftask.meta_non_parallel
+    elif which_type == "parallel":
+        wftask_meta = wftask.meta_parallel
+    else:
+        raise ValueError(
+            "`get_local_backend_config` received an invalid argument"
+            f" {which_type=}."
+        )
+
+    if wftask_meta and key in wftask_meta:
+        parallel_tasks_per_job = wftask.meta[key]
+    else:
+        if not config_path:
+            settings = Inject(get_settings)
+            config_path = settings.FRACTAL_LOCAL_CONFIG_FILE
+        if config_path is None:
+            parallel_tasks_per_job = default_value
+        else:
+            with config_path.open("r") as f:
+                env = json.load(f)
+            try:
+                _ = LocalBackendConfig(**env)
+            except ValidationError as e:
+                raise LocalBackendConfigError(
+                    f"Error while loading {config_path=}. "
+                    f"Original error:\n{str(e)}"
+                )
+
+            parallel_tasks_per_job = env.get(key, default_value)
+    return LocalBackendConfig(parallel_tasks_per_job=parallel_tasks_per_job)

fractal_server/app/runner/v2/_local/_submit_setup.py

@@ -0,0 +1,52 @@
+# Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
+# University of Zurich
+#
+# Original authors:
+# Tommaso Comparin <tommaso.comparin@exact-lab.it>
+#
+# This file is part of Fractal and was originally developed by eXact lab S.r.l.
+# <exact-lab.it> under contract with Liberali Lab from the Friedrich Miescher
+# Institute for Biomedical Research and Pelkmans Lab from the University of
+# Zurich.
+"""
+Submodule to define _local_submit_setup
+"""
+from pathlib import Path
+from typing import Literal
+from typing import Optional
+
+from ....models.v2 import WorkflowTaskV2
+from ._local_config import get_local_backend_config
+
+
+def _local_submit_setup(
+    *,
+    wftask: WorkflowTaskV2,
+    workflow_dir: Optional[Path] = None,
+    workflow_dir_user: Optional[Path] = None,
+    which_type: Literal["non_parallel", "parallel"],
+) -> dict[str, object]:
+    """
+    Collect WorfklowTask-specific configuration parameters from different
+    sources, and inject them for execution.
+
+    Arguments:
+        wftask:
+            WorkflowTask for which the configuration is to be assembled
+        workflow_dir:
+            Not used in this function.
+        workflow_dir_user:
+            Not used in this function.
+
+    Returns:
+        submit_setup_dict:
+            A dictionary that will be passed on to
+            `FractalThreadPoolExecutor.submit` and
+            `FractalThreadPoolExecutor.map`, so as to set extra options.
+    """
+
+    local_backend_config = get_local_backend_config(
+        wftask=wftask, which_type=which_type
+    )
+
+    return dict(local_backend_config=local_backend_config)

fractal_server/app/runner/v2/_slurm/__init__.py

@@ -0,0 +1,157 @@
+# Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
+# University of Zurich
+#
+# Original authors:
+# Jacopo Nespolo <jacopo.nespolo@exact-lab.it>
+# Tommaso Comparin <tommaso.comparin@exact-lab.it>
+# Marco Franzon <marco.franzon@exact-lab.it>
+#
+# This file is part of Fractal and was originally developed by eXact lab S.r.l.
+# <exact-lab.it> under contract with Liberali Lab from the Friedrich Miescher
+# Institute for Biomedical Research and Pelkmans Lab from the University of
+# Zurich.
+"""
+Slurm Bakend
+
+This backend runs fractal workflows in a SLURM cluster using Clusterfutures
+Executor objects.
+"""
+from pathlib import Path
+from typing import Any
+from typing import Optional
+from typing import Union
+
+from fractal_server.app.models.v2 import WorkflowV2
+
+# from ...executors.slurm.executor import FractalSlurmExecutor
+# from ._submit_setup import _slurm_submit_setup
+# from .get_slurm_config import get_slurm_config
+
+# from .._common import execute_tasks
+# from ..common import async_wrap
+# from ..common import set_start_and_last_task_index
+# from ..common import TaskParameters
+
+
+def _process_workflow(
+    *,
+    workflow: WorkflowV2,
+    input_paths: list[Path],
+    output_path: Path,
+    input_metadata: dict[str, Any],
+    input_history: list[dict[str, Any]],
+    logger_name: str,
+    workflow_dir: Path,
+    workflow_dir_user: Path,
+    first_task_index: int,
+    last_task_index: int,
+    slurm_user: Optional[str] = None,
+    slurm_account: Optional[str] = None,
+    user_cache_dir: str,
+    worker_init: Optional[Union[str, list[str]]] = None,
+) -> dict[str, Any]:
+    """
+    Internal processing routine for the SLURM backend
+
+    This function initialises the a FractalSlurmExecutor, setting logging,
+    workflow working dir and user to impersonate. It then schedules the
+    workflow tasks and returns the output dataset metadata.
+
+    Cf. [process_workflow][fractal_server.app.runner._local.process_workflow]
+
+    Returns:
+        output_dataset_metadata: Metadata of the output dataset
+    """
+
+    raise NotImplementedError
+
+    # if not slurm_user:
+    #     raise RuntimeError(
+    #         "slurm_user argument is required, for slurm backend"
+    #     )
+
+    # if isinstance(worker_init, str):
+    #     worker_init = worker_init.split("\n")
+
+    # with FractalSlurmExecutor(
+    #     debug=True,
+    #     keep_logs=True,
+    #     slurm_user=slurm_user,
+    #     user_cache_dir=user_cache_dir,
+    #     working_dir=workflow_dir,
+    #     working_dir_user=workflow_dir_user,
+    #     common_script_lines=worker_init,
+    #     slurm_account=slurm_account,
+    # ) as executor:
+    #     output_task_pars = execute_tasks(
+    #         executor=executor,
+    #         task_list=workflow.task_list[
+    #             first_task_index : (last_task_index + 1)  # noqa
+    #         ],  # noqa
+    #         task_pars=TaskParameters(
+    #             input_paths=input_paths,
+    #             output_path=output_path,
+    #             metadata=input_metadata,
+    #             history=input_history,
+    #         ),
+    #         workflow_dir=workflow_dir,
+    #         workflow_dir_user=workflow_dir_user,
+    #         submit_setup_call=_slurm_submit_setup,
+    #         logger_name=logger_name,
+    #     )
+    # output_dataset_metadata_history = dict(
+    #     metadata=output_task_pars.metadata, history=output_task_pars.history
+    # )
+    # return output_dataset_metadata_history
+
+
+async def process_workflow(
+    *,
+    workflow: WorkflowV2,
+    input_paths: list[Path],
+    output_path: Path,
+    input_metadata: dict[str, Any],
+    input_history: list[dict[str, Any]],
+    logger_name: str,
+    workflow_dir: Path,
+    workflow_dir_user: Optional[Path] = None,
+    user_cache_dir: Optional[str] = None,
+    slurm_user: Optional[str] = None,
+    slurm_account: Optional[str] = None,
+    worker_init: Optional[str] = None,
+    first_task_index: Optional[int] = None,
+    last_task_index: Optional[int] = None,
+) -> dict[str, Any]:
+    """
+    Process workflow (SLURM backend public interface)
+
+    Cf. [process_workflow][fractal_server.app.runner._local.process_workflow]
+    """
+
+    raise NotImplementedError
+
+    # # Set values of first_task_index and last_task_index
+    # num_tasks = len(workflow.task_list)
+    # first_task_index, last_task_index = set_start_and_last_task_index(
+    #     num_tasks,
+    #     first_task_index=first_task_index,
+    #     last_task_index=last_task_index,
+    # )
+
+    # output_dataset_metadata_history = await async_wrap(_process_workflow)(
+    #     workflow=workflow,
+    #     input_paths=input_paths,
+    #     output_path=output_path,
+    #     input_metadata=input_metadata,
+    #     input_history=input_history,
+    #     logger_name=logger_name,
+    #     workflow_dir=workflow_dir,
+    #     workflow_dir_user=workflow_dir_user,
+    #     slurm_user=slurm_user,
+    #     slurm_account=slurm_account,
+    #     user_cache_dir=user_cache_dir,
+    #     worker_init=worker_init,
+    #     first_task_index=first_task_index,
+    #     last_task_index=last_task_index,
+    # )
+    # return output_dataset_metadata_history

fractal_server/app/runner/v2/_slurm/_submit_setup.py

@@ -0,0 +1,83 @@
+# Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
+# University of Zurich
+#
+# Original authors:
+# Jacopo Nespolo <jacopo.nespolo@exact-lab.it>
+# Tommaso Comparin <tommaso.comparin@exact-lab.it>
+#
+# This file is part of Fractal and was originally developed by eXact lab S.r.l.
+# <exact-lab.it> under contract with Liberali Lab from the Friedrich Miescher
+# Institute for Biomedical Research and Pelkmans Lab from the University of
+# Zurich.
+"""
+Submodule to define _slurm_submit_setup, which is also the reference
+implementation of `submit_setup_call` in
+[fractal_server.app.runner._common][]).
+"""
+from pathlib import Path
+from typing import Literal
+
+from ...task_files import get_task_file_paths
+from .get_slurm_config import get_slurm_config
+from fractal_server.app.models.v2 import WorkflowTaskV2
+
+
+def _slurm_submit_setup(
+    *,
+    wftask: WorkflowTaskV2,
+    workflow_dir: Path,
+    workflow_dir_user: Path,
+    which_type: Literal["non_parallel", "parallel"],
+) -> dict[str, object]:
+    """
+    Collect WorfklowTask-specific configuration parameters from different
+    sources, and inject them for execution.
+
+    Here goes all the logic for reading attributes from the appropriate sources
+    and transforming them into an appropriate `SlurmConfig` object (encoding
+    SLURM configuration) and `TaskFiles` object (with details e.g. about file
+    paths or filename prefixes).
+
+    For now, this is the reference implementation for the argument
+    `submit_setup_call` of
+    [fractal_server.app.runner._common.execute_tasks][].
+
+    Arguments:
+        wftask:
+            WorkflowTask for which the configuration is to be assembled
+        workflow_dir:
+            Server-owned directory to store all task-execution-related relevant
+            files (inputs, outputs, errors, and all meta files related to the
+            job execution). Note: users cannot write directly to this folder.
+        workflow_dir_user:
+            User-side directory with the same scope as `workflow_dir`, and
+            where a user can write.
+
+    Returns:
+        submit_setup_dict:
+            A dictionary that will be passed on to
+            `FractalSlurmExecutor.submit` and `FractalSlurmExecutor.map`, so
+            as to set extra options.
+    """
+
+    # Get SlurmConfig object
+    slurm_config = get_slurm_config(
+        wftask=wftask,
+        workflow_dir=workflow_dir,
+        workflow_dir_user=workflow_dir_user,
+        which_type=which_type,
+    )
+
+    # Get TaskFiles object
+    task_files = get_task_file_paths(
+        workflow_dir=workflow_dir,
+        workflow_dir_user=workflow_dir_user,
+        task_order=wftask.order,
+    )
+
+    # Prepare and return output dictionary
+    submit_setup_dict = dict(
+        slurm_config=slurm_config,
+        task_files=task_files,
+    )
+    return submit_setup_dict