fractal-server 2.11.1__py3-none-any.whl → 2.12.0__py3-none-any.whl

fractal_server/app/runner/v1/_local/__init__.py

@@ -1,186 +0,0 @@
-# Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
-# University of Zurich
-#
-# Original authors:
-# Jacopo Nespolo <jacopo.nespolo@exact-lab.it>
-# Tommaso Comparin <tommaso.comparin@exact-lab.it>
-# Marco Franzon <marco.franzon@exact-lab.it>
-#
-# This file is part of Fractal and was originally developed by eXact lab S.r.l.
-# <exact-lab.it> under contract with Liberali Lab from the Friedrich Miescher
-# Institute for Biomedical Research and Pelkmans Lab from the University of
-# Zurich.
-"""
-Local Bakend
-
-This backend runs Fractal workflows using `FractalThreadPoolExecutor` (a custom
-version of Python
-[ThreadPoolExecutor](https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor))
-to run tasks in several threads.
-Incidentally, it also represents the reference implementation for a backend.
-"""
-from pathlib import Path
-from typing import Any
-from typing import Optional
-
-from ....models.v1 import Workflow
-from ...async_wrap_v1 import async_wrap_v1
-from ...set_start_and_last_task_index import set_start_and_last_task_index
-from .._common import execute_tasks
-from ..common import TaskParameters
-from ._submit_setup import _local_submit_setup
-from .executor import FractalThreadPoolExecutor
-
-
-def _process_workflow(
-    *,
-    workflow: Workflow,
-    input_paths: list[Path],
-    output_path: Path,
-    input_metadata: dict[str, Any],
-    input_history: list[dict[str, Any]],
-    logger_name: str,
-    workflow_dir_local: Path,
-    first_task_index: int,
-    last_task_index: int,
-) -> dict[str, Any]:
-    """
-    Internal processing routine
-
-    Schedules the workflow using a `FractalThreadPoolExecutor`.
-
-    Cf.
-    [process_workflow][fractal_server.app.runner.v1._local.process_workflow]
-    for the call signature.
-    """
-
-    with FractalThreadPoolExecutor() as executor:
-        output_task_pars = execute_tasks(
-            executor=executor,
-            task_list=workflow.task_list[
-                first_task_index : (last_task_index + 1)  # noqa
-            ],  # noqa
-            task_pars=TaskParameters(
-                input_paths=input_paths,
-                output_path=output_path,
-                metadata=input_metadata,
-                history=input_history,
-            ),
-            workflow_dir_local=workflow_dir_local,
-            workflow_dir_remote=workflow_dir_local,
-            logger_name=logger_name,
-            submit_setup_call=_local_submit_setup,
-        )
-    output_dataset_metadata_history = dict(
-        metadata=output_task_pars.metadata, history=output_task_pars.history
-    )
-    return output_dataset_metadata_history
-
-
-async def process_workflow(
-    *,
-    workflow: Workflow,
-    input_paths: list[Path],
-    output_path: Path,
-    input_metadata: dict[str, Any],
-    input_history: list[dict[str, Any]],
-    logger_name: str,
-    workflow_dir_local: Path,
-    workflow_dir_remote: Optional[Path] = None,
-    slurm_user: Optional[str] = None,
-    slurm_account: Optional[str] = None,
-    user_cache_dir: Optional[str] = None,
-    worker_init: Optional[str] = None,
-    first_task_index: Optional[int] = None,
-    last_task_index: Optional[int] = None,
-) -> dict[str, Any]:
-    """
-    Run a workflow
-
-    This function is responsible for running a workflow on some input data,
-    saving the output and taking care of any exception raised during the run.
-
-    NOTE: This is the `local` backend's public interface, which also works as
-    a reference implementation for other backends.
-
-    Args:
-        workflow:
-            The workflow to be run
-        input_paths:
-            The paths to the input files to pass to the first task of the
-            workflow
-        output_path:
-            The destination path for the last task of the workflow
-        input_metadata:
-            Initial metadata, passed to the first task
-        logger_name:
-            Name of the logger to log information on the run to
-        workflow_dir_local:
-            Working directory for this run.
-        workflow_dir_remote:
-            Working directory for this run, on the user side. This argument is
-            present for compatibility with the standard backend interface, but
-            for the `local` backend it cannot be different from
-            `workflow_dir_local`.
-        slurm_user:
-            Username to impersonate to run the workflow. This argument is
-            present for compatibility with the standard backend interface, but
-            is ignored in the `local` backend.
-        slurm_account:
-            SLURM account to use when running the workflow. This argument is
-            present for compatibility with the standard backend interface, but
-            is ignored in the `local` backend.
-        user_cache_dir:
-            Cache directory of the user who will run the workflow. This
-            argument is present for compatibility with the standard backend
-            interface, but is ignored in the `local` backend.
-        worker_init:
-            Any additional, usually backend specific, information to be passed
-            to the backend executor. This argument is present for compatibility
-            with the standard backend interface, but is ignored in the `local`
-            backend.
-        first_task_index:
-            Positional index of the first task to execute; if `None`, start
-            from `0`.
-        last_task_index:
-            Positional index of the last task to execute; if `None`, proceed
-            until the last task.
-
-    Raises:
-        TaskExecutionError: wrapper for errors raised during tasks' execution
-                            (positive exit codes).
-        JobExecutionError: wrapper for errors raised by the tasks' executors
-                           (negative exit codes).
-
-    Returns:
-        output_dataset_metadata:
-            The updated metadata for the dataset, as returned by the last task
-            of the workflow
-    """
-
-    if workflow_dir_remote and (workflow_dir_remote != workflow_dir_local):
-        raise NotImplementedError(
-            "Local backend does not support different directories "
-            f"{workflow_dir_local=} and {workflow_dir_remote=}"
-        )
-
-    # Set values of first_task_index and last_task_index
-    num_tasks = len(workflow.task_list)
-    first_task_index, last_task_index = set_start_and_last_task_index(
-        num_tasks,
-        first_task_index=first_task_index,
-        last_task_index=last_task_index,
-    )
-
-    output_dataset_metadata_history = await async_wrap_v1(_process_workflow)(
-        workflow=workflow,
-        input_paths=input_paths,
-        output_path=output_path,
-        input_metadata=input_metadata,
-        input_history=input_history,
-        logger_name=logger_name,
-        workflow_dir_local=workflow_dir_local,
-        first_task_index=first_task_index,
-        last_task_index=last_task_index,
-    )
-    return output_dataset_metadata_history

fractal_server/app/runner/v1/_local/_local_config.py

@@ -1,105 +0,0 @@
-# Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
-# University of Zurich
-#
-# Original authors:
-# Tommaso Comparin <tommaso.comparin@exact-lab.it>
-#
-# This file is part of Fractal and was originally developed by eXact lab S.r.l.
-# <exact-lab.it> under contract with Liberali Lab from the Friedrich Miescher
-# Institute for Biomedical Research and Pelkmans Lab from the University of
-# Zurich.
-"""
-Submodule to handle the local-backend configuration for a WorkflowTask
-"""
-import json
-from pathlib import Path
-from typing import Optional
-
-from pydantic import BaseModel
-from pydantic import Extra
-from pydantic.error_wrappers import ValidationError
-
-from .....config import get_settings
-from .....syringe import Inject
-from ....models.v1 import WorkflowTask
-
-
-class LocalBackendConfigError(ValueError):
-    """
-    Local-backend configuration error
-    """
-
-    pass
-
-
-class LocalBackendConfig(BaseModel, extra=Extra.forbid):
-    """
-    Specifications of the local-backend configuration
-
-    Attributes:
-        parallel_tasks_per_job:
-            Maximum number of tasks to be run in parallel as part of a call to
-            `FractalThreadPoolExecutor.map`; if `None`, then all tasks will
-            start at the same time.
-    """
-
-    parallel_tasks_per_job: Optional[int]
-
-
-def get_default_local_backend_config():
-    """
-    Return a default `LocalBackendConfig` configuration object
-    """
-    return LocalBackendConfig(parallel_tasks_per_job=None)
-
-
-def get_local_backend_config(
-    wftask: WorkflowTask,
-    config_path: Optional[Path] = None,
-) -> LocalBackendConfig:
-    """
-    Prepare a `LocalBackendConfig` configuration object
-
-    The sources for `parallel_tasks_per_job` attributes, starting from the
-    highest-priority one, are
-
-    1. Properties in `wftask.meta`;
-    2. The general content of the local-backend configuration file;
-    3. The default value (`None`).
-
-    Arguments:
-        wftask:
-            WorkflowTask (V1) for which the backend configuration should
-            be prepared.
-        config_path:
-            Path of local-backend configuration file; if `None`, use
-            `FRACTAL_LOCAL_CONFIG_FILE` variable from settings.
-
-    Returns:
-        A local-backend configuration object
-    """
-
-    key = "parallel_tasks_per_job"
-    default = None
-
-    if wftask.meta and key in wftask.meta:
-        parallel_tasks_per_job = wftask.meta[key]
-    else:
-        if not config_path:
-            settings = Inject(get_settings)
-            config_path = settings.FRACTAL_LOCAL_CONFIG_FILE
-        if config_path is None:
-            parallel_tasks_per_job = default
-        else:
-            with config_path.open("r") as f:
-                env = json.load(f)
-            try:
-                _ = LocalBackendConfig(**env)
-            except ValidationError as e:
-                raise LocalBackendConfigError(
-                    f"Error while loading {config_path=}. "
-                    f"Original error:\n{str(e)}"
-                )
-
-            parallel_tasks_per_job = env.get(key, default)
-    return LocalBackendConfig(parallel_tasks_per_job=parallel_tasks_per_job)

fractal_server/app/runner/v1/_local/_submit_setup.py

@@ -1,48 +0,0 @@
-# Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
-# University of Zurich
-#
-# Original authors:
-# Tommaso Comparin <tommaso.comparin@exact-lab.it>
-#
-# This file is part of Fractal and was originally developed by eXact lab S.r.l.
-# <exact-lab.it> under contract with Liberali Lab from the Friedrich Miescher
-# Institute for Biomedical Research and Pelkmans Lab from the University of
-# Zurich.
-"""
-Submodule to define _local_submit_setup
-"""
-from pathlib import Path
-from typing import Optional
-
-from ....models.v1 import WorkflowTask
-from ._local_config import get_local_backend_config
-
-
-def _local_submit_setup(
-    *,
-    wftask: WorkflowTask,
-    workflow_dir_local: Optional[Path] = None,
-    workflow_dir_remote: Optional[Path] = None,
-) -> dict[str, object]:
-    """
-    Collect WorfklowTask-specific configuration parameters from different
-    sources, and inject them for execution.
-
-    Arguments:
-        wftask:
-            WorkflowTask for which the configuration is to be assembled
-        workflow_dir_local:
-            Not used in this function.
-        workflow_dir_remote:
-            Not used in this function.
-
-    Returns:
-        submit_setup_dict:
-            A dictionary that will be passed on to
-            `FractalThreadPoolExecutor.submit` and
-            `FractalThreadPoolExecutor.map`, so as to set extra options.
-    """
-
-    local_backend_config = get_local_backend_config(wftask=wftask)
-
-    return dict(local_backend_config=local_backend_config)

fractal_server/app/runner/v1/_local/executor.py

@@ -1,100 +0,0 @@
-# Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
-# University of Zurich
-#
-# Original authors:
-# Tommaso Comparin <tommaso.comparin@exact-lab.it>
-#
-# This file is part of Fractal and was originally developed by eXact lab S.r.l.
-# <exact-lab.it> under contract with Liberali Lab from the Friedrich Miescher
-# Institute for Biomedical Research and Pelkmans Lab from the University of
-# Zurich.
-"""
-Custom version of Python
-[ThreadPoolExecutor](https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor)).
-"""
-from concurrent.futures import ThreadPoolExecutor
-from typing import Callable
-from typing import Iterable
-from typing import Optional
-from typing import Sequence
-
-from ._local_config import get_default_local_backend_config
-from ._local_config import LocalBackendConfig
-
-
-class FractalThreadPoolExecutor(ThreadPoolExecutor):
-    """
-    Custom version of
-    [ThreadPoolExecutor](https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor))
-    that overrides the `submit` and `map` methods
-    """
-
-    def submit(
-        self,
-        *args,
-        local_backend_config: Optional[LocalBackendConfig] = None,
-        **kwargs,
-    ):
-        """
-        Compared to the `ThreadPoolExecutor` method, here we accept an addition
-        keyword argument (`local_backend_config`), which is then simply
-        ignored.
-        """
-        return super().submit(*args, **kwargs)
-
-    def map(
-        self,
-        fn: Callable,
-        *iterables: Sequence[Iterable],
-        local_backend_config: Optional[LocalBackendConfig] = None,
-    ):
-        """
-        Custom version of the `Executor.map` method
-
-        The main change with the respect to the original `map` method is that
-        the list of tasks to be executed is split into chunks, and then
-        `super().map` is called (sequentially) on each chunk. The goal of this
-        change is to limit parallelism, e.g. due to limited computational
-        resources.
-
-        Other changes from the `concurrent.futures` `map` method:
-
-        1. Removed `timeout` argument;
-        2. Removed `chunksize`;
-        3. All iterators (both inputs and output ones) are transformed into
-           lists.
-
-        Args:
-            fn: A callable function.
-            iterables: The argument iterables (one iterable per argument of
-                       `fn`).
-           local_backend_config: The backend configuration, needed to extract
-                                 `parallel_tasks_per_job`.
-        """
-
-        # Preliminary check
-        iterable_lengths = [len(it) for it in iterables]
-        if not len(set(iterable_lengths)) == 1:
-            raise ValueError("Iterables have different lengths.")
-
-        # Set total number of arguments
-        n_elements = len(iterables[0])
-
-        # Set parallel_tasks_per_job
-        if local_backend_config is None:
-            local_backend_config = get_default_local_backend_config()
-        parallel_tasks_per_job = local_backend_config.parallel_tasks_per_job
-        if parallel_tasks_per_job is None:
-            parallel_tasks_per_job = n_elements
-
-        # Execute tasks, in chunks of size parallel_tasks_per_job
-        results = []
-        for ind_chunk in range(0, n_elements, parallel_tasks_per_job):
-            chunk_iterables = [
-                it[ind_chunk : ind_chunk + parallel_tasks_per_job]  # noqa
-                for it in iterables
-            ]
-            map_iter = super().map(fn, *chunk_iterables)
-            results.extend(list(map_iter))
-
-        return iter(results)