fractal-server 1.4.10__py3-none-any.whl → 2.0.0a1__py3-none-any.whl

fractal_server/app/runner/v2/_local/executor.py

@@ -0,0 +1,100 @@
+# Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
+# University of Zurich
+#
+# Original authors:
+# Tommaso Comparin <tommaso.comparin@exact-lab.it>
+#
+# This file is part of Fractal and was originally developed by eXact lab S.r.l.
+# <exact-lab.it> under contract with Liberali Lab from the Friedrich Miescher
+# Institute for Biomedical Research and Pelkmans Lab from the University of
+# Zurich.
+"""
+Custom version of Python
+[ThreadPoolExecutor](https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor)).
+"""
+from concurrent.futures import ThreadPoolExecutor
+from typing import Callable
+from typing import Iterable
+from typing import Optional
+from typing import Sequence
+
+from ._local_config import get_default_local_backend_config
+from ._local_config import LocalBackendConfig
+
+
+class FractalThreadPoolExecutor(ThreadPoolExecutor):
+    """
+    Custom version of
+    [ThreadPoolExecutor](https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor))
+    that overrides the `submit` and `map` methods
+    """
+
+    def submit(
+        self,
+        *args,
+        local_backend_config: Optional[LocalBackendConfig] = None,
+        **kwargs,
+    ):
+        """
+        Compared to the `ThreadPoolExecutor` method, here we accept an addition
+        keyword argument (`local_backend_config`), which is then simply
+        ignored.
+        """
+        return super().submit(*args, **kwargs)
+
+    def map(
+        self,
+        fn: Callable,
+        *iterables: Sequence[Iterable],
+        local_backend_config: Optional[LocalBackendConfig] = None,
+    ):
+        """
+        Custom version of the `Executor.map` method
+
+        The main change with the respect to the original `map` method is that
+        the list of tasks to be executed is split into chunks, and then
+        `super().map` is called (sequentially) on each chunk. The goal of this
+        change is to limit parallelism, e.g. due to limited computational
+        resources.
+
+        Other changes from the `concurrent.futures` `map` method:
+
+        1. Removed `timeout` argument;
+        2. Removed `chunksize`;
+        3. All iterators (both inputs and output ones) are transformed into
+           lists.
+
+        Args:
+            fn: A callable function.
+            iterables: The argument iterables (one iterable per argument of
+                       `fn`).
+           local_backend_config: The backend configuration, needed to extract
+                                 `parallel_tasks_per_job`.
+        """
+
+        # Preliminary check
+        iterable_lengths = [len(it) for it in iterables]
+        if not len(set(iterable_lengths)) == 1:
+            raise ValueError("Iterables have different lengths.")
+
+        # Set total number of arguments
+        n_elements = len(iterables[0])
+
+        # Set parallel_tasks_per_job
+        if local_backend_config is None:
+            local_backend_config = get_default_local_backend_config()
+        parallel_tasks_per_job = local_backend_config.parallel_tasks_per_job
+        if parallel_tasks_per_job is None:
+            parallel_tasks_per_job = n_elements
+
+        # Execute tasks, in chunks of size parallel_tasks_per_job
+        results = []
+        for ind_chunk in range(0, n_elements, parallel_tasks_per_job):
+            chunk_iterables = [
+                it[ind_chunk : ind_chunk + parallel_tasks_per_job]  # noqa
+                for it in iterables
+            ]
+            map_iter = super().map(fn, *chunk_iterables)
+            results.extend(list(map_iter))
+
+        return iter(results)

fractal_server/app/runner/{_slurm → v2/_slurm}/__init__.py

@@ -21,22 +21,24 @@ from typing import Any
 from typing import Optional
 from typing import Union
 
-from ...models import Workflow
-from .._common import execute_tasks
-from ..common import async_wrap
-from ..common import set_start_and_last_task_index
-from ..common import TaskParameters
+from ....models.v2 import DatasetV2
+from ....models.v2 import WorkflowV2
+from ...async_wrap import async_wrap
+from ...executors.slurm.executor import FractalSlurmExecutor
+from ...set_start_and_last_task_index import set_start_and_last_task_index
+from ..runner import execute_tasks_v2
 from ._submit_setup import _slurm_submit_setup
-from .executor import FractalSlurmExecutor
+
+# from .._common import execute_tasks
+# from ..common import async_wrap
+# from ..common import set_start_and_last_task_index
+# from ..common import TaskParameters
 
 
 def _process_workflow(
     *,
-    workflow: Workflow,
-    input_paths: list[Path],
-    output_path: Path,
-    input_metadata: dict[str, Any],
-    input_history: list[dict[str, Any]],
+    workflow: WorkflowV2,
+    dataset: DatasetV2,
     logger_name: str,
     workflow_dir: Path,
     workflow_dir_user: Path,
@@ -52,12 +54,12 @@ def _process_workflow(
 
     This function initialises the a FractalSlurmExecutor, setting logging,
     workflow working dir and user to impersonate. It then schedules the
-    workflow tasks and returns the output dataset metadata.
+    workflow tasks and returns the new dataset attributes
 
     Cf. [process_workflow][fractal_server.app.runner._local.process_workflow]
 
     Returns:
-        output_dataset_metadata: Metadata of the output dataset
+        new_dataset_attributes:
     """
 
     if not slurm_user:
@@ -78,45 +80,35 @@ def _process_workflow(
         common_script_lines=worker_init,
         slurm_account=slurm_account,
     ) as executor:
-        output_task_pars = execute_tasks(
-            executor=executor,
-            task_list=workflow.task_list[
+        new_dataset_attributes = execute_tasks_v2(
+            wf_task_list=workflow.task_list[
                 first_task_index : (last_task_index + 1)  # noqa
             ],  # noqa
-            task_pars=TaskParameters(
-                input_paths=input_paths,
-                output_path=output_path,
-                metadata=input_metadata,
-                history=input_history,
-            ),
+            dataset=dataset,
+            executor=executor,
             workflow_dir=workflow_dir,
             workflow_dir_user=workflow_dir_user,
-            submit_setup_call=_slurm_submit_setup,
             logger_name=logger_name,
+            submit_setup_call=_slurm_submit_setup,
         )
-    output_dataset_metadata_history = dict(
-        metadata=output_task_pars.metadata, history=output_task_pars.history
-    )
-    return output_dataset_metadata_history
+    return new_dataset_attributes
 
 
 async def process_workflow(
     *,
-    workflow: Workflow,
-    input_paths: list[Path],
-    output_path: Path,
-    input_metadata: dict[str, Any],
-    input_history: list[dict[str, Any]],
-    logger_name: str,
+    workflow: WorkflowV2,
+    dataset: DatasetV2,
     workflow_dir: Path,
     workflow_dir_user: Optional[Path] = None,
+    first_task_index: Optional[int] = None,
+    last_task_index: Optional[int] = None,
+    logger_name: str,
+    # Slurm-specific
     user_cache_dir: Optional[str] = None,
     slurm_user: Optional[str] = None,
     slurm_account: Optional[str] = None,
     worker_init: Optional[str] = None,
-    first_task_index: Optional[int] = None,
-    last_task_index: Optional[int] = None,
-) -> dict[str, Any]:
+) -> dict:
     """
     Process workflow (SLURM backend public interface)
 
@@ -131,20 +123,17 @@ async def process_workflow(
         last_task_index=last_task_index,
     )
 
-    output_dataset_metadata_history = await async_wrap(_process_workflow)(
+    new_dataset_attributes = await async_wrap(_process_workflow)(
         workflow=workflow,
-        input_paths=input_paths,
-        output_path=output_path,
-        input_metadata=input_metadata,
-        input_history=input_history,
+        dataset=dataset,
         logger_name=logger_name,
         workflow_dir=workflow_dir,
         workflow_dir_user=workflow_dir_user,
+        first_task_index=first_task_index,
+        last_task_index=last_task_index,
+        user_cache_dir=user_cache_dir,
         slurm_user=slurm_user,
         slurm_account=slurm_account,
-        user_cache_dir=user_cache_dir,
         worker_init=worker_init,
-        first_task_index=first_task_index,
-        last_task_index=last_task_index,
     )
-    return output_dataset_metadata_history
+    return new_dataset_attributes

fractal_server/app/runner/v2/_slurm/_submit_setup.py

@@ -0,0 +1,83 @@
+# Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
+# University of Zurich
+#
+# Original authors:
+# Jacopo Nespolo <jacopo.nespolo@exact-lab.it>
+# Tommaso Comparin <tommaso.comparin@exact-lab.it>
+#
+# This file is part of Fractal and was originally developed by eXact lab S.r.l.
+# <exact-lab.it> under contract with Liberali Lab from the Friedrich Miescher
+# Institute for Biomedical Research and Pelkmans Lab from the University of
+# Zurich.
+"""
+Submodule to define _slurm_submit_setup, which is also the reference
+implementation of `submit_setup_call` in
+[fractal_server.app.runner._common][]).
+"""
+from pathlib import Path
+from typing import Literal
+
+from ...task_files import get_task_file_paths
+from .get_slurm_config import get_slurm_config
+from fractal_server.app.models.v2 import WorkflowTaskV2
+
+
+def _slurm_submit_setup(
+    *,
+    wftask: WorkflowTaskV2,
+    workflow_dir: Path,
+    workflow_dir_user: Path,
+    which_type: Literal["non_parallel", "parallel"],
+) -> dict[str, object]:
+    """
+    Collect WorfklowTask-specific configuration parameters from different
+    sources, and inject them for execution.
+
+    Here goes all the logic for reading attributes from the appropriate sources
+    and transforming them into an appropriate `SlurmConfig` object (encoding
+    SLURM configuration) and `TaskFiles` object (with details e.g. about file
+    paths or filename prefixes).
+
+    For now, this is the reference implementation for the argument
+    `submit_setup_call` of
+    [fractal_server.app.runner._common.execute_tasks][].
+
+    Arguments:
+        wftask:
+            WorkflowTask for which the configuration is to be assembled
+        workflow_dir:
+            Server-owned directory to store all task-execution-related relevant
+            files (inputs, outputs, errors, and all meta files related to the
+            job execution). Note: users cannot write directly to this folder.
+        workflow_dir_user:
+            User-side directory with the same scope as `workflow_dir`, and
+            where a user can write.
+
+    Returns:
+        submit_setup_dict:
+            A dictionary that will be passed on to
+            `FractalSlurmExecutor.submit` and `FractalSlurmExecutor.map`, so
+            as to set extra options.
+    """
+
+    # Get SlurmConfig object
+    slurm_config = get_slurm_config(
+        wftask=wftask,
+        workflow_dir=workflow_dir,
+        workflow_dir_user=workflow_dir_user,
+        which_type=which_type,
+    )
+
+    # Get TaskFiles object
+    task_files = get_task_file_paths(
+        workflow_dir=workflow_dir,
+        workflow_dir_user=workflow_dir_user,
+        task_order=wftask.order,
+    )
+
+    # Prepare and return output dictionary
+    submit_setup_dict = dict(
+        slurm_config=slurm_config,
+        task_files=task_files,
+    )
+    return submit_setup_dict

fractal_server/app/runner/v2/_slurm/get_slurm_config.py

@@ -0,0 +1,179 @@
+from pathlib import Path
+from typing import Literal
+from typing import Optional
+
+from fractal_server.app.models.v2 import WorkflowTaskV2
+from fractal_server.app.runner.executors.slurm._slurm_config import (
+    _parse_mem_value,
+)
+from fractal_server.app.runner.executors.slurm._slurm_config import (
+    load_slurm_config_file,
+)
+from fractal_server.app.runner.executors.slurm._slurm_config import logger
+from fractal_server.app.runner.executors.slurm._slurm_config import SlurmConfig
+from fractal_server.app.runner.executors.slurm._slurm_config import (
+    SlurmConfigError,
+)
+
+
+def get_slurm_config(
+    wftask: WorkflowTaskV2,
+    workflow_dir: Path,
+    workflow_dir_user: Path,
+    which_type: Literal["non_parallel", "parallel"],
+    config_path: Optional[Path] = None,
+) -> SlurmConfig:
+    """
+    Prepare a `SlurmConfig` configuration object
+
+    The argument `which_type` determines whether we use `wftask.meta_parallel`
+    or `wftask.meta_non_parallel`. In the following descritpion, let us assume
+    that `which_type="parallel"`.
+
+    The sources for `SlurmConfig` attributes, in increasing priority order, are
+
+    1. The general content of the Fractal SLURM configuration file.
+    2. The GPU-specific content of the Fractal SLURM configuration file, if
+        appropriate.
+    3. Properties in `wftask.meta_parallel` (which typically include those in
+       `wftask.task.meta_parallel`). Note that `wftask.meta_parallel` may be
+       `None`.
+
+    Arguments:
+        wftask:
+            WorkflowTask for which the SLURM configuration is is to be
+            prepared.
+        workflow_dir:
+            Server-owned directory to store all task-execution-related relevant
+            files (inputs, outputs, errors, and all meta files related to the
+            job execution). Note: users cannot write directly to this folder.
+        workflow_dir_user:
+            User-side directory with the same scope as `workflow_dir`, and
+            where a user can write.
+        config_path:
+            Path of a Fractal SLURM configuration file; if `None`, use
+            `FRACTAL_SLURM_CONFIG_FILE` variable from settings.
+        which_type:
+            Determines whether to use `meta_parallel` or `meta_non_parallel`.
+
+    Returns:
+        slurm_config:
+            The SlurmConfig object
+    """
+
+    if which_type == "non_parallel":
+        wftask_meta = wftask.meta_non_parallel
+    elif which_type == "parallel":
+        wftask_meta = wftask.meta_parallel
+    else:
+        raise ValueError(
+            f"get_slurm_config received invalid argument {which_type=}."
+        )
+
+    logger.debug(
+        "[get_slurm_config] WorkflowTask meta attribute: {wftask_meta=}"
+    )
+
+    # Incorporate slurm_env.default_slurm_config
+    slurm_env = load_slurm_config_file(config_path=config_path)
+    slurm_dict = slurm_env.default_slurm_config.dict(
+        exclude_unset=True, exclude={"mem"}
+    )
+    if slurm_env.default_slurm_config.mem:
+        slurm_dict["mem_per_task_MB"] = slurm_env.default_slurm_config.mem
+
+    # Incorporate slurm_env.batching_config
+    for key, value in slurm_env.batching_config.dict().items():
+        slurm_dict[key] = value
+
+    # Incorporate slurm_env.user_local_exports
+    slurm_dict["user_local_exports"] = slurm_env.user_local_exports
+
+    logger.debug(
+        "[get_slurm_config] Fractal SLURM configuration file: "
+        f"{slurm_env.dict()=}"
+    )
+
+    # GPU-related options
+    # Notes about priority:
+    # 1. This block of definitions takes priority over other definitions from
+    #    slurm_env which are not under the `needs_gpu` subgroup
+    # 2. This block of definitions has lower priority than whatever comes next
+    #    (i.e. from WorkflowTask.meta).
+    if wftask_meta is not None:
+        needs_gpu = wftask_meta.get("needs_gpu", False)
+    else:
+        needs_gpu = False
+    logger.debug(f"[get_slurm_config] {needs_gpu=}")
+    if needs_gpu:
+        for key, value in slurm_env.gpu_slurm_config.dict(
+            exclude_unset=True, exclude={"mem"}
+        ).items():
+            slurm_dict[key] = value
+        if slurm_env.gpu_slurm_config.mem:
+            slurm_dict["mem_per_task_MB"] = slurm_env.gpu_slurm_config.mem
+
+    # Number of CPUs per task, for multithreading
+    if wftask_meta is not None and "cpus_per_task" in wftask_meta:
+        cpus_per_task = int(wftask_meta["cpus_per_task"])
+        slurm_dict["cpus_per_task"] = cpus_per_task
+
+    # Required memory per task, in MB
+    if wftask_meta is not None and "mem" in wftask_meta:
+        raw_mem = wftask_meta["mem"]
+        mem_per_task_MB = _parse_mem_value(raw_mem)
+        slurm_dict["mem_per_task_MB"] = mem_per_task_MB
+
+    # Job name
+    job_name = wftask.task.name.replace(" ", "_")
+    slurm_dict["job_name"] = job_name
+
+    # Optional SLURM arguments and extra lines
+    if wftask_meta is not None:
+        account = wftask_meta.get("account", None)
+        if account is not None:
+            error_msg = (
+                f"Invalid {account=} property in WorkflowTask `meta` "
+                "attribute.\n"
+                "SLURM account must be set in the request body of the "
+                "apply-workflow endpoint, or by modifying the user properties."
+            )
+            logger.error(error_msg)
+            raise SlurmConfigError(error_msg)
+        for key in ["time", "gres", "constraint"]:
+            value = wftask_meta.get(key, None)
+            if value:
+                slurm_dict[key] = value
+    if wftask_meta is not None:
+        extra_lines = wftask_meta.get("extra_lines", [])
+    else:
+        extra_lines = []
+    extra_lines = slurm_dict.get("extra_lines", []) + extra_lines
+    if len(set(extra_lines)) != len(extra_lines):
+        logger.debug(
+            "[get_slurm_config] Removing repeated elements "
+            f"from {extra_lines=}."
+        )
+        extra_lines = list(set(extra_lines))
+    slurm_dict["extra_lines"] = extra_lines
+
+    # Job-batching parameters (if None, they will be determined heuristically)
+    if wftask_meta is not None:
+        tasks_per_job = wftask_meta.get("tasks_per_job", None)
+        parallel_tasks_per_job = wftask_meta.get(
+            "parallel_tasks_per_job", None
+        )
+    else:
+        tasks_per_job = None
+        parallel_tasks_per_job = None
+    slurm_dict["tasks_per_job"] = tasks_per_job
+    slurm_dict["parallel_tasks_per_job"] = parallel_tasks_per_job
+
+    # Put everything together
+    logger.debug(
+        "[get_slurm_config] Now create a SlurmConfig object based "
+        f"on {slurm_dict=}"
+    )
+    slurm_config = SlurmConfig(**slurm_dict)
+
+    return slurm_config

fractal_server/app/runner/v2/deduplicate_list.py

@@ -0,0 +1,22 @@
+from typing import TypeVar
+
+from ....images import SingleImage
+from .task_interface import InitArgsModel
+
+T = TypeVar("T", SingleImage, InitArgsModel)
+
+
+def deduplicate_list(
+    this_list: list[T],
+) -> list[T]:
+    """
+    Custom replacement for `set(this_list)`, when items are non-hashable.
+    """
+    new_list_dict = []
+    new_list_objs = []
+    for this_obj in this_list:
+        this_dict = this_obj.dict()
+        if this_dict not in new_list_dict:
+            new_list_dict.append(this_dict)
+            new_list_objs.append(this_obj)
+    return new_list_objs

fractal_server/app/runner/v2/handle_failed_job.py

@@ -0,0 +1,156 @@
+# Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
+# University of Zurich
+#
+# Original authors:
+# Tommaso Comparin <tommaso.comparin@exact-lab.it>
+# Marco Franzon <marco.franzon@exact-lab.it>
+#
+# This file is part of Fractal and was originally developed by eXact lab S.r.l.
+# <exact-lab.it> under contract with Liberali Lab from the Friedrich Miescher
+# Institute for Biomedical Research and Pelkmans Lab from the University of
+# Zurich.
+"""
+Helper functions to handle Dataset history.
+"""
+import json
+import logging
+from pathlib import Path
+from typing import Any
+from typing import Optional
+
+from ...models.v2 import DatasetV2
+from ...models.v2 import JobV2
+from ...models.v2 import WorkflowTaskV2
+from ...models.v2 import WorkflowV2
+from ...schemas.v2 import WorkflowTaskStatusTypeV2
+from ..filenames import FILTERS_FILENAME
+from ..filenames import HISTORY_FILENAME
+from ..filenames import IMAGES_FILENAME
+
+
+def assemble_history_failed_job(
+    job: JobV2,
+    dataset: DatasetV2,
+    workflow: WorkflowV2,
+    logger: logging.Logger,
+    failed_wftask: Optional[WorkflowTaskV2] = None,
+) -> list[dict[str, Any]]:
+    """
+    Assemble `history` after a workflow-execution job fails.
+
+    Args:
+        job:
+            The failed `ApplyWorkflow` object.
+        output_dataset:
+            The `dataset` associated to `job`.
+        workflow:
+            The `workflow` associated to `job`.
+        logger: A logger instance.
+        failed_wftask:
+            If set, append it to `history` during step 3; if `None`, infer
+            it by comparing the job task list and the one in
+            `HISTORY_FILENAME`.
+
+    Returns:
+        The new value of `history`, to be merged into
+        `output_dataset.meta`.
+    """
+
+    # The final value of the history attribute should include up to three
+    # parts, coming from: the database, the temporary file, the failed-task
+    # information.
+
+    # Part 1: Read exising history from DB
+    new_history = dataset.history
+
+    # Part 2: Extend history based on temporary-file contents
+    tmp_history_file = Path(job.working_dir) / HISTORY_FILENAME
+    try:
+        with tmp_history_file.open("r") as f:
+            tmp_file_history = json.load(f)
+            new_history.extend(tmp_file_history)
+    except FileNotFoundError:
+        tmp_file_history = []
+
+    # Part 3/A: Identify failed task, if needed
+    if failed_wftask is None:
+        job_wftasks = workflow.task_list[
+            job.first_task_index : (job.last_task_index + 1)  # noqa
+        ]
+        tmp_file_wftasks = [
+            history_item["workflowtask"] for history_item in tmp_file_history
+        ]
+        if len(job_wftasks) <= len(tmp_file_wftasks):
+            n_tasks_job = len(job_wftasks)
+            n_tasks_tmp = len(tmp_file_wftasks)
+            logger.error(
+                "Cannot identify the failed task based on job task list "
+                f"(length {n_tasks_job}) and temporary-file task list "
+                f"(length {n_tasks_tmp})."
+            )
+            logger.error("Failed task not appended to history.")
+        else:
+            failed_wftask = job_wftasks[len(tmp_file_wftasks)]
+
+    # Part 3/B: Append failed task to history
+    if failed_wftask is not None:
+        failed_wftask_dump = failed_wftask.model_dump(exclude={"task"})
+        failed_wftask_dump["task"] = failed_wftask.task.model_dump()
+        new_history_item = dict(
+            workflowtask=failed_wftask_dump,
+            status=WorkflowTaskStatusTypeV2.FAILED,
+            parallelization=dict(),  # FIXME: re-include parallelization
+        )
+        new_history.append(new_history_item)
+
+    return new_history
+
+
+def assemble_images_failed_job(job: JobV2) -> Optional[dict[str, Any]]:
+    """
+    Assemble `DatasetV2.images` for a failed workflow-execution.
+
+    Assemble new value of `images` based on the last successful task, i.e.
+    based on the content of the temporary `IMAGES_FILENAME` file. If the file
+    is missing, return `None`.
+
+    Argumentss:
+        job:
+            The failed `JobV2` object.
+
+    Returns:
+        The new value of `dataset.images`, or `None` if `IMAGES_FILENAME`
+        is missing.
+    """
+    tmp_file = Path(job.working_dir) / IMAGES_FILENAME
+    try:
+        with tmp_file.open("r") as f:
+            new_images = json.load(f)
+        return new_images
+    except FileNotFoundError:
+        return None
+
+
+def assemble_filters_failed_job(job: JobV2) -> Optional[dict[str, Any]]:
+    """
+    Assemble `DatasetV2.filters` for a failed workflow-execution.
+
+    Assemble new value of `filters` based on the last successful task, i.e.
+    based on the content of the temporary `FILTERS_FILENAME` file. If the file
+    is missing, return `None`.
+
+    Argumentss:
+        job:
+            The failed `JobV2` object.
+
+    Returns:
+        The new value of `dataset.filters`, or `None` if `FILTERS_FILENAME`
+        is missing.
+    """
+    tmp_file = Path(job.working_dir) / FILTERS_FILENAME
+    try:
+        with tmp_file.open("r") as f:
+            new_filters = json.load(f)
+        return new_filters
+    except FileNotFoundError:
+        return None