fractal-server 1.4.9__py3-none-any.whl → 2.0.0a0__py3-none-any.whl

fractal_server/app/runner/{_common.py → v1/_common.py}

@@ -11,7 +11,6 @@ import subprocess  # nosec
 import traceback
 from concurrent.futures import Executor
 from copy import deepcopy
-from functools import lru_cache
 from functools import partial
 from pathlib import Path
 from shlex import split as shlex_split
@@ -19,22 +18,19 @@ from typing import Any
 from typing import Callable
 from typing import Optional
 
-from ...config import get_settings
-from ...logger import get_logger
-from ...syringe import Inject
-from ..models import Task
-from ..models import WorkflowTask
-from ..schemas import WorkflowTaskStatusType
-from .common import JobExecutionError
-from .common import TaskExecutionError
+from ....config import get_settings
+from ....logger import get_logger
+from ....syringe import Inject
+from ...models import Task
+from ...models import WorkflowTask
+from ...schemas.v1 import WorkflowTaskStatusTypeV1
+from ..exceptions import JobExecutionError
+from ..exceptions import TaskExecutionError
 from .common import TaskParameters
 from .common import write_args_file
-
-
-HISTORY_FILENAME = "history.json"
-METADATA_FILENAME = "metadata.json"
-SHUTDOWN_FILENAME = "shutdown"
-WORKFLOW_LOG_FILENAME = "workflow.log"
+from fractal_server.app.runner.filenames import HISTORY_FILENAME
+from fractal_server.app.runner.filenames import METADATA_FILENAME
+from fractal_server.app.runner.task_files import get_task_file_paths
 
 
 def no_op_submit_setup_call(
@@ -42,7 +38,6 @@ def no_op_submit_setup_call(
     wftask: WorkflowTask,
     workflow_dir: Path,
     workflow_dir_user: Path,
-    task_pars: TaskParameters,
 ) -> dict:
     """
     Default (no-operation) interface of submit_setup_call.
@@ -50,14 +45,6 @@ def no_op_submit_setup_call(
     return {}
 
 
-def sanitize_component(value: str) -> str:
-    """
-    Remove {" ", "/", "."} form a string, e.g. going from
-    'plate.zarr/B/03/0' to 'plate_zarr_B_03_0'.
-    """
-    return value.replace(" ", "_").replace("/", "_").replace(".", "_")
-
-
 def _task_needs_image_list(_task: Task) -> bool:
     """
     Whether a task requires `metadata["image"]` in its `args.json` file.
@@ -78,98 +65,6 @@ def _task_needs_image_list(_task: Task) -> bool:
         return False
 
 
-class TaskFiles:
-    """
-    Group all file paths pertaining to a task
-
-    Attributes:
-        workflow_dir:
-            Server-owned directory to store all task-execution-related relevant
-            files (inputs, outputs, errors, and all meta files related to the
-            job execution). Note: users cannot write directly to this folder.
-        workflow_dir_user:
-            User-side directory with the same scope as `workflow_dir`, and
-            where a user can write.
-        task_order:
-            Positional order of the task within a workflow.
-        component:
-            Specific component to run the task for (relevant for tasks that
-            will be executed in parallel over many components).
-        file_prefix:
-            Prefix for all task-related files.
-        args:
-            Path for input json file.
-        metadiff:
-            Path for output json file with metadata update.
-        out:
-            Path for task-execution stdout.
-        err:
-            Path for task-execution stderr.
-    """
-
-    workflow_dir: Path
-    workflow_dir_user: Path
-    task_order: Optional[int] = None
-    component: Optional[str] = None
-
-    file_prefix: str
-    args: Path
-    out: Path
-    err: Path
-    metadiff: Path
-
-    def __init__(
-        self,
-        workflow_dir: Path,
-        workflow_dir_user: Path,
-        task_order: Optional[int] = None,
-        component: Optional[str] = None,
-    ):
-        self.workflow_dir = workflow_dir
-        self.workflow_dir_user = workflow_dir_user
-        self.task_order = task_order
-        self.component = component
-
-        if self.component is not None:
-            component_safe = sanitize_component(str(self.component))
-            component_safe = f"_par_{component_safe}"
-        else:
-            component_safe = ""
-
-        if self.task_order is not None:
-            order = str(self.task_order)
-        else:
-            order = "task"
-        self.file_prefix = f"{order}{component_safe}"
-        self.args = self.workflow_dir_user / f"{self.file_prefix}.args.json"
-        self.out = self.workflow_dir_user / f"{self.file_prefix}.out"
-        self.err = self.workflow_dir_user / f"{self.file_prefix}.err"
-        self.metadiff = (
-            self.workflow_dir_user / f"{self.file_prefix}.metadiff.json"
-        )
-
-
-@lru_cache()
-def get_task_file_paths(
-    workflow_dir: Path,
-    workflow_dir_user: Path,
-    task_order: Optional[int] = None,
-    component: Optional[str] = None,
-) -> TaskFiles:
-    """
-    Return the corrisponding TaskFiles object
-
-    This function is mainly used as a cache to avoid instantiating needless
-    objects.
-    """
-    return TaskFiles(
-        workflow_dir=workflow_dir,
-        workflow_dir_user=workflow_dir_user,
-        task_order=task_order,
-        component=component,
-    )
-
-
 def _call_command_wrapper(cmd: str, stdout: Path, stderr: Path) -> None:
     """
     Call a command and write its stdout and stderr to files
@@ -331,7 +226,7 @@ def call_single_task(
     wftask_dump["task"] = wftask.task.model_dump()
     new_history_item = dict(
         workflowtask=wftask_dump,
-        status=WorkflowTaskStatusType.DONE,
+        status=WorkflowTaskStatusTypeV1.DONE,
         parallelization=None,
     )
     updated_history = task_pars.history.copy()
@@ -529,7 +424,6 @@ def call_parallel_task(
     try:
         extra_setup = submit_setup_call(
             wftask=wftask,
-            task_pars=task_pars_depend,
             workflow_dir=workflow_dir,
             workflow_dir_user=workflow_dir_user,
         )
@@ -592,7 +486,7 @@ def call_parallel_task(
     wftask_dump["task"] = wftask.task.model_dump()
     new_history_item = dict(
         workflowtask=wftask_dump,
-        status=WorkflowTaskStatusType.DONE,
+        status=WorkflowTaskStatusTypeV1.DONE,
         parallelization=dict(
             parallelization_level=wftask.parallelization_level,
             component_list=component_list,
@@ -681,7 +575,6 @@ def execute_tasks(
             try:
                 extra_setup = submit_setup_call(
                     wftask=this_wftask,
-                    task_pars=current_task_pars,
                     workflow_dir=workflow_dir,
                     workflow_dir_user=workflow_dir_user,
                 )

fractal_server/app/runner/{_local → v1/_local}/__init__.py

@@ -23,13 +23,13 @@ from pathlib import Path
 from typing import Any
 from typing import Optional
 
-from ...models import Workflow
-from .._common import execute_tasks
-from ..common import async_wrap
-from ..common import set_start_and_last_task_index
-from ..common import TaskParameters
+from ....models import Workflow  # FIXME: this is v1 specific
+from ...async_wrap import async_wrap
+from ...executors.local.executor import FractalThreadPoolExecutor
+from ...set_start_and_last_task_index import set_start_and_last_task_index
+from .._common import execute_tasks  # FIXME: this is v1 specific
+from ..common import TaskParameters  # FIXME: this is v1 specific
 from ._submit_setup import _local_submit_setup
-from .executor import FractalThreadPoolExecutor
 
 
 def _process_workflow(

fractal_server/app/runner/{_local → v1/_local}/_local_config.py

@@ -19,9 +19,9 @@ from pydantic import BaseModel
 from pydantic import Extra
 from pydantic.error_wrappers import ValidationError
 
-from ....config import get_settings
-from ....syringe import Inject
-from ...models import WorkflowTask
+from .....config import get_settings
+from .....syringe import Inject
+from ....models.v1 import WorkflowTask
 
 
 class LocalBackendConfigError(ValueError):
@@ -63,15 +63,14 @@ def get_local_backend_config(
     The sources for `parallel_tasks_per_job` attributes, starting from the
     highest-priority one, are
 
-    1. Properties in `wftask.meta` (which, for `WorkflowTask`s added through
-       `Workflow.insert_task`, also includes `wftask.task.meta`);
+    1. Properties in `wftask.meta`;
     2. The general content of the local-backend configuration file;
     3. The default value (`None`).
 
     Arguments:
         wftask:
-            WorkflowTask for which the backend configuration is is to be
-            prepared.
+            WorkflowTask (V1) for which the backend configuration should
+            be prepared.
         config_path:
             Path of local-backend configuration file; if `None`, use
             `FRACTAL_LOCAL_CONFIG_FILE` variable from settings.

fractal_server/app/runner/{_local → v1/_local}/_submit_setup.py

@@ -14,8 +14,7 @@ Submodule to define _local_submit_setup
 from pathlib import Path
 from typing import Optional
 
-from ...models import WorkflowTask
-from ..common import TaskParameters
+from ....models.v1 import WorkflowTask
 from ._local_config import get_local_backend_config
 
 
@@ -24,7 +23,6 @@ def _local_submit_setup(
     wftask: WorkflowTask,
     workflow_dir: Optional[Path] = None,
     workflow_dir_user: Optional[Path] = None,
-    task_pars: Optional[TaskParameters] = None,
 ) -> dict[str, object]:
     """
     Collect WorfklowTask-specific configuration parameters from different
@@ -33,8 +31,6 @@ def _local_submit_setup(
     Arguments:
         wftask:
             WorkflowTask for which the configuration is to be assembled
-        task_pars:
-            Not used in this function.
         workflow_dir:
             Not used in this function.
         workflow_dir_user:

fractal_server/app/runner/v1/_slurm/__init__.py

@@ -0,0 +1,310 @@
+# Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
+# University of Zurich
+#
+# Original authors:
+# Jacopo Nespolo <jacopo.nespolo@exact-lab.it>
+# Tommaso Comparin <tommaso.comparin@exact-lab.it>
+# Marco Franzon <marco.franzon@exact-lab.it>
+#
+# This file is part of Fractal and was originally developed by eXact lab S.r.l.
+# <exact-lab.it> under contract with Liberali Lab from the Friedrich Miescher
+# Institute for Biomedical Research and Pelkmans Lab from the University of
+# Zurich.
+"""
+Slurm Bakend
+
+This backend runs fractal workflows in a SLURM cluster using Clusterfutures
+Executor objects.
+"""
+from pathlib import Path
+from typing import Any
+from typing import Optional
+from typing import Union
+
+from ...async_wrap import async_wrap
+from ...executors.slurm.executor import FractalSlurmExecutor
+from ...set_start_and_last_task_index import set_start_and_last_task_index
+from .._common import execute_tasks
+from ..common import TaskParameters
+from ._submit_setup import _slurm_submit_setup
+from fractal_server.app.models.v1 import Workflow
+from fractal_server.app.models.v1 import WorkflowTask
+from fractal_server.app.runner.executors.slurm._slurm_config import (
+    _parse_mem_value,
+)
+from fractal_server.app.runner.executors.slurm._slurm_config import (
+    load_slurm_config_file,
+)
+from fractal_server.app.runner.executors.slurm._slurm_config import logger
+from fractal_server.app.runner.executors.slurm._slurm_config import SlurmConfig
+from fractal_server.app.runner.executors.slurm._slurm_config import (
+    SlurmConfigError,
+)
+
+
+def _process_workflow(
+    *,
+    workflow: Workflow,
+    input_paths: list[Path],
+    output_path: Path,
+    input_metadata: dict[str, Any],
+    input_history: list[dict[str, Any]],
+    logger_name: str,
+    workflow_dir: Path,
+    workflow_dir_user: Path,
+    first_task_index: int,
+    last_task_index: int,
+    slurm_user: Optional[str] = None,
+    slurm_account: Optional[str] = None,
+    user_cache_dir: str,
+    worker_init: Optional[Union[str, list[str]]] = None,
+) -> dict[str, Any]:
+    """
+    Internal processing routine for the SLURM backend
+
+    This function initialises the a FractalSlurmExecutor, setting logging,
+    workflow working dir and user to impersonate. It then schedules the
+    workflow tasks and returns the output dataset metadata.
+
+    Cf. [process_workflow][fractal_server.app.runner._local.process_workflow]
+
+    Returns:
+        output_dataset_metadata: Metadata of the output dataset
+    """
+
+    if not slurm_user:
+        raise RuntimeError(
+            "slurm_user argument is required, for slurm backend"
+        )
+
+    if isinstance(worker_init, str):
+        worker_init = worker_init.split("\n")
+
+    with FractalSlurmExecutor(
+        debug=True,
+        keep_logs=True,
+        slurm_user=slurm_user,
+        user_cache_dir=user_cache_dir,
+        working_dir=workflow_dir,
+        working_dir_user=workflow_dir_user,
+        common_script_lines=worker_init,
+        slurm_account=slurm_account,
+    ) as executor:
+        output_task_pars = execute_tasks(
+            executor=executor,
+            task_list=workflow.task_list[
+                first_task_index : (last_task_index + 1)  # noqa
+            ],  # noqa
+            task_pars=TaskParameters(
+                input_paths=input_paths,
+                output_path=output_path,
+                metadata=input_metadata,
+                history=input_history,
+            ),
+            workflow_dir=workflow_dir,
+            workflow_dir_user=workflow_dir_user,
+            submit_setup_call=_slurm_submit_setup,
+            logger_name=logger_name,
+        )
+    output_dataset_metadata_history = dict(
+        metadata=output_task_pars.metadata, history=output_task_pars.history
+    )
+    return output_dataset_metadata_history
+
+
+async def process_workflow(
+    *,
+    workflow: Workflow,
+    input_paths: list[Path],
+    output_path: Path,
+    input_metadata: dict[str, Any],
+    input_history: list[dict[str, Any]],
+    logger_name: str,
+    workflow_dir: Path,
+    workflow_dir_user: Optional[Path] = None,
+    user_cache_dir: Optional[str] = None,
+    slurm_user: Optional[str] = None,
+    slurm_account: Optional[str] = None,
+    worker_init: Optional[str] = None,
+    first_task_index: Optional[int] = None,
+    last_task_index: Optional[int] = None,
+) -> dict[str, Any]:
+    """
+    Process workflow (SLURM backend public interface)
+
+    Cf. [process_workflow][fractal_server.app.runner._local.process_workflow]
+    """
+
+    # Set values of first_task_index and last_task_index
+    num_tasks = len(workflow.task_list)
+    first_task_index, last_task_index = set_start_and_last_task_index(
+        num_tasks,
+        first_task_index=first_task_index,
+        last_task_index=last_task_index,
+    )
+
+    output_dataset_metadata_history = await async_wrap(_process_workflow)(
+        workflow=workflow,
+        input_paths=input_paths,
+        output_path=output_path,
+        input_metadata=input_metadata,
+        input_history=input_history,
+        logger_name=logger_name,
+        workflow_dir=workflow_dir,
+        workflow_dir_user=workflow_dir_user,
+        slurm_user=slurm_user,
+        slurm_account=slurm_account,
+        user_cache_dir=user_cache_dir,
+        worker_init=worker_init,
+        first_task_index=first_task_index,
+        last_task_index=last_task_index,
+    )
+    return output_dataset_metadata_history
+
+
+def get_slurm_config(
+    wftask: WorkflowTask,
+    workflow_dir: Path,
+    workflow_dir_user: Path,
+    config_path: Optional[Path] = None,
+) -> SlurmConfig:
+    """
+    Prepare a `SlurmConfig` configuration object
+
+    The sources for `SlurmConfig` attributes, in increasing priority order, are
+
+    1. The general content of the Fractal SLURM configuration file.
+    2. The GPU-specific content of the Fractal SLURM configuration file, if
+        appropriate.
+    3. Properties in `wftask.meta` (which, for `WorkflowTask`s added through
+       `Workflow.insert_task`, also includes `wftask.task.meta`);
+
+    Note: `wftask.meta` may be `None`.
+
+    Arguments:
+        wftask:
+            WorkflowTask for which the SLURM configuration is is to be
+            prepared.
+        workflow_dir:
+            Server-owned directory to store all task-execution-related relevant
+            files (inputs, outputs, errors, and all meta files related to the
+            job execution). Note: users cannot write directly to this folder.
+        workflow_dir_user:
+            User-side directory with the same scope as `workflow_dir`, and
+            where a user can write.
+        config_path:
+            Path of aFractal  SLURM configuration file; if `None`, use
+            `FRACTAL_SLURM_CONFIG_FILE` variable from settings.
+
+    Returns:
+        slurm_config:
+            The SlurmConfig object
+    """
+
+    logger.debug(
+        "[get_slurm_config] WorkflowTask meta attribute: {wftask.meta=}"
+    )
+
+    # Incorporate slurm_env.default_slurm_config
+    slurm_env = load_slurm_config_file(config_path=config_path)
+    slurm_dict = slurm_env.default_slurm_config.dict(
+        exclude_unset=True, exclude={"mem"}
+    )
+    if slurm_env.default_slurm_config.mem:
+        slurm_dict["mem_per_task_MB"] = slurm_env.default_slurm_config.mem
+
+    # Incorporate slurm_env.batching_config
+    for key, value in slurm_env.batching_config.dict().items():
+        slurm_dict[key] = value
+
+    # Incorporate slurm_env.user_local_exports
+    slurm_dict["user_local_exports"] = slurm_env.user_local_exports
+
+    logger.debug(
+        "[get_slurm_config] Fractal SLURM configuration file: "
+        f"{slurm_env.dict()=}"
+    )
+
+    # GPU-related options
+    # Notes about priority:
+    # 1. This block of definitions takes priority over other definitions from
+    #    slurm_env which are not under the `needs_gpu` subgroup
+    # 2. This block of definitions has lower priority than whatever comes next
+    #    (i.e. from WorkflowTask.meta).
+    if wftask.meta is not None:
+        needs_gpu = wftask.meta.get("needs_gpu", False)
+    else:
+        needs_gpu = False
+    logger.debug(f"[get_slurm_config] {needs_gpu=}")
+    if needs_gpu:
+        for key, value in slurm_env.gpu_slurm_config.dict(
+            exclude_unset=True, exclude={"mem"}
+        ).items():
+            slurm_dict[key] = value
+        if slurm_env.gpu_slurm_config.mem:
+            slurm_dict["mem_per_task_MB"] = slurm_env.gpu_slurm_config.mem
+
+    # Number of CPUs per task, for multithreading
+    if wftask.meta is not None and "cpus_per_task" in wftask.meta:
+        cpus_per_task = int(wftask.meta["cpus_per_task"])
+        slurm_dict["cpus_per_task"] = cpus_per_task
+
+    # Required memory per task, in MB
+    if wftask.meta is not None and "mem" in wftask.meta:
+        raw_mem = wftask.meta["mem"]
+        mem_per_task_MB = _parse_mem_value(raw_mem)
+        slurm_dict["mem_per_task_MB"] = mem_per_task_MB
+
+    # Job name
+    job_name = wftask.task.name.replace(" ", "_")
+    slurm_dict["job_name"] = job_name
+
+    # Optional SLURM arguments and extra lines
+    if wftask.meta is not None:
+        account = wftask.meta.get("account", None)
+        if account is not None:
+            error_msg = (
+                f"Invalid {account=} property in WorkflowTask `meta` "
+                "attribute.\n"
+                "SLURM account must be set in the request body of the "
+                "apply-workflow endpoint, or by modifying the user properties."
+            )
+            logger.error(error_msg)
+            raise SlurmConfigError(error_msg)
+        for key in ["time", "gres", "constraint"]:
+            value = wftask.meta.get(key, None)
+            if value:
+                slurm_dict[key] = value
+    if wftask.meta is not None:
+        extra_lines = wftask.meta.get("extra_lines", [])
+    else:
+        extra_lines = []
+    extra_lines = slurm_dict.get("extra_lines", []) + extra_lines
+    if len(set(extra_lines)) != len(extra_lines):
+        logger.debug(
+            "[get_slurm_config] Removing repeated elements "
+            f"from {extra_lines=}."
+        )
+        extra_lines = list(set(extra_lines))
+    slurm_dict["extra_lines"] = extra_lines
+
+    # Job-batching parameters (if None, they will be determined heuristically)
+    if wftask.meta is not None:
+        tasks_per_job = wftask.meta.get("tasks_per_job", None)
+        parallel_tasks_per_job = wftask.meta.get(
+            "parallel_tasks_per_job", None
+        )
+    else:
+        tasks_per_job = None
+        parallel_tasks_per_job = None
+    slurm_dict["tasks_per_job"] = tasks_per_job
+    slurm_dict["parallel_tasks_per_job"] = parallel_tasks_per_job
+
+    # Put everything together
+    logger.debug(
+        "[get_slurm_config] Now create a SlurmConfig object based "
+        f"on {slurm_dict=}"
+    )
+    slurm_config = SlurmConfig(**slurm_dict)
+
+    return slurm_config

fractal_server/app/runner/{_slurm → v1/_slurm}/_submit_setup.py

@@ -15,12 +15,10 @@ implementation of `submit_setup_call` in
 [fractal_server.app.runner._common][]).
 """
 from pathlib import Path
-from typing import Optional
 
-from ...models import WorkflowTask
-from .._common import get_task_file_paths
-from ..common import TaskParameters
-from ._slurm_config import get_slurm_config
+from ...task_files import get_task_file_paths
+from .get_slurm_config import get_slurm_config
+from fractal_server.app.models.v1 import WorkflowTask
 
 
 def _slurm_submit_setup(
@@ -28,7 +26,6 @@ def _slurm_submit_setup(
     wftask: WorkflowTask,
     workflow_dir: Path,
     workflow_dir_user: Path,
-    task_pars: Optional[TaskParameters] = None,
 ) -> dict[str, object]:
     """
     Collect WorfklowTask-specific configuration parameters from different
@@ -46,9 +43,6 @@ def _slurm_submit_setup(
     Arguments:
         wftask:
             WorkflowTask for which the configuration is to be assembled
-        task_pars:
-            Task parameters to be passed to the task
-            (not used in this function)
         workflow_dir:
             Server-owned directory to store all task-execution-related relevant
             files (inputs, outputs, errors, and all meta files related to the