fractal-server 2.2.0a1__py3-none-any.whl → 2.3.0__py3-none-any.whl

fractal_server/app/runner/v2/_local_experimental/__init__.py

@@ -1,9 +1,11 @@
+from concurrent.futures.process import BrokenProcessPool
 from pathlib import Path
 from typing import Optional
 
 from ....models.v2 import DatasetV2
 from ....models.v2 import WorkflowV2
 from ...async_wrap import async_wrap
+from ...exceptions import JobExecutionError
 from ...filenames import SHUTDOWN_FILENAME
 from ...set_start_and_last_task_index import set_start_and_last_task_index
 from ..runner import execute_tasks_v2
@@ -29,21 +31,29 @@ def _process_workflow(
     [process_workflow][fractal_server.app.runner.v2._local_experimental.process_workflow]
     for the call signature.
     """
-
     with FractalProcessPoolExecutor(
         shutdown_file=workflow_dir_local / SHUTDOWN_FILENAME
     ) as executor:
-        new_dataset_attributes = execute_tasks_v2(
-            wf_task_list=workflow.task_list[
-                first_task_index : (last_task_index + 1)  # noqa
-            ],  # noqa
-            dataset=dataset,
-            executor=executor,
-            workflow_dir_local=workflow_dir_local,
-            workflow_dir_remote=workflow_dir_local,
-            logger_name=logger_name,
-            submit_setup_call=_local_submit_setup,
-        )
+        try:
+            new_dataset_attributes = execute_tasks_v2(
+                wf_task_list=workflow.task_list[
+                    first_task_index : (last_task_index + 1)  # noqa
+                ],
+                dataset=dataset,
+                executor=executor,
+                workflow_dir_local=workflow_dir_local,
+                workflow_dir_remote=workflow_dir_local,
+                logger_name=logger_name,
+                submit_setup_call=_local_submit_setup,
+            )
+        except BrokenProcessPool as e:
+            raise JobExecutionError(
+                info=(
+                    "Job failed with BrokenProcessPool error, likely due to "
+                    f"an executor shutdown.\nOriginal error:\n{e.args[0]}"
+                )
+            )
+
     return new_dataset_attributes
 
 

fractal_server/app/runner/v2/_local_experimental/executor.py

@@ -2,8 +2,6 @@
 Custom version of Python
 [ProcessPoolExecutor](https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ProcessPoolExecutor)).
 """
-import os
-import signal
 import threading
 import time
 from concurrent.futures import ProcessPoolExecutor
@@ -14,13 +12,14 @@ from typing import Iterable
 from typing import Optional
 from typing import Sequence
 
+import psutil
+
 from ._local_config import get_default_local_backend_config
 from ._local_config import LocalBackendConfig
 from fractal_server.app.runner.exceptions import JobExecutionError
-from fractal_server.logger import get_logger
-
+from fractal_server.logger import set_logger
 
-logger = get_logger("FractalProcessPoolExecutor")
+logger = set_logger("FractalProcessPoolExecutor")
 
 
 class FractalProcessPoolExecutor(ProcessPoolExecutor):
@@ -66,12 +65,17 @@ class FractalProcessPoolExecutor(ProcessPoolExecutor):
         """
         Running on '_shutdown_file_thread'.
         """
+
         logger.info("Start terminating FractalProcessPoolExecutor processes.")
+        # We use 'psutil' in order to easily access the PIDs of the children.
         if self._processes is not None:
             for pid in self._processes.keys():
-                logger.debug(f"Sending SIGTERM to process {pid}")
-                os.kill(pid, signal.SIGTERM)
-                logger.debug(f"Process {pid} terminated.")
+                parent = psutil.Process(pid)
+                children = parent.children(recursive=True)
+                for child in children:
+                    child.kill()
+                parent.kill()
+                logger.info(f"Process {pid} and its children terminated.")
         logger.info("FractalProcessPoolExecutor processes terminated.")
 
     def shutdown(self, *args, **kwargs) -> None:

fractal_server/app/runner/v2/_slurm/__init__.py

@@ -24,16 +24,11 @@ from typing import Union
 from ....models.v2 import DatasetV2
 from ....models.v2 import WorkflowV2
 from ...async_wrap import async_wrap
-from ...executors.slurm.executor import FractalSlurmExecutor
+from ...executors.slurm.sudo.executor import FractalSlurmExecutor
 from ...set_start_and_last_task_index import set_start_and_last_task_index
 from ..runner import execute_tasks_v2
 from ._submit_setup import _slurm_submit_setup
 
-# from .._common import execute_tasks
-# from ..common import async_wrap
-# from ..common import set_start_and_last_task_index
-# from ..common import TaskParameters
-
 
 def _process_workflow(
     *,

fractal_server/app/runner/v2/_slurm_ssh/__init__.py

@@ -0,0 +1,125 @@
+# Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
+# University of Zurich
+#
+# Original authors:
+# Jacopo Nespolo <jacopo.nespolo@exact-lab.it>
+# Tommaso Comparin <tommaso.comparin@exact-lab.it>
+# Marco Franzon <marco.franzon@exact-lab.it>
+#
+# This file is part of Fractal and was originally developed by eXact lab S.r.l.
+# <exact-lab.it> under contract with Liberali Lab from the Friedrich Miescher
+# Institute for Biomedical Research and Pelkmans Lab from the University of
+# Zurich.
+"""
+Slurm Bakend
+
+This backend runs fractal workflows in a SLURM cluster using Clusterfutures
+Executor objects.
+"""
+from pathlib import Path
+from typing import Any
+from typing import Optional
+from typing import Union
+
+from .....ssh._fabric import FractalSSH
+from ....models.v2 import DatasetV2
+from ....models.v2 import WorkflowV2
+from ...async_wrap import async_wrap
+from ...executors.slurm.ssh.executor import FractalSlurmSSHExecutor
+from ...set_start_and_last_task_index import set_start_and_last_task_index
+from ..runner import execute_tasks_v2
+from ._submit_setup import _slurm_submit_setup
+
+
+def _process_workflow(
+    *,
+    workflow: WorkflowV2,
+    dataset: DatasetV2,
+    logger_name: str,
+    workflow_dir_local: Path,
+    workflow_dir_remote: Path,
+    first_task_index: int,
+    last_task_index: int,
+    fractal_ssh: FractalSSH,
+    worker_init: Optional[Union[str, list[str]]] = None,
+) -> dict[str, Any]:
+    """
+    Internal processing routine for the SLURM backend
+
+    This function initialises the a FractalSlurmExecutor, setting logging,
+    workflow working dir and user to impersonate. It then schedules the
+    workflow tasks and returns the new dataset attributes
+
+    Cf.
+    [process_workflow][fractal_server.app.runner.v2._local.process_workflow]
+
+    Returns:
+        new_dataset_attributes:
+    """
+
+    if isinstance(worker_init, str):
+        worker_init = worker_init.split("\n")
+
+    with FractalSlurmSSHExecutor(
+        fractal_ssh=fractal_ssh,
+        workflow_dir_local=workflow_dir_local,
+        workflow_dir_remote=workflow_dir_remote,
+        common_script_lines=worker_init,
+    ) as executor:
+        new_dataset_attributes = execute_tasks_v2(
+            wf_task_list=workflow.task_list[
+                first_task_index : (last_task_index + 1)  # noqa
+            ],  # noqa
+            dataset=dataset,
+            executor=executor,
+            workflow_dir_local=workflow_dir_local,
+            workflow_dir_remote=workflow_dir_remote,
+            logger_name=logger_name,
+            submit_setup_call=_slurm_submit_setup,
+        )
+    return new_dataset_attributes
+
+
+async def process_workflow(
+    *,
+    workflow: WorkflowV2,
+    dataset: DatasetV2,
+    workflow_dir_local: Path,
+    workflow_dir_remote: Optional[Path] = None,
+    first_task_index: Optional[int] = None,
+    last_task_index: Optional[int] = None,
+    logger_name: str,
+    # Not used
+    fractal_ssh: FractalSSH,
+    user_cache_dir: Optional[str] = None,
+    slurm_user: Optional[str] = None,
+    slurm_account: Optional[str] = None,
+    worker_init: Optional[str] = None,
+) -> dict:
+    """
+    Process workflow (SLURM backend public interface)
+
+    Cf.
+    [process_workflow][fractal_server.app.runner.v2._local.process_workflow]
+    """
+
+    # Set values of first_task_index and last_task_index
+    num_tasks = len(workflow.task_list)
+    first_task_index, last_task_index = set_start_and_last_task_index(
+        num_tasks,
+        first_task_index=first_task_index,
+        last_task_index=last_task_index,
+    )
+
+    new_dataset_attributes = await async_wrap(_process_workflow)(
+        workflow=workflow,
+        dataset=dataset,
+        logger_name=logger_name,
+        workflow_dir_local=workflow_dir_local,
+        workflow_dir_remote=workflow_dir_remote,
+        first_task_index=first_task_index,
+        last_task_index=last_task_index,
+        worker_init=worker_init,
+        fractal_ssh=fractal_ssh,
+    )
+    return new_dataset_attributes

fractal_server/app/runner/v2/_slurm_ssh/_submit_setup.py

@@ -0,0 +1,83 @@
+# Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
+# University of Zurich
+#
+# Original authors:
+# Jacopo Nespolo <jacopo.nespolo@exact-lab.it>
+# Tommaso Comparin <tommaso.comparin@exact-lab.it>
+#
+# This file is part of Fractal and was originally developed by eXact lab S.r.l.
+# <exact-lab.it> under contract with Liberali Lab from the Friedrich Miescher
+# Institute for Biomedical Research and Pelkmans Lab from the University of
+# Zurich.
+"""
+Submodule to define _slurm_submit_setup, which is also the reference
+implementation of `submit_setup_call`.
+"""
+from pathlib import Path
+from typing import Literal
+
+from ...task_files import get_task_file_paths
+from .get_slurm_config import get_slurm_config
+from fractal_server.app.models.v2 import WorkflowTaskV2
+
+
+def _slurm_submit_setup(
+    *,
+    wftask: WorkflowTaskV2,
+    workflow_dir_local: Path,
+    workflow_dir_remote: Path,
+    which_type: Literal["non_parallel", "parallel"],
+) -> dict[str, object]:
+    """
+    Collect WorfklowTask-specific configuration parameters from different
+    sources, and inject them for execution.
+
+    Here goes all the logic for reading attributes from the appropriate sources
+    and transforming them into an appropriate `SlurmConfig` object (encoding
+    SLURM configuration) and `TaskFiles` object (with details e.g. about file
+    paths or filename prefixes).
+
+    For now, this is the reference implementation for the argument
+    `submit_setup_call` of
+    [fractal_server.app.runner.v2.runner][].
+
+    Arguments:
+        wftask:
+            WorkflowTask for which the configuration is to be assembled
+        workflow_dir_local:
+            Server-owned directory to store all task-execution-related relevant
+            files (inputs, outputs, errors, and all meta files related to the
+            job execution). Note: users cannot write directly to this folder.
+        workflow_dir_remote:
+            User-side directory with the same scope as `workflow_dir_local`,
+            and where a user can write.
+
+    Returns:
+        submit_setup_dict:
+            A dictionary that will be passed on to
+            `FractalSlurmExecutor.submit` and `FractalSlurmExecutor.map`, so
+            as to set extra options.
+    """
+
+    # Get SlurmConfig object
+    slurm_config = get_slurm_config(
+        wftask=wftask,
+        workflow_dir_local=workflow_dir_local,
+        workflow_dir_remote=workflow_dir_remote,
+        which_type=which_type,
+    )
+
+    # Get TaskFiles object
+    task_files = get_task_file_paths(
+        workflow_dir_local=workflow_dir_local,
+        workflow_dir_remote=workflow_dir_remote,
+        task_order=wftask.order,
+        task_name=wftask.task.name,
+    )
+
+    # Prepare and return output dictionary
+    submit_setup_dict = dict(
+        slurm_config=slurm_config,
+        task_files=task_files,
+    )
+    return submit_setup_dict

fractal_server/app/runner/v2/_slurm_ssh/get_slurm_config.py

@@ -0,0 +1,182 @@
+from pathlib import Path
+from typing import Literal
+from typing import Optional
+
+from fractal_server.app.models.v2 import WorkflowTaskV2
+from fractal_server.app.runner.executors.slurm._slurm_config import (
+    _parse_mem_value,
+)
+from fractal_server.app.runner.executors.slurm._slurm_config import (
+    load_slurm_config_file,
+)
+from fractal_server.app.runner.executors.slurm._slurm_config import logger
+from fractal_server.app.runner.executors.slurm._slurm_config import SlurmConfig
+from fractal_server.app.runner.executors.slurm._slurm_config import (
+    SlurmConfigError,
+)
+
+
+def get_slurm_config(
+    wftask: WorkflowTaskV2,
+    workflow_dir_local: Path,
+    workflow_dir_remote: Path,
+    which_type: Literal["non_parallel", "parallel"],
+    config_path: Optional[Path] = None,
+) -> SlurmConfig:
+    """
+    Prepare a `SlurmConfig` configuration object
+
+    The argument `which_type` determines whether we use `wftask.meta_parallel`
+    or `wftask.meta_non_parallel`. In the following descritpion, let us assume
+    that `which_type="parallel"`.
+
+    The sources for `SlurmConfig` attributes, in increasing priority order, are
+
+    1. The general content of the Fractal SLURM configuration file.
+    2. The GPU-specific content of the Fractal SLURM configuration file, if
+        appropriate.
+    3. Properties in `wftask.meta_parallel` (which typically include those in
+       `wftask.task.meta_parallel`). Note that `wftask.meta_parallel` may be
+       `None`.
+
+    Arguments:
+        wftask:
+            WorkflowTask for which the SLURM configuration is is to be
+            prepared.
+        workflow_dir_local:
+            Server-owned directory to store all task-execution-related relevant
+            files (inputs, outputs, errors, and all meta files related to the
+            job execution). Note: users cannot write directly to this folder.
+        workflow_dir_remote:
+            User-side directory with the same scope as `workflow_dir_local`,
+            and where a user can write.
+        config_path:
+            Path of a Fractal SLURM configuration file; if `None`, use
+            `FRACTAL_SLURM_CONFIG_FILE` variable from settings.
+        which_type:
+            Determines whether to use `meta_parallel` or `meta_non_parallel`.
+
+    Returns:
+        slurm_config:
+            The SlurmConfig object
+    """
+
+    if which_type == "non_parallel":
+        wftask_meta = wftask.meta_non_parallel
+    elif which_type == "parallel":
+        wftask_meta = wftask.meta_parallel
+    else:
+        raise ValueError(
+            f"get_slurm_config received invalid argument {which_type=}."
+        )
+
+    logger.debug(
+        f"[get_slurm_config] WorkflowTask meta attribute: {wftask_meta=}"
+    )
+
+    # Incorporate slurm_env.default_slurm_config
+    slurm_env = load_slurm_config_file(config_path=config_path)
+    slurm_dict = slurm_env.default_slurm_config.dict(
+        exclude_unset=True, exclude={"mem"}
+    )
+    if slurm_env.default_slurm_config.mem:
+        slurm_dict["mem_per_task_MB"] = slurm_env.default_slurm_config.mem
+
+    # Incorporate slurm_env.batching_config
+    for key, value in slurm_env.batching_config.dict().items():
+        slurm_dict[key] = value
+
+    # Incorporate slurm_env.user_local_exports
+    slurm_dict["user_local_exports"] = slurm_env.user_local_exports
+
+    logger.debug(
+        "[get_slurm_config] Fractal SLURM configuration file: "
+        f"{slurm_env.dict()=}"
+    )
+
+    # GPU-related options
+    # Notes about priority:
+    # 1. This block of definitions takes priority over other definitions from
+    #    slurm_env which are not under the `needs_gpu` subgroup
+    # 2. This block of definitions has lower priority than whatever comes next
+    #    (i.e. from WorkflowTask.meta).
+    if wftask_meta is not None:
+        needs_gpu = wftask_meta.get("needs_gpu", False)
+    else:
+        needs_gpu = False
+    logger.debug(f"[get_slurm_config] {needs_gpu=}")
+    if needs_gpu and slurm_env.gpu_slurm_config is not None:  # FIXME
+        for key, value in slurm_env.gpu_slurm_config.dict(
+            exclude_unset=True, exclude={"mem"}
+        ).items():
+            slurm_dict[key] = value
+        if slurm_env.gpu_slurm_config.mem:
+            slurm_dict["mem_per_task_MB"] = slurm_env.gpu_slurm_config.mem
+
+    # Number of CPUs per task, for multithreading
+    if wftask_meta is not None and "cpus_per_task" in wftask_meta:
+        cpus_per_task = int(wftask_meta["cpus_per_task"])
+        slurm_dict["cpus_per_task"] = cpus_per_task
+
+    # Required memory per task, in MB
+    if wftask_meta is not None and "mem" in wftask_meta:
+        raw_mem = wftask_meta["mem"]
+        mem_per_task_MB = _parse_mem_value(raw_mem)
+        slurm_dict["mem_per_task_MB"] = mem_per_task_MB
+
+    # Job name
+    if wftask.is_legacy_task:
+        job_name = wftask.task_legacy.name.replace(" ", "_")
+    else:
+        job_name = wftask.task.name.replace(" ", "_")
+    slurm_dict["job_name"] = job_name
+
+    # Optional SLURM arguments and extra lines
+    if wftask_meta is not None:
+        account = wftask_meta.get("account", None)
+        if account is not None:
+            error_msg = (
+                f"Invalid {account=} property in WorkflowTask `meta` "
+                "attribute.\n"
+                "SLURM account must be set in the request body of the "
+                "apply-workflow endpoint, or by modifying the user properties."
+            )
+            logger.error(error_msg)
+            raise SlurmConfigError(error_msg)
+        for key in ["time", "gres", "constraint"]:
+            value = wftask_meta.get(key, None)
+            if value:
+                slurm_dict[key] = value
+    if wftask_meta is not None:
+        extra_lines = wftask_meta.get("extra_lines", [])
+    else:
+        extra_lines = []
+    extra_lines = slurm_dict.get("extra_lines", []) + extra_lines
+    if len(set(extra_lines)) != len(extra_lines):
+        logger.debug(
+            "[get_slurm_config] Removing repeated elements "
+            f"from {extra_lines=}."
+        )
+        extra_lines = list(set(extra_lines))
+    slurm_dict["extra_lines"] = extra_lines
+
+    # Job-batching parameters (if None, they will be determined heuristically)
+    if wftask_meta is not None:
+        tasks_per_job = wftask_meta.get("tasks_per_job", None)
+        parallel_tasks_per_job = wftask_meta.get(
+            "parallel_tasks_per_job", None
+        )
+    else:
+        tasks_per_job = None
+        parallel_tasks_per_job = None
+    slurm_dict["tasks_per_job"] = tasks_per_job
+    slurm_dict["parallel_tasks_per_job"] = parallel_tasks_per_job
+
+    # Put everything together
+    logger.debug(
+        "[get_slurm_config] Now create a SlurmConfig object based "
+        f"on {slurm_dict=}"
+    )
+    slurm_config = SlurmConfig(**slurm_dict)
+
+    return slurm_config

fractal_server/app/runner/v2/runner_functions_low_level.py

@@ -34,17 +34,15 @@ def _call_command_wrapper(cmd: str, log_path: Path) -> None:
         )
         raise TaskExecutionError(msg)
 
-    fp_log = open(log_path, "w")
-    try:
-        result = subprocess.run(  # nosec
-            shlex_split(cmd),
-            stderr=fp_log,
-            stdout=fp_log,
-        )
-    except Exception as e:
-        raise e
-    finally:
-        fp_log.close()
+    with open(log_path, "w") as fp_log:
+        try:
+            result = subprocess.run(  # nosec
+                shlex_split(cmd),
+                stderr=fp_log,
+                stdout=fp_log,
+            )
+        except Exception as e:
+            raise e
 
     if result.returncode > 0:
         with log_path.open("r") as fp_stderr:

fractal_server/app/runner/versions.py

@@ -0,0 +1,30 @@
+import json
+import sys
+from typing import Union
+
+import cloudpickle
+
+import fractal_server
+
+
+def get_versions() -> dict[str, Union[list[int], str]]:
+    """
+    Extract versions of Python, fractal-server and cloudpickle.
+
+    This information is useful to check compatibility of two Python
+    interpreters when running tasks: the current interpreter (which prepares
+    the input pickles and orchestrates workflow execution) and a remote
+    interpreter (e.g. the one defined in the `FRACTAL_SLURM_WORKER_PYTHON`
+    configuration variable) that executes the tasks.
+    """
+    versions = dict(
+        python=list(sys.version_info[:3]),
+        cloudpickle=cloudpickle.__version__,
+        fractal_server=fractal_server.__VERSION__,
+    )
+    return versions
+
+
+if __name__ == "__main__":
+    versions = get_versions()
+    print(json.dumps(versions))

fractal_server/app/schemas/v1/__init__.py

@@ -17,6 +17,7 @@ from .manifest import TaskManifestV1  # noqa: F401
 from .project import ProjectCreateV1  # noqa: F401
 from .project import ProjectReadV1  # noqa: F401
 from .project import ProjectUpdateV1  # noqa: F401
+from .state import StateRead  # noqa: F401
 from .task import TaskCreateV1  # noqa: F401
 from .task import TaskImportV1  # noqa: F401
 from .task import TaskReadV1  # noqa: F401

fractal_server/app/schemas/{state.py → v1/state.py}

@@ -5,29 +5,10 @@ from typing import Optional
 from pydantic import BaseModel
 from pydantic import validator
 
-from ._validators import valutc
+from fractal_server.app.schemas._validators import valutc
 
-__all__ = (
-    "_StateBase",
-    "StateRead",
-)
 
-
-class _StateBase(BaseModel):
-    """
-    Base class for `State`.
-
-    Attributes:
-        id: Primary key
-        data: Content of the state
-        timestamp: Time stamp of the state
-    """
-
-    data: dict[str, Any]
-    timestamp: datetime
-
-
-class StateRead(_StateBase):
+class StateRead(BaseModel):
     """
     Class for `State` read from database.
 
@@ -36,5 +17,7 @@ class StateRead(_StateBase):
     """
 
     id: Optional[int]
+    data: dict[str, Any]
+    timestamp: datetime
 
     _timestamp = validator("timestamp", allow_reuse=True)(valutc("timestamp"))

fractal_server/app/schemas/v2/__init__.py

@@ -13,6 +13,7 @@ from .job import JobReadV2  # noqa F401
 from .job import JobStatusTypeV2  # noqa F401
 from .job import JobUpdateV2  # noqa F401
 from .manifest import ManifestV2  # noqa F401
+from .manifest import TaskManifestV2  # noqa F401
 from .project import ProjectCreateV2  # noqa F401
 from .project import ProjectReadV2  # noqa F401
 from .project import ProjectUpdateV2  # noqa F401
@@ -22,8 +23,10 @@ from .task import TaskImportV2  # noqa F401
 from .task import TaskLegacyReadV2  # noqa F401
 from .task import TaskReadV2  # noqa F401
 from .task import TaskUpdateV2  # noqa F401
+from .task_collection import CollectionStateReadV2  # noqa F401
+from .task_collection import CollectionStatusV2  # noqa F401
+from .task_collection import TaskCollectCustomV2  # noqa F401
 from .task_collection import TaskCollectPipV2  # noqa F401
-from .task_collection import TaskCollectStatusV2  # noqa F401
 from .workflow import WorkflowCreateV2  # noqa F401
 from .workflow import WorkflowExportV2  # noqa F401
 from .workflow import WorkflowImportV2  # noqa F401