fractal-server 2.12.1__py3-none-any.whl → 2.13.0__py3-none-any.whl

fractal_server/app/runner/v2/_local_experimental/__init__.py

@@ -1,121 +0,0 @@
-from concurrent.futures.process import BrokenProcessPool
-from pathlib import Path
-from typing import Optional
-
-from ....models.v2 import DatasetV2
-from ....models.v2 import WorkflowV2
-from ...exceptions import JobExecutionError
-from ...filenames import SHUTDOWN_FILENAME
-from ...set_start_and_last_task_index import set_start_and_last_task_index
-from ..runner import execute_tasks_v2
-from ._submit_setup import _local_submit_setup
-from .executor import FractalProcessPoolExecutor
-from fractal_server.images.models import AttributeFiltersType
-
-
-def process_workflow(
-    *,
-    workflow: WorkflowV2,
-    dataset: DatasetV2,
-    workflow_dir_local: Path,
-    workflow_dir_remote: Optional[Path] = None,
-    first_task_index: Optional[int] = None,
-    last_task_index: Optional[int] = None,
-    logger_name: str,
-    job_attribute_filters: AttributeFiltersType,
-    # Slurm-specific
-    user_cache_dir: Optional[str] = None,
-    slurm_user: Optional[str] = None,
-    slurm_account: Optional[str] = None,
-    worker_init: Optional[str] = None,
-) -> None:
-    """
-    Run a workflow
-
-    This function is responsible for running a workflow on some input data,
-    saving the output and taking care of any exception raised during the run.
-
-    NOTE: This is the `local_experimental` backend's public interface,
-    which also works as a reference implementation for other backends.
-
-    Args:
-        workflow:
-            The workflow to be run
-        dataset:
-            Initial dataset.
-        workflow_dir_local:
-            Working directory for this run.
-        workflow_dir_remote:
-            Working directory for this run, on the user side. This argument is
-            present for compatibility with the standard backend interface, but
-            for the `local` backend it cannot be different from
-            `workflow_dir_local`.
-        first_task_index:
-            Positional index of the first task to execute; if `None`, start
-            from `0`.
-        last_task_index:
-            Positional index of the last task to execute; if `None`, proceed
-            until the last task.
-        logger_name: Logger name
-        slurm_user:
-            Username to impersonate to run the workflow. This argument is
-            present for compatibility with the standard backend interface, but
-            is ignored in the `local` backend.
-        slurm_account:
-            SLURM account to use when running the workflow. This argument is
-            present for compatibility with the standard backend interface, but
-            is ignored in the `local` backend.
-        user_cache_dir:
-            Cache directory of the user who will run the workflow. This
-            argument is present for compatibility with the standard backend
-            interface, but is ignored in the `local` backend.
-        worker_init:
-            Any additional, usually backend specific, information to be passed
-            to the backend executor. This argument is present for compatibility
-            with the standard backend interface, but is ignored in the `local`
-            backend.
-
-    Raises:
-        TaskExecutionError: wrapper for errors raised during tasks' execution
-                            (positive exit codes).
-        JobExecutionError: wrapper for errors raised by the tasks' executors
-                           (negative exit codes).
-    """
-
-    if workflow_dir_remote and (workflow_dir_remote != workflow_dir_local):
-        raise NotImplementedError(
-            "LocalExperimental backend does not support different directories "
-            f"{workflow_dir_local=} and {workflow_dir_remote=}"
-        )
-
-    # Set values of first_task_index and last_task_index
-    num_tasks = len(workflow.task_list)
-    first_task_index, last_task_index = set_start_and_last_task_index(
-        num_tasks,
-        first_task_index=first_task_index,
-        last_task_index=last_task_index,
-    )
-
-    with FractalProcessPoolExecutor(
-        shutdown_file=workflow_dir_local / SHUTDOWN_FILENAME
-    ) as executor:
-        try:
-            execute_tasks_v2(
-                wf_task_list=workflow.task_list[
-                    first_task_index : (last_task_index + 1)
-                ],
-                dataset=dataset,
-                executor=executor,
-                workflow_dir_local=workflow_dir_local,
-                workflow_dir_remote=workflow_dir_local,
-                logger_name=logger_name,
-                submit_setup_call=_local_submit_setup,
-                job_attribute_filters=job_attribute_filters,
-            )
-        except BrokenProcessPool as e:
-            raise JobExecutionError(
-                info=(
-                    "Job failed with BrokenProcessPool error, likely due to "
-                    f"an executor shutdown.\nOriginal error:\n{e.args[0]}"
-                )
-            )

fractal_server/app/runner/v2/_local_experimental/_local_config.py

@@ -1,108 +0,0 @@
-"""
-Submodule to handle the local-backend configuration for a WorkflowTask
-"""
-import json
-from pathlib import Path
-from typing import Literal
-from typing import Optional
-
-from pydantic import BaseModel
-from pydantic import Extra
-from pydantic.error_wrappers import ValidationError
-
-from .....config import get_settings
-from .....syringe import Inject
-from ....models.v2 import WorkflowTaskV2
-
-
-class LocalBackendConfigError(ValueError):
-    """
-    Local-backend configuration error
-    """
-
-    pass
-
-
-class LocalBackendConfig(BaseModel, extra=Extra.forbid):
-    """
-    Specifications of the local-backend configuration
-
-    Attributes:
-        parallel_tasks_per_job:
-            Maximum number of tasks to be run in parallel as part of a call to
-            `FractalProcessPoolExecutor.map`; if `None`, then all tasks will
-            start at the same time.
-    """
-
-    parallel_tasks_per_job: Optional[int]
-
-
-def get_default_local_backend_config():
-    """
-    Return a default `LocalBackendConfig` configuration object
-    """
-    return LocalBackendConfig(parallel_tasks_per_job=None)
-
-
-def get_local_backend_config(
-    wftask: WorkflowTaskV2,
-    which_type: Literal["non_parallel", "parallel"],
-    config_path: Optional[Path] = None,
-) -> LocalBackendConfig:
-    """
-    Prepare a `LocalBackendConfig` configuration object
-
-    The sources for `parallel_tasks_per_job` attributes, starting from the
-    highest-priority one, are
-
-    1. Properties in `wftask.meta_parallel` or `wftask.meta_non_parallel`
-       (depending on `which_type`);
-    2. The general content of the local-backend configuration file;
-    3. The default value (`None`).
-
-    Arguments:
-        wftask:
-            WorkflowTaskV2 for which the backend configuration should
-            be prepared.
-        config_path:
-            Path of local-backend configuration file; if `None`, use
-            `FRACTAL_LOCAL_CONFIG_FILE` variable from settings.
-
-    Returns:
-        A local-backend configuration object
-    """
-
-    key = "parallel_tasks_per_job"
-    default_value = None
-
-    if which_type == "non_parallel":
-        wftask_meta = wftask.meta_non_parallel
-    elif which_type == "parallel":
-        wftask_meta = wftask.meta_parallel
-    else:
-        raise ValueError(
-            "`get_local_backend_config` received an invalid argument"
-            f" {which_type=}."
-        )
-
-    if wftask_meta and key in wftask_meta:
-        parallel_tasks_per_job = wftask_meta[key]
-    else:
-        if not config_path:
-            settings = Inject(get_settings)
-            config_path = settings.FRACTAL_LOCAL_CONFIG_FILE
-        if config_path is None:
-            parallel_tasks_per_job = default_value
-        else:
-            with config_path.open("r") as f:
-                env = json.load(f)
-            try:
-                _ = LocalBackendConfig(**env)
-            except ValidationError as e:
-                raise LocalBackendConfigError(
-                    f"Error while loading {config_path=}. "
-                    f"Original error:\n{str(e)}"
-                )
-
-            parallel_tasks_per_job = env.get(key, default_value)
-    return LocalBackendConfig(parallel_tasks_per_job=parallel_tasks_per_job)

fractal_server/app/runner/v2/_local_experimental/_submit_setup.py

@@ -1,42 +0,0 @@
-"""
-Submodule to define _local_submit_setup
-"""
-from pathlib import Path
-from typing import Literal
-from typing import Optional
-
-from ....models.v2 import WorkflowTaskV2
-from ._local_config import get_local_backend_config
-
-
-def _local_submit_setup(
-    *,
-    wftask: WorkflowTaskV2,
-    workflow_dir_local: Optional[Path] = None,
-    workflow_dir_remote: Optional[Path] = None,
-    which_type: Literal["non_parallel", "parallel"],
-) -> dict[str, object]:
-    """
-    Collect WorfklowTask-specific configuration parameters from different
-    sources, and inject them for execution.
-
-    Arguments:
-        wftask:
-            WorkflowTask for which the configuration is to be assembled
-        workflow_dir_local:
-            Not used in this function.
-        workflow_dir_remote:
-            Not used in this function.
-
-    Returns:
-        submit_setup_dict:
-            A dictionary that will be passed on to
-            `FractalProcessPoolExecutor.submit` and
-            `FractalProcessPoolExecutor.map`, so as to set extra options.
-    """
-
-    local_backend_config = get_local_backend_config(
-        wftask=wftask, which_type=which_type
-    )
-
-    return dict(local_backend_config=local_backend_config)

fractal_server/app/runner/v2/_local_experimental/executor.py

@@ -1,157 +0,0 @@
-"""
-Custom version of Python
-[ProcessPoolExecutor](https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ProcessPoolExecutor)).
-"""
-import multiprocessing as mp
-import threading
-import time
-from concurrent.futures import ProcessPoolExecutor
-from concurrent.futures.process import BrokenProcessPool
-from pathlib import Path
-from typing import Callable
-from typing import Iterable
-from typing import Optional
-from typing import Sequence
-
-import psutil
-
-from ._local_config import get_default_local_backend_config
-from ._local_config import LocalBackendConfig
-from fractal_server.app.runner.exceptions import JobExecutionError
-from fractal_server.logger import set_logger
-
-logger = set_logger("FractalProcessPoolExecutor")
-
-
-class FractalProcessPoolExecutor(ProcessPoolExecutor):
-
-    shutdown_file: Path
-    interval: float
-    _shutdown: bool
-    _shutdown_file_thread: threading.Thread
-
-    def __init__(
-        self, shutdown_file: Path, interval: float = 1.0, *args, **kwargs
-    ):
-        super().__init__(*args, **kwargs, mp_context=mp.get_context("spawn"))
-        self.shutdown_file = Path(shutdown_file)
-        self.interval = float(interval)
-        logger.debug(
-            f"Start monitoring {shutdown_file} every {interval} seconds"
-        )
-        self._shutdown = False
-        self._shutdown_file_thread = threading.Thread(
-            target=self._run, daemon=True
-        )
-        self._shutdown_file_thread.start()
-
-    def _run(self):
-        """
-        Running on '_shutdown_file_thread'.
-        """
-        while True:
-            if self.shutdown_file.exists() or self._shutdown:
-                try:
-                    self._terminate_processes()
-                except Exception as e:
-                    logger.error(
-                        "Terminate processes failed. "
-                        f"Original error: {str(e)}."
-                    )
-                finally:
-                    return
-            time.sleep(self.interval)
-
-    def _terminate_processes(self):
-        """
-        Running on '_shutdown_file_thread'.
-        """
-
-        logger.info("Start terminating FractalProcessPoolExecutor processes.")
-        # We use 'psutil' in order to easily access the PIDs of the children.
-        if self._processes is not None:
-            for pid in self._processes.keys():
-                parent = psutil.Process(pid)
-                children = parent.children(recursive=True)
-                for child in children:
-                    child.kill()
-                parent.kill()
-                logger.info(f"Process {pid} and its children terminated.")
-        logger.info("FractalProcessPoolExecutor processes terminated.")
-
-    def shutdown(self, *args, **kwargs) -> None:
-        self._shutdown = True
-        self._shutdown_file_thread.join()
-        return super().shutdown(*args, **kwargs)
-
-    def submit(
-        self,
-        *args,
-        local_backend_config: Optional[LocalBackendConfig] = None,
-        **kwargs,
-    ):
-        """
-        Compared to the `ProcessPoolExecutor` method, here we accept an
-        additional keyword argument (`local_backend_config`), which is then
-        simply ignored.
-        """
-        return super().submit(*args, **kwargs)
-
-    def map(
-        self,
-        fn: Callable,
-        *iterables: Sequence[Iterable],
-        local_backend_config: Optional[LocalBackendConfig] = None,
-    ):
-        """
-        Custom version of the `Executor.map` method
-
-        The main change with the respect to the original `map` method is that
-        the list of tasks to be executed is split into chunks, and then
-        `super().map` is called (sequentially) on each chunk. The goal of this
-        change is to limit parallelism, e.g. due to limited computational
-        resources.
-
-        Other changes from the `concurrent.futures` `map` method:
-
-        1. Removed `timeout` argument;
-        2. Removed `chunksize`;
-        3. All iterators (both inputs and output ones) are transformed into
-           lists.
-
-        Args:
-            fn: A callable function.
-            iterables: The argument iterables (one iterable per argument of
-                       `fn`).
-           local_backend_config: The backend configuration, needed to extract
-                                 `parallel_tasks_per_job`.
-        """
-        # Preliminary check
-        iterable_lengths = [len(it) for it in iterables]
-        if not len(set(iterable_lengths)) == 1:
-            raise ValueError("Iterables have different lengths.")
-        # Set total number of arguments
-        n_elements = len(iterables[0])
-
-        # Set parallel_tasks_per_job
-        if local_backend_config is None:
-            local_backend_config = get_default_local_backend_config()
-        parallel_tasks_per_job = local_backend_config.parallel_tasks_per_job
-        if parallel_tasks_per_job is None:
-            parallel_tasks_per_job = n_elements
-
-        # Execute tasks, in chunks of size parallel_tasks_per_job
-        results = []
-        for ind_chunk in range(0, n_elements, parallel_tasks_per_job):
-            chunk_iterables = [
-                it[ind_chunk : ind_chunk + parallel_tasks_per_job]  # noqa
-                for it in iterables
-            ]
-            map_iter = super().map(fn, *chunk_iterables)
-
-            try:
-                results.extend(list(map_iter))
-            except BrokenProcessPool as e:
-                raise JobExecutionError(info=e.args[0])
-
-        return iter(results)