executorlib 0.3.0__tar.gz → 0.4.1__tar.gz

{executorlib-0.3.0/executorlib.egg-info → executorlib-0.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: executorlib
-Version: 0.3.0
+Version: 0.4.1
 Summary: Up-scale python functions for high performance computing (HPC) with executorlib.
 Author-email: Jan Janssen <janssen@lanl.gov>
 License: BSD 3-Clause License
@@ -53,7 +53,7 @@ License-File: LICENSE
 Requires-Dist: cloudpickle<=3.1.1,>=2.0.0
 Requires-Dist: pyzmq<=26.2.1,>=25.0.0
 Provides-Extra: cache
-Requires-Dist: h5py<=3.12.1,>=3.6.0; extra == "cache"
+Requires-Dist: h5py<=3.13.0,>=3.6.0; extra == "cache"
 Provides-Extra: graph
 Requires-Dist: pygraphviz<=1.14,>=1.10; extra == "graph"
 Requires-Dist: networkx<=3.4.2,>=2.8.8; extra == "graph"
@@ -65,11 +65,11 @@ Provides-Extra: mpi
 Requires-Dist: mpi4py<=4.0.1,>=3.1.4; extra == "mpi"
 Provides-Extra: cluster
 Requires-Dist: pysqa==0.2.3; extra == "cluster"
-Requires-Dist: h5py<=3.12.1,>=3.6.0; extra == "cluster"
+Requires-Dist: h5py<=3.13.0,>=3.6.0; extra == "cluster"
 Provides-Extra: all
 Requires-Dist: mpi4py<=4.0.1,>=3.1.4; extra == "all"
 Requires-Dist: pysqa==0.2.3; extra == "all"
-Requires-Dist: h5py<=3.12.1,>=3.6.0; extra == "all"
+Requires-Dist: h5py<=3.13.0,>=3.6.0; extra == "all"
 Requires-Dist: pygraphviz<=1.14,>=1.10; extra == "all"
 Requires-Dist: networkx<=3.4.2,>=2.8.8; extra == "all"
 Requires-Dist: ipython<=8.32.0,>=7.33.0; extra == "all"

{executorlib-0.3.0 → executorlib-0.4.1}/executorlib/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-02-14T16:51:45+0100",
+ "date": "2025-02-19T13:21:35+0100",
  "dirty": true,
  "error": null,
- "full-revisionid": "ff49de3af89e8739b3b3c5fa041055cf4bc9bc14",
- "version": "0.3.0"
+ "full-revisionid": "4f9ef5fb9005188f0b6cc87ef7c3e7fc6006f3e0",
+ "version": "0.4.1"
 }
 '''  # END VERSION_JSON
 

{executorlib-0.3.0 → executorlib-0.4.1}/executorlib/base/executor.py

@@ -27,10 +27,19 @@ class ExecutorBase(FutureExecutor):
         Initialize the ExecutorBase class.
         """
         cloudpickle_register(ind=3)
+        self._process_kwargs: dict = {}
         self._max_cores = max_cores
         self._future_queue: Optional[queue.Queue] = queue.Queue()
         self._process: Optional[Union[Thread, list[Thread]]] = None
 
+    @property
+    def max_workers(self) -> Optional[int]:
+        return self._process_kwargs.get("max_workers")
+
+    @max_workers.setter
+    def max_workers(self, max_workers: int):
+        raise NotImplementedError("The max_workers setter is not implemented.")
+
     @property
     def info(self) -> Optional[dict]:
         """
@@ -39,16 +48,13 @@ class ExecutorBase(FutureExecutor):
         Returns:
             Optional[dict]: Information about the executor.
         """
+        meta_data_dict = self._process_kwargs.copy()
+        if "future_queue" in meta_data_dict:
+            del meta_data_dict["future_queue"]
         if self._process is not None and isinstance(self._process, list):
-            meta_data_dict = self._process[0]._kwargs.copy()  # type: ignore
-            if "future_queue" in meta_data_dict:
-                del meta_data_dict["future_queue"]
             meta_data_dict["max_workers"] = len(self._process)
             return meta_data_dict
         elif self._process is not None:
-            meta_data_dict = self._process._kwargs.copy()  # type: ignore
-            if "future_queue" in meta_data_dict:
-                del meta_data_dict["future_queue"]
             return meta_data_dict
         else:
             return None

{executorlib-0.3.0 → executorlib-0.4.1}/executorlib/cache/executor.py

@@ -63,19 +63,20 @@ class FileExecutor(ExecutorBase):
             terminate_function = terminate_subprocess
         cache_directory_path = os.path.abspath(cache_directory)
         os.makedirs(cache_directory_path, exist_ok=True)
+        self._process_kwargs = {
+            "future_queue": self._future_queue,
+            "execute_function": execute_function,
+            "cache_directory": cache_directory_path,
+            "resource_dict": resource_dict,
+            "terminate_function": terminate_function,
+            "pysqa_config_directory": pysqa_config_directory,
+            "backend": backend,
+            "disable_dependencies": disable_dependencies,
+        }
         self._set_process(
             Thread(
                 target=execute_tasks_h5,
-                kwargs={
-                    "future_queue": self._future_queue,
-                    "execute_function": execute_function,
-                    "cache_directory": cache_directory_path,
-                    "resource_dict": resource_dict,
-                    "terminate_function": terminate_function,
-                    "pysqa_config_directory": pysqa_config_directory,
-                    "backend": backend,
-                    "disable_dependencies": disable_dependencies,
-                },
+                kwargs=self._process_kwargs,
             )
         )
 

executorlib-0.4.1/executorlib/interactive/blockallocation.py

@@ -0,0 +1,176 @@
+import queue
+from concurrent.futures import Future
+from threading import Thread
+from typing import Callable, Optional
+
+from executorlib.base.executor import ExecutorBase, cancel_items_in_queue
+from executorlib.interactive.shared import execute_tasks
+from executorlib.standalone.inputcheck import (
+    check_resource_dict,
+    check_resource_dict_is_empty,
+)
+from executorlib.standalone.interactive.spawner import BaseSpawner, MpiExecSpawner
+
+
+class BlockAllocationExecutor(ExecutorBase):
+    """
+    The executorlib.interactive.executor.InteractiveExecutor leverages the exeutorlib interfaces to distribute python
+    tasks on a workstation or inside a queuing system allocation. In contrast to the mpi4py.futures.MPIPoolExecutor the
+    executorlib.interactive.executor.InteractiveExecutor can be executed in a serial python process and does not require
+    the python script to be executed with MPI. Consequently, it is primarily an abstraction of its functionality to
+    improves the usability in particular when used in combination with Jupyter notebooks.
+
+    Args:
+        max_workers (int): defines the number workers which can execute functions in parallel
+        executor_kwargs (dict): keyword arguments for the executor
+        spawner (BaseSpawner): interface class to initiate python processes
+
+    Examples:
+
+        >>> import numpy as np
+        >>> from executorlib.interactive.blockallocation import BlockAllocationExecutor
+        >>>
+        >>> def calc(i, j, k):
+        >>>     from mpi4py import MPI
+        >>>     size = MPI.COMM_WORLD.Get_size()
+        >>>     rank = MPI.COMM_WORLD.Get_rank()
+        >>>     return np.array([i, j, k]), size, rank
+        >>>
+        >>> def init_k():
+        >>>     return {"k": 3}
+        >>>
+        >>> with BlockAllocationExecutor(max_workers=2, executor_kwargs={"init_function": init_k}) as p:
+        >>>     fs = p.submit(calc, 2, j=4)
+        >>>     print(fs.result())
+        [(array([2, 4, 3]), 2, 0), (array([2, 4, 3]), 2, 1)]
+
+    """
+
+    def __init__(
+        self,
+        max_workers: int = 1,
+        executor_kwargs: Optional[dict] = None,
+        spawner: type[BaseSpawner] = MpiExecSpawner,
+    ):
+        if executor_kwargs is None:
+            executor_kwargs = {}
+        super().__init__(max_cores=executor_kwargs.get("max_cores"))
+        executor_kwargs["future_queue"] = self._future_queue
+        executor_kwargs["spawner"] = spawner
+        executor_kwargs["queue_join_on_shutdown"] = False
+        self._process_kwargs = executor_kwargs
+        self._max_workers = max_workers
+        self._set_process(
+            process=[
+                Thread(
+                    target=execute_tasks,
+                    kwargs=executor_kwargs,
+                )
+                for _ in range(self._max_workers)
+            ],
+        )
+
+    @property
+    def max_workers(self) -> int:
+        return self._max_workers
+
+    @max_workers.setter
+    def max_workers(self, max_workers: int):
+        if isinstance(self._future_queue, queue.Queue) and isinstance(
+            self._process, list
+        ):
+            if self._max_workers > max_workers:
+                for _ in range(self._max_workers - max_workers):
+                    self._future_queue.queue.insert(0, {"shutdown": True, "wait": True})
+                while len(self._process) > max_workers:
+                    self._process = [
+                        process for process in self._process if process.is_alive()
+                    ]
+            elif self._max_workers < max_workers:
+                new_process_lst = [
+                    Thread(
+                        target=execute_tasks,
+                        kwargs=self._process_kwargs,
+                    )
+                    for _ in range(max_workers - self._max_workers)
+                ]
+                for process_instance in new_process_lst:
+                    process_instance.start()
+                self._process += new_process_lst
+            self._max_workers = max_workers
+
+    def submit(  # type: ignore
+        self, fn: Callable, *args, resource_dict: Optional[dict] = None, **kwargs
+    ) -> Future:
+        """
+        Submits a callable to be executed with the given arguments.
+
+        Schedules the callable to be executed as fn(*args, **kwargs) and returns
+        a Future instance representing the execution of the callable.
+
+        Args:
+            fn (Callable): function to submit for execution
+            args: arguments for the submitted function
+            kwargs: keyword arguments for the submitted function
+            resource_dict (dict): resource dictionary, which defines the resources used for the execution of the
+                                  function. Example resource dictionary: {
+                                      cores: 1,
+                                      threads_per_core: 1,
+                                      gpus_per_worker: 0,
+                                      oversubscribe: False,
+                                      cwd: None,
+                                      executor: None,
+                                      hostname_localhost: False,
+                                  }
+
+        Returns:
+            Future: A Future representing the given call.
+        """
+        if resource_dict is None:
+            resource_dict = {}
+        check_resource_dict_is_empty(resource_dict=resource_dict)
+        check_resource_dict(function=fn)
+        f: Future = Future()
+        if self._future_queue is not None:
+            self._future_queue.put(
+                {"fn": fn, "args": args, "kwargs": kwargs, "future": f}
+            )
+        return f
+
+    def shutdown(self, wait: bool = True, *, cancel_futures: bool = False):
+        """Clean-up the resources associated with the Executor.
+
+        It is safe to call this method several times. Otherwise, no other
+        methods can be called after this one.
+
+        Args:
+            wait: If True then shutdown will not return until all running
+                futures have finished executing and the resources used by the
+                parallel_executors have been reclaimed.
+            cancel_futures: If True then shutdown will cancel all pending
+                futures. Futures that are completed or running will not be
+                cancelled.
+        """
+        if self._future_queue is not None:
+            if cancel_futures:
+                cancel_items_in_queue(que=self._future_queue)
+            if isinstance(self._process, list):
+                for _ in range(len(self._process)):
+                    self._future_queue.put({"shutdown": True, "wait": wait})
+                if wait:
+                    for process in self._process:
+                        process.join()
+                    self._future_queue.join()
+        self._process = None
+        self._future_queue = None
+
+    def _set_process(self, process: list[Thread]):  # type: ignore
+        """
+        Set the process for the executor.
+
+        Args:
+            process (List[RaisingThread]): The process for the executor.
+        """
+        self._process = process
+        for process_instance in self._process:
+            process_instance.start()

executorlib-0.4.1/executorlib/interactive/dependency.py

@@ -0,0 +1,287 @@
+import queue
+from concurrent.futures import Future
+from threading import Thread
+from time import sleep
+from typing import Any, Callable, Optional
+
+from executorlib.base.executor import ExecutorBase
+from executorlib.standalone.interactive.arguments import (
+    check_exception_was_raised,
+    get_exception_lst,
+    get_future_objects_from_input,
+    update_futures_in_input,
+)
+from executorlib.standalone.plot import (
+    draw,
+    generate_nodes_and_edges,
+    generate_task_hash,
+)
+
+
+class DependencyExecutor(ExecutorBase):
+    """
+    ExecutorWithDependencies is a class that extends ExecutorBase and provides functionality for executing tasks with
+    dependencies.
+
+    Args:
+        refresh_rate (float, optional): The refresh rate for updating the executor queue. Defaults to 0.01.
+        plot_dependency_graph (bool, optional): Whether to generate and plot the dependency graph. Defaults to False.
+        plot_dependency_graph_filename (str): Name of the file to store the plotted graph in.
+
+    Attributes:
+        _future_hash_dict (Dict[str, Future]): A dictionary mapping task hash to future object.
+        _task_hash_dict (Dict[str, Dict]): A dictionary mapping task hash to task dictionary.
+        _generate_dependency_graph (bool): Whether to generate the dependency graph.
+        _generate_dependency_graph (str): Name of the file to store the plotted graph in.
+
+    """
+
+    def __init__(
+        self,
+        executor: ExecutorBase,
+        max_cores: Optional[int] = None,
+        refresh_rate: float = 0.01,
+        plot_dependency_graph: bool = False,
+        plot_dependency_graph_filename: Optional[str] = None,
+    ) -> None:
+        super().__init__(max_cores=max_cores)
+        self._process_kwargs = {
+            "future_queue": self._future_queue,
+            "executor_queue": executor._future_queue,
+            "executor": executor,
+            "refresh_rate": refresh_rate,
+        }
+        self._set_process(
+            Thread(
+                target=_execute_tasks_with_dependencies,
+                kwargs=self._process_kwargs,
+            )
+        )
+        self._future_hash_dict: dict = {}
+        self._task_hash_dict: dict = {}
+        self._plot_dependency_graph_filename = plot_dependency_graph_filename
+        if plot_dependency_graph_filename is None:
+            self._generate_dependency_graph = plot_dependency_graph
+        else:
+            self._generate_dependency_graph = True
+
+    @property
+    def info(self) -> Optional[dict]:
+        """
+        Get the information about the executor.
+
+        Returns:
+            Optional[dict]: Information about the executor.
+        """
+        if isinstance(self._future_queue, queue.Queue):
+            f: Future = Future()
+            self._future_queue.queue.insert(
+                0, {"internal": True, "task": "get_info", "future": f}
+            )
+            return f.result()
+        else:
+            return None
+
+    @property
+    def max_workers(self) -> Optional[int]:
+        if isinstance(self._future_queue, queue.Queue):
+            f: Future = Future()
+            self._future_queue.queue.insert(
+                0, {"internal": True, "task": "get_max_workers", "future": f}
+            )
+            return f.result()
+        else:
+            return None
+
+    @max_workers.setter
+    def max_workers(self, max_workers: int):
+        if isinstance(self._future_queue, queue.Queue):
+            f: Future = Future()
+            self._future_queue.queue.insert(
+                0,
+                {
+                    "internal": True,
+                    "task": "set_max_workers",
+                    "max_workers": max_workers,
+                    "future": f,
+                },
+            )
+            if not f.result():
+                raise NotImplementedError("The max_workers setter is not implemented.")
+
+    def submit(  # type: ignore
+        self,
+        fn: Callable[..., Any],
+        *args: Any,
+        resource_dict: Optional[dict[str, Any]] = None,
+        **kwargs: Any,
+    ) -> Future:
+        """
+        Submits a task to the executor.
+
+        Args:
+            fn (Callable): The function to be executed.
+            *args: Variable length argument list.
+            resource_dict (dict, optional): A dictionary of resources required by the task. Defaults to {}.
+            **kwargs: Arbitrary keyword arguments.
+
+        Returns:
+            Future: A future object representing the result of the task.
+
+        """
+        if resource_dict is None:
+            resource_dict = {}
+        if not self._generate_dependency_graph:
+            f = super().submit(fn, *args, resource_dict=resource_dict, **kwargs)
+        else:
+            f = Future()
+            f.set_result(None)
+            task_dict = {
+                "fn": fn,
+                "args": args,
+                "kwargs": kwargs,
+                "future": f,
+                "resource_dict": resource_dict,
+            }
+            task_hash = generate_task_hash(
+                task_dict=task_dict,
+                future_hash_inverse_dict={
+                    v: k for k, v in self._future_hash_dict.items()
+                },
+            )
+            self._future_hash_dict[task_hash] = f
+            self._task_hash_dict[task_hash] = task_dict
+        return f
+
+    def __exit__(
+        self,
+        exc_type: Any,
+        exc_val: Any,
+        exc_tb: Any,
+    ) -> None:
+        """
+        Exit method called when exiting the context manager.
+
+        Args:
+            exc_type: The type of the exception.
+            exc_val: The exception instance.
+            exc_tb: The traceback object.
+
+        """
+        super().__exit__(exc_type=exc_type, exc_val=exc_val, exc_tb=exc_tb)  # type: ignore
+        if self._generate_dependency_graph:
+            node_lst, edge_lst = generate_nodes_and_edges(
+                task_hash_dict=self._task_hash_dict,
+                future_hash_inverse_dict={
+                    v: k for k, v in self._future_hash_dict.items()
+                },
+            )
+            return draw(
+                node_lst=node_lst,
+                edge_lst=edge_lst,
+                filename=self._plot_dependency_graph_filename,
+            )
+
+
+def _execute_tasks_with_dependencies(
+    future_queue: queue.Queue,
+    executor_queue: queue.Queue,
+    executor: ExecutorBase,
+    refresh_rate: float = 0.01,
+):
+    """
+    Resolve the dependencies of multiple tasks, by analysing which task requires concurrent.future.Futures objects from
+    other tasks.
+
+    Args:
+        future_queue (Queue): Queue for receiving new tasks.
+        executor_queue (Queue): Queue for the internal executor.
+        executor (ExecutorBase): Executor to execute the tasks with after the dependencies are resolved.
+        refresh_rate (float): Set the refresh rate in seconds, how frequently the input queue is checked.
+    """
+    wait_lst = []
+    while True:
+        try:
+            task_dict = future_queue.get_nowait()
+        except queue.Empty:
+            task_dict = None
+        if (  # shutdown the executor
+            task_dict is not None and "shutdown" in task_dict and task_dict["shutdown"]
+        ):
+            executor.shutdown(wait=task_dict["wait"])
+            future_queue.task_done()
+            future_queue.join()
+            break
+        if (  # shutdown the executor
+            task_dict is not None and "internal" in task_dict and task_dict["internal"]
+        ):
+            if task_dict["task"] == "get_info":
+                task_dict["future"].set_result(executor.info)
+            elif task_dict["task"] == "get_max_workers":
+                task_dict["future"].set_result(executor.max_workers)
+            elif task_dict["task"] == "set_max_workers":
+                try:
+                    executor.max_workers = task_dict["max_workers"]
+                except NotImplementedError:
+                    task_dict["future"].set_result(False)
+                else:
+                    task_dict["future"].set_result(True)
+        elif (  # handle function submitted to the executor
+            task_dict is not None and "fn" in task_dict and "future" in task_dict
+        ):
+            future_lst, ready_flag = get_future_objects_from_input(
+                args=task_dict["args"], kwargs=task_dict["kwargs"]
+            )
+            exception_lst = get_exception_lst(future_lst=future_lst)
+            if not check_exception_was_raised(future_obj=task_dict["future"]):
+                if len(exception_lst) > 0:
+                    task_dict["future"].set_exception(exception_lst[0])
+                elif len(future_lst) == 0 or ready_flag:
+                    # No future objects are used in the input or all future objects are already done
+                    task_dict["args"], task_dict["kwargs"] = update_futures_in_input(
+                        args=task_dict["args"], kwargs=task_dict["kwargs"]
+                    )
+                    executor_queue.put(task_dict)
+                else:  # Otherwise add the function to the wait list
+                    task_dict["future_lst"] = future_lst
+                    wait_lst.append(task_dict)
+            future_queue.task_done()
+        elif len(wait_lst) > 0:
+            number_waiting = len(wait_lst)
+            # Check functions in the wait list and execute them if all future objects are now ready
+            wait_lst = _update_waiting_task(
+                wait_lst=wait_lst, executor_queue=executor_queue
+            )
+            # if no job is ready, sleep for a moment
+            if len(wait_lst) == number_waiting:
+                sleep(refresh_rate)
+        else:
+            # If there is nothing else to do, sleep for a moment
+            sleep(refresh_rate)
+
+
+def _update_waiting_task(wait_lst: list[dict], executor_queue: queue.Queue) -> list:
+    """
+    Submit the waiting tasks, which future inputs have been completed, to the executor
+
+    Args:
+        wait_lst (list): List of waiting tasks
+        executor_queue (Queue): Queue of the internal executor
+
+    Returns:
+        list: list tasks which future inputs have not been completed
+    """
+    wait_tmp_lst = []
+    for task_wait_dict in wait_lst:
+        exception_lst = get_exception_lst(future_lst=task_wait_dict["future_lst"])
+        if len(exception_lst) > 0:
+            task_wait_dict["future"].set_exception(exception_lst[0])
+        elif all(future.done() for future in task_wait_dict["future_lst"]):
+            del task_wait_dict["future_lst"]
+            task_wait_dict["args"], task_wait_dict["kwargs"] = update_futures_in_input(
+                args=task_wait_dict["args"], kwargs=task_wait_dict["kwargs"]
+            )
+            executor_queue.put(task_wait_dict)
+        else:
+            wait_tmp_lst.append(task_wait_dict)
+    return wait_tmp_lst

executorlib-0.3.0/executorlib/interactive/flux.py → executorlib-0.4.1/executorlib/interactive/fluxspawner.py

@@ -30,8 +30,11 @@ class FluxPythonSpawner(BaseSpawner):
         threads_per_core (int, optional): The number of threads per base. Defaults to 1.
         gpus_per_core (int, optional): The number of GPUs per base. Defaults to 0.
         num_nodes (int, optional): The number of compute nodes to use for executing the task. Defaults to None.
-        exclusive (bool): Whether to exclusively reserve the compute nodes, or allow sharing compute notes. Defaults to False.
+        exclusive (bool): Whether to exclusively reserve the compute nodes, or allow sharing compute notes. Defaults to
+                          False.
         openmpi_oversubscribe (bool, optional): Whether to oversubscribe. Defaults to False.
+        priority (int, optional): job urgency 0 (lowest) through 31 (highest) (default is 16). Priorities 0 through 15
+                                  are restricted to the instance owner.
         flux_executor (flux.job.FluxExecutor, optional): The FluxExecutor instance. Defaults to None.
         flux_executor_pmi_mode (str, optional): The PMI option. Defaults to None.
         flux_executor_nesting (bool, optional): Whether to use nested FluxExecutor. Defaults to False.
@@ -46,6 +49,7 @@ class FluxPythonSpawner(BaseSpawner):
         gpus_per_core: int = 0,
         num_nodes: Optional[int] = None,
         exclusive: bool = False,
+        priority: Optional[int] = None,
         openmpi_oversubscribe: bool = False,
         flux_executor: Optional[flux.job.FluxExecutor] = None,
         flux_executor_pmi_mode: Optional[str] = None,
@@ -65,6 +69,7 @@ class FluxPythonSpawner(BaseSpawner):
         self._flux_executor_pmi_mode = flux_executor_pmi_mode
         self._flux_executor_nesting = flux_executor_nesting
         self._flux_log_files = flux_log_files
+        self._priority = priority
         self._future = None
 
     def bootup(
@@ -114,7 +119,12 @@ class FluxPythonSpawner(BaseSpawner):
         elif self._flux_log_files:
             jobspec.stderr = os.path.abspath("flux.err")
             jobspec.stdout = os.path.abspath("flux.out")
-        self._future = self._flux_executor.submit(jobspec)
+        if self._priority is not None:
+            self._future = self._flux_executor.submit(
+                jobspec=jobspec, urgency=self._priority
+            )
+        else:
+            self._future = self._flux_executor.submit(jobspec=jobspec)
 
     def shutdown(self, wait: bool = True):
         """