executorlib 1.10.3__tar.gz → 1.10.4__tar.gz

{executorlib-1.10.3 → executorlib-1.10.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: executorlib
-Version: 1.10.3
+Version: 1.10.4
 Summary: Up-scale python functions for high performance computing (HPC) with executorlib.
 Project-URL: Homepage, https://github.com/pyiron/executorlib
 Project-URL: Documentation, https://executorlib.readthedocs.io
@@ -55,25 +55,25 @@ Requires-Dist: h5py<=3.16.0,>=3.6.0; extra == 'all'
 Requires-Dist: ipython<=9.9.0,>=7.33.0; extra == 'all'
 Requires-Dist: mpi4py<=4.1.2,>=3.1.4; extra == 'all'
 Requires-Dist: networkx<=3.6.1,>=2.8.8; extra == 'all'
-Requires-Dist: pydantic<=2.13.5,>=2.5.3; extra == 'all'
-Requires-Dist: pygraphviz<=2.0.1,>=1.10; extra == 'all'
-Requires-Dist: pysqa==1.0.0; extra == 'all'
+Requires-Dist: pydantic<=2.14.1,>=2.5.3; extra == 'all'
+Requires-Dist: pygraphviz<=2.0.3,>=1.10; extra == 'all'
+Requires-Dist: pysqa==1.0.1; extra == 'all'
 Provides-Extra: cache
 Requires-Dist: h5py<=3.16.0,>=3.6.0; extra == 'cache'
 Provides-Extra: cluster
 Requires-Dist: h5py<=3.16.0,>=3.6.0; extra == 'cluster'
-Requires-Dist: pysqa==1.0.0; extra == 'cluster'
+Requires-Dist: pysqa==1.0.1; extra == 'cluster'
 Provides-Extra: graph
 Requires-Dist: networkx<=3.6.1,>=2.8.8; extra == 'graph'
-Requires-Dist: pygraphviz<=2.0.1,>=1.10; extra == 'graph'
+Requires-Dist: pygraphviz<=2.0.3,>=1.10; extra == 'graph'
 Provides-Extra: graphnotebook
 Requires-Dist: ipython<=9.9.0,>=7.33.0; extra == 'graphnotebook'
 Requires-Dist: networkx<=3.6.1,>=2.8.8; extra == 'graphnotebook'
-Requires-Dist: pygraphviz<=2.0.1,>=1.10; extra == 'graphnotebook'
+Requires-Dist: pygraphviz<=2.0.3,>=1.10; extra == 'graphnotebook'
 Provides-Extra: mpi
 Requires-Dist: mpi4py<=4.1.2,>=3.1.4; extra == 'mpi'
 Provides-Extra: validation
-Requires-Dist: pydantic<=2.13.5,>=2.5.3; extra == 'validation'
+Requires-Dist: pydantic<=2.14.1,>=2.5.3; extra == 'validation'
 Description-Content-Type: text/markdown
 
 # executorlib

{executorlib-1.10.3 → executorlib-1.10.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [build-system]
 requires = [
-    "hatchling>=1.27.0,<=1.32.0",
+    "hatchling>=1.27.0,<=1.32.4",
     "hatch-vcs==0.5.0",
     "cloudpickle>=2.0.0,<=3.1.2",
     "pyzmq>=25.0.0,<=27.2.0",
@@ -43,28 +43,28 @@ Repository = "https://github.com/pyiron/executorlib"
 [project.optional-dependencies]
 cache = ["h5py>=3.6.0,<=3.16.0"]
 graph = [
-    "pygraphviz>=1.10,<=2.0.1",
+    "pygraphviz>=1.10,<=2.0.3",
     "networkx>=2.8.8,<=3.6.1",
 ]
 graphnotebook = [
-    "pygraphviz>=1.10,<=2.0.1",
+    "pygraphviz>=1.10,<=2.0.3",
     "networkx>=2.8.8,<=3.6.1",
     "ipython>=7.33.0,<=9.9.0",
 ]
 mpi = ["mpi4py>=3.1.4,<=4.1.2"]
 cluster = [
-    "pysqa==1.0.0",
+    "pysqa==1.0.1",
     "h5py>=3.6.0,<=3.16.0",
 ]
 validation = [
-    "pydantic>=2.5.3,<=2.13.5",
+    "pydantic>=2.5.3,<=2.14.1",
 ]
 all = [
     "mpi4py>=3.1.4,<=4.1.2",
-    "pysqa==1.0.0",
+    "pysqa==1.0.1",
     "h5py>=3.6.0,<=3.16.0",
-    "pydantic>=2.5.3,<=2.13.5",
-    "pygraphviz>=1.10,<=2.0.1",
+    "pydantic>=2.5.3,<=2.14.1",
+    "pygraphviz>=1.10,<=2.0.3",
     "networkx>=2.8.8,<=3.6.1",
     "ipython>=7.33.0,<=9.9.0",
 ]

{executorlib-1.10.3 → executorlib-1.10.4}/src/executorlib/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.10.3'
-__version_tuple__ = version_tuple = (1, 10, 3)
+__version__ = version = '1.10.4'
+__version_tuple__ = version_tuple = (1, 10, 4)
 
 __commit_id__ = commit_id = None

{executorlib-1.10.3 → executorlib-1.10.4}/src/executorlib/standalone/hdf.py

@@ -31,18 +31,22 @@ def dump(file_name: Optional[str], data_dict: dict) -> None:
     """
     if file_name is not None:
         file_name_abs = os.path.abspath(file_name)
+        # Serialize all values before opening the file, so a serialization error does
+        # not leave an incomplete HDF5 file behind.
+        binary_dict = {
+            group_dict[data_key]: cloudpickle.dumps(data_value)
+            for data_key, data_value in data_dict.items()
+            if data_key in group_dict
+        }
         os.makedirs(os.path.dirname(file_name_abs), exist_ok=True)
         with h5py.File(file_name_abs, "a") as fname:
-            for data_key, data_value in data_dict.items():
-                if data_key in group_dict:
-                    with contextlib.suppress(ValueError):
-                        fname.create_dataset(
-                            name="/" + group_dict[data_key],
-                            data=np.frombuffer(
-                                cloudpickle.dumps(data_value), dtype=np.uint8
-                            ),
-                            compression="gzip",
-                        )
+            for group_name, binary in binary_dict.items():
+                with contextlib.suppress(ValueError):
+                    fname.create_dataset(
+                        name="/" + group_name,
+                        data=np.frombuffer(binary, dtype=np.uint8),
+                        compression="gzip",
+                    )
 
 
 def load(file_name: str) -> dict:

executorlib-1.10.4/src/executorlib/standalone/queue.py

@@ -0,0 +1,36 @@
+import queue
+from typing import Any
+
+
+def put_front(que: queue.Queue, item: Any):
+    """
+    Insert an item at the front of the queue, ahead of any items already waiting, and wake a consumer blocked in
+    get(). Equivalent to queue.Queue.put() except the item is placed at the front rather than the back; mutating
+    que.queue directly instead would silently skip the notification, leaving a waiting consumer asleep forever.
+
+    Args:
+        que (queue.Queue): Queue with task objects which should be executed
+        item (Any): item to place at the front of the queue
+    """
+    with que.not_full:
+        que.queue.insert(0, item)
+        que.unfinished_tasks += 1
+        que.not_empty.notify()
+
+
+def cancel_items_in_queue(que: queue.Queue):
+    """
+    Cancel items which are still waiting in the queue. If the executor is busy tasks remain in the queue, so the future
+    objects have to be cancelled when the executor shuts down.
+
+    Args:
+        que (queue.Queue): Queue with task objects which should be executed
+    """
+    while True:
+        try:
+            item = que.get_nowait()
+            if isinstance(item, dict) and "future" in item:
+                item["future"].cancel()
+                que.task_done()
+        except queue.Empty:
+            break

{executorlib-1.10.3 → executorlib-1.10.4}/src/executorlib/standalone/select.py

@@ -21,9 +21,10 @@ class FutureSelector(Future):
             future (Future): The underlying future whose result is a collection.
             selector (int | str): Index or key used to extract an element from the result.
         """
+        # Future.__init__() is not called, as its attributes would be forwarded to the
+        # wrapped future by __setattr__() and reset the state of an already completed future.
         self._future = future
         self._selector = selector
-        super().__init__()
 
     def __getattr__(self, attr: str) -> Any:
         """Delegate attribute access to the wrapped future."""

{executorlib-1.10.3 → executorlib-1.10.4}/src/executorlib/task_scheduler/file/shared.py

@@ -1,7 +1,7 @@
 import contextlib
 import os
 import queue
-from concurrent.futures import Future
+from concurrent.futures import CancelledError, Future
 from time import sleep
 from typing import Any, Callable, Optional
 
@@ -118,83 +118,93 @@ def execute_tasks_h5(
             future_queue.join()
             break
         elif task_dict is not None:
-            task_args, task_kwargs, future_wait_key_lst = _convert_args_and_kwargs(
-                task_dict=task_dict,
-                memory_dict=memory_dict,
-                file_name_dict=file_name_dict,
-            )
-            task_resource_dict, cache_key, cache_directory, error_log_file = (
-                _get_task_input(
-                    task_resource_dict=task_dict["resource_dict"].copy(),
-                    executor_kwargs=executor_kwargs,
+            try:
+                task_args, task_kwargs, future_wait_key_lst = _convert_args_and_kwargs(
+                    task_dict=task_dict,
+                    memory_dict=memory_dict,
+                    file_name_dict=file_name_dict,
+                    duplicate_dict=duplicate_dict,
                 )
-            )
-            task_key, data_dict, serialize_exception = serialize_funct(
-                fn=task_dict["fn"],
-                fn_args=task_args,
-                fn_kwargs=task_kwargs,
-                resource_dict=task_resource_dict,
-                cache_key=cache_key,
-            )
-            data_dict["error_log_file"] = error_log_file
-            if serialize_exception is not None:
-                task_dict["future"].set_exception(serialize_exception)
-            elif task_key not in memory_dict:
-                if os.path.join(
-                    cache_directory, task_key + "_o.h5"
-                ) not in get_cache_files(cache_directory=cache_directory):
-                    file_name = os.path.join(cache_directory, task_key + "_i.h5")
-                    if not disable_dependencies:
-                        task_dependent_lst = [
-                            process_dict[k]
-                            for k in future_wait_key_lst
-                            if k in process_dict
-                        ]
-                    else:
-                        if len(future_wait_key_lst) > 0:
-                            task_dict["future"].set_exception(
-                                ValueError(
-                                    "Future objects are not supported as input if disable_dependencies=True."
+                task_resource_dict, cache_key, cache_directory, error_log_file = (
+                    _get_task_input(
+                        task_resource_dict=task_dict["resource_dict"].copy(),
+                        executor_kwargs=executor_kwargs,
+                    )
+                )
+                task_key, data_dict, serialize_exception = serialize_funct(
+                    fn=task_dict["fn"],
+                    fn_args=task_args,
+                    fn_kwargs=task_kwargs,
+                    resource_dict=task_resource_dict,
+                    cache_key=cache_key,
+                )
+                data_dict["error_log_file"] = error_log_file
+                if serialize_exception is not None:
+                    task_dict["future"].set_exception(serialize_exception)
+                elif task_key not in memory_dict:
+                    if os.path.join(
+                        cache_directory, task_key + "_o.h5"
+                    ) not in get_cache_files(cache_directory=cache_directory):
+                        file_name = os.path.join(cache_directory, task_key + "_i.h5")
+                        if not disable_dependencies:
+                            task_dependent_lst = [
+                                process_dict[k]
+                                for k in future_wait_key_lst
+                                if k in process_dict
+                            ]
+                        else:
+                            if len(future_wait_key_lst) > 0:
+                                task_dict["future"].set_exception(
+                                    ValueError(
+                                        "Future objects are not supported as input if disable_dependencies=True."
+                                    )
                                 )
-                            )
-                        task_dependent_lst = []
-                    try:
-                        process_dict[task_key] = execute_function(
-                            command=get_cache_execute_command(
+                                continue
+                            task_dependent_lst = []
+                        try:
+                            process_dict[task_key] = execute_function(
+                                command=get_cache_execute_command(
+                                    file_name=file_name,
+                                    cores=task_resource_dict["cores"],
+                                    backend=backend,
+                                    exclusive=task_resource_dict.get(
+                                        "exclusive", False
+                                    ),
+                                    openmpi_oversubscribe=task_resource_dict.get(
+                                        "openmpi_oversubscribe", False
+                                    ),
+                                    pmi_mode=pmi_mode,
+                                ),
                                 file_name=file_name,
-                                cores=task_resource_dict["cores"],
+                                data_dict=data_dict,
+                                task_dependent_lst=task_dependent_lst,
+                                resource_dict=task_resource_dict,
+                                config_directory=pysqa_config_directory,
                                 backend=backend,
-                                exclusive=task_resource_dict.get("exclusive", False),
-                                openmpi_oversubscribe=task_resource_dict.get(
-                                    "openmpi_oversubscribe", False
-                                ),
-                                pmi_mode=pmi_mode,
-                            ),
-                            file_name=file_name,
-                            data_dict=data_dict,
-                            task_dependent_lst=task_dependent_lst,
-                            resource_dict=task_resource_dict,
-                            config_directory=pysqa_config_directory,
-                            backend=backend,
-                            cache_directory=cache_directory,
-                        )
-                    except Exception as e:
-                        task_dict["future"].set_exception(e)
-                        memory_dict[task_key] = task_dict["future"]
-                        future_queue.task_done()
-                        continue
-                file_name = os.path.join(cache_directory, task_key + "_o.h5")
-                file_name_dict[task_key] = file_name
-                queue_id = get_queue_id(file_name=file_name)
-                if queue_id is not None:
-                    process_dict[task_key] = queue_id
-                memory_dict[task_key] = task_dict["future"]
-                cache_dir_dict[task_key] = cache_directory
-            elif memory_dict[task_key] != task_dict["future"]:
-                if task_key not in duplicate_dict:
-                    duplicate_dict[task_key] = []
-                duplicate_dict[task_key].append(task_dict["future"])
-            future_queue.task_done()
+                                cache_directory=cache_directory,
+                            )
+                        except Exception as e:
+                            task_dict["future"].set_exception(e)
+                            memory_dict[task_key] = task_dict["future"]
+                            continue
+                    file_name = os.path.join(cache_directory, task_key + "_o.h5")
+                    file_name_dict[task_key] = file_name
+                    queue_id = get_queue_id(file_name=file_name)
+                    if queue_id is not None:
+                        process_dict[task_key] = queue_id
+                    memory_dict[task_key] = task_dict["future"]
+                    cache_dir_dict[task_key] = cache_directory
+                elif memory_dict[task_key] != task_dict["future"]:
+                    if task_key not in duplicate_dict:
+                        duplicate_dict[task_key] = []
+                    duplicate_dict[task_key].append(task_dict["future"])
+            except Exception as e:
+                # Fail only the current task, never the scheduler thread, e.g. when
+                # one of the input futures of this task failed before submission.
+                if not task_dict["future"].done():
+                    task_dict["future"].set_exception(e)
+            finally:
+                future_queue.task_done()
         else:
             memory_dict = _refresh_memory_dict(
                 memory_dict=memory_dict,
@@ -308,7 +318,10 @@ def _update_future(
 
 
 def _convert_args_and_kwargs(
-    task_dict: dict, memory_dict: dict, file_name_dict: dict
+    task_dict: dict,
+    memory_dict: dict,
+    file_name_dict: dict,
+    duplicate_dict: Optional[dict] = None,
 ) -> tuple[list, dict, list]:
     """
     Convert the arguments and keyword arguments in a task dictionary to the appropriate types.
@@ -317,59 +330,84 @@ def _convert_args_and_kwargs(
         task_dict (dict): The task dictionary containing the arguments and keyword arguments.
         memory_dict (dict): The dictionary mapping future objects to their associated task keys.
         file_name_dict (dict): The dictionary mapping task keys to their corresponding file names.
+        duplicate_dict (dict, optional): The dictionary mapping task keys to their associated duplicate future objects.
 
     Returns:
         Tuple[list, dict, list]: A tuple containing the converted arguments, converted keyword arguments, and a list of future wait keys.
 
+    Raises:
+        Exception: The exception of an input future which already failed, so the dependent task fails as well.
+
     """
-    task_args = []
-    task_kwargs = {}
-    future_wait_key_lst = []
-    for arg in task_dict["args"]:
-        selector = None
-        if isinstance(arg, Future):
-            if hasattr(arg, "_future") and hasattr(arg, "_selector"):
-                selector = arg._selector
-                future = arg._future
-            else:
-                future = arg
-            match_found = False
-            for k, v in memory_dict.items():
-                if future == v:
-                    task_args.append(
-                        FutureItem(file_name=file_name_dict[k], selector=selector)
-                    )
-                    future_wait_key_lst.append(k)
-                    match_found = True
-                    break
-            if not match_found:
-                task_args.append(future.result())
-        else:
-            task_args.append(arg)
-    for key, arg in task_dict["kwargs"].items():
-        selector = None
-        if isinstance(arg, Future):
-            if hasattr(arg, "_future") and hasattr(arg, "_selector"):
-                selector = arg._selector
-                future = arg._future
-            else:
-                future = arg
-            match_found = False
-            for k, v in memory_dict.items():
-                if future == v:
-                    task_kwargs[key] = FutureItem(
-                        file_name=file_name_dict[k], selector=selector
-                    )
-                    future_wait_key_lst.append(k)
-                    match_found = True
-                    break
-            if not match_found:
-                task_kwargs[key] = future.result()
-        else:
-            task_kwargs[key] = arg
+    if duplicate_dict is None:
+        duplicate_dict = {}
+    future_wait_key_lst: list = []
+    task_args = [
+        _convert_future(
+            arg=arg,
+            memory_dict=memory_dict,
+            file_name_dict=file_name_dict,
+            duplicate_dict=duplicate_dict,
+            future_wait_key_lst=future_wait_key_lst,
+        )
+        for arg in task_dict["args"]
+    ]
+    task_kwargs = {
+        key: _convert_future(
+            arg=arg,
+            memory_dict=memory_dict,
+            file_name_dict=file_name_dict,
+            duplicate_dict=duplicate_dict,
+            future_wait_key_lst=future_wait_key_lst,
+        )
+        for key, arg in task_dict["kwargs"].items()
+    }
     return task_args, task_kwargs, future_wait_key_lst
 
 
+def _convert_future(
+    arg: Any,
+    memory_dict: dict,
+    file_name_dict: dict,
+    duplicate_dict: dict,
+    future_wait_key_lst: list,
+) -> Any:
+    """
+    Convert a single argument, replacing future objects by FutureItem objects or their results.
+
+    Args:
+        arg (Any): The argument to convert.
+        memory_dict (dict): The dictionary mapping task keys to their associated future objects.
+        file_name_dict (dict): The dictionary mapping task keys to their corresponding file names.
+        duplicate_dict (dict): The dictionary mapping task keys to their associated duplicate future objects.
+        future_wait_key_lst (list): List of task keys the current task depends on, extended in place.
+
+    Returns:
+        Any: The converted argument.
+
+    Raises:
+        Exception: The exception of the input future, if it already failed.
+
+    """
+    if not isinstance(arg, Future):
+        return arg
+    if hasattr(arg, "_future") and hasattr(arg, "_selector"):
+        selector = arg._selector
+        future = arg._future
+    else:
+        selector = None
+        future = arg
+    if future.done():
+        exception = CancelledError() if future.cancelled() else future.exception()
+        if exception is not None:
+            raise exception
+    for k, v in memory_dict.items():
+        if k in file_name_dict and (future == v or future in duplicate_dict.get(k, [])):
+            future_wait_key_lst.append(k)
+            return FutureItem(file_name=file_name_dict[k], selector=selector)
+    return arg.result()
+
+
 def _refresh_memory_dict(
     memory_dict: dict,
     cache_dir_dict: dict,

{executorlib-1.10.3 → executorlib-1.10.4}/src/executorlib/task_scheduler/interactive/blockallocation.py

@@ -15,7 +15,7 @@ from executorlib.standalone.interactive.communication import (
     interface_bootup,
 )
 from executorlib.standalone.interactive.spawner import BaseSpawner, MpiExecSpawner
-from executorlib.standalone.queue import cancel_items_in_queue
+from executorlib.standalone.queue import cancel_items_in_queue, put_front
 from executorlib.task_scheduler.base import TaskSchedulerBase, validate_resource_dict
 from executorlib.task_scheduler.interactive.shared import (
     execute_task_dict,
@@ -122,7 +122,7 @@ class BlockAllocationTaskScheduler(TaskSchedulerBase):
         ):
             if self._max_workers > max_workers:
                 for _ in range(self._max_workers - max_workers):
-                    self._future_queue.queue.insert(0, {"shutdown": True, "wait": True})
+                    put_front(self._future_queue, {"shutdown": True, "wait": True})
                 while len(self._process) > max_workers:
                     self._process = [
                         process for process in self._process if process.is_alive()

executorlib-1.10.3/src/executorlib/standalone/queue.py

@@ -1,19 +0,0 @@
-import queue
-
-
-def cancel_items_in_queue(que: queue.Queue):
-    """
-    Cancel items which are still waiting in the queue. If the executor is busy tasks remain in the queue, so the future
-    objects have to be cancelled when the executor shuts down.
-
-    Args:
-        que (queue.Queue): Queue with task objects which should be executed
-    """
-    while True:
-        try:
-            item = que.get_nowait()
-            if isinstance(item, dict) and "future" in item:
-                item["future"].cancel()
-                que.task_done()
-        except queue.Empty:
-            break