executorlib 1.9.0__tar.gz → 1.9.2__tar.gz

{executorlib-1.9.0 → executorlib-1.9.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: executorlib
-Version: 1.9.0
+Version: 1.9.2
 Summary: Up-scale python functions for high performance computing (HPC) with executorlib.
 Project-URL: Homepage, https://github.com/pyiron/executorlib
 Project-URL: Documentation, https://executorlib.readthedocs.io

{executorlib-1.9.0 → executorlib-1.9.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [build-system]
 requires = [
-    "hatchling>=1.27.0,<=1.28.0",
+    "hatchling>=1.27.0,<=1.29.0",
     "hatch-vcs==0.5.0",
     "cloudpickle>=2.0.0,<=3.1.2",
     "pyzmq>=25.0.0,<=27.1.0",

{executorlib-1.9.0 → executorlib-1.9.2}/src/executorlib/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '1.9.0'
-__version_tuple__ = version_tuple = (1, 9, 0)
+__version__ = version = '1.9.2'
+__version_tuple__ = version_tuple = (1, 9, 2)
 
 __commit_id__ = commit_id = None

{executorlib-1.9.0 → executorlib-1.9.2}/src/executorlib/executor/flux.py

@@ -45,6 +45,7 @@ class FluxJobExecutor(BaseExecutor):
                               * cwd (str): current working directory where the parallel python task is executed
                               * cache_key (str): Rather than using the internal hashing of executorlib the user can
                                                  provide an external cache_key to identify tasks on the file system.
+                              * cache_directory (str): The directory to store cache files.
                               * num_nodes (int): number of compute nodes used for the evaluation of the Python function.
                               * exclusive (bool): boolean flag to reserve exclusive access to selected compute nodes -
                                                   do not allow other tasks to use the same compute node.
@@ -54,7 +55,7 @@ class FluxJobExecutor(BaseExecutor):
                                                       allowed to take in seconds.
                               * priority (int): the queuing system priority assigned to a given Python function to
                                                 influence the scheduling.
-                              *`slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
+                              * slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
         pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None
         flux_executor (flux.job.FluxExecutor): Flux Python interface to submit the workers to flux
         flux_executor_nesting (bool): Provide hierarchically nested Flux job scheduler inside the submitted function.
@@ -146,6 +147,7 @@ class FluxJobExecutor(BaseExecutor):
                                   * cwd (str): current working directory where the parallel python task is executed
                                   * cache_key (str): Rather than using the internal hashing of executorlib the user can
                                                       provide an external cache_key to identify tasks on the file system.
+                                  * cache_directory (str): The directory to store cache files.
                                   * num_nodes (int): number of compute nodes used for the evaluation of the Python
                                                      function.
                                   * exclusive (bool): boolean flag to reserve exclusive access to selected compute nodes
@@ -275,6 +277,7 @@ class FluxClusterExecutor(BaseExecutor):
                               * cwd (str): current working directory where the parallel python task is executed
                               * cache_key (str): Rather than using the internal hashing of executorlib the user can
                                                  provide an external cache_key to identify tasks on the file system.
+                              * cache_directory (str): The directory to store cache files.
                               * num_nodes (int): number of compute nodes used for the evaluation of the Python function.
                               * exclusive (bool): boolean flag to reserve exclusive access to selected compute nodes -
                                                   do not allow other tasks to use the same compute node.
@@ -284,7 +287,7 @@ class FluxClusterExecutor(BaseExecutor):
                                                       allowed to take in seconds.
                               * priority (int): the queuing system priority assigned to a given Python function to
                                                 influence the scheduling.
-                              *`slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
+                              * slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
         pysqa_config_directory (str, optional): path to the pysqa config directory (only for pysqa based backend).
         pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None
         hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the
@@ -370,6 +373,7 @@ class FluxClusterExecutor(BaseExecutor):
                                   * cwd (str): current working directory where the parallel python task is executed
                                   * cache_key (str): Rather than using the internal hashing of executorlib the user can
                                                       provide an external cache_key to identify tasks on the file system.
+                                  * cache_directory (str): The directory to store cache files.
                                   * num_nodes (int): number of compute nodes used for the evaluation of the Python
                                                      function.
                                   * exclusive (bool): boolean flag to reserve exclusive access to selected compute nodes
@@ -530,6 +534,7 @@ def create_flux_executor(
                               * cwd (str): current working directory where the parallel python task is executed
                               * cache_key (str): Rather than using the internal hashing of executorlib the user can
                                                  provide an external cache_key to identify tasks on the file system.
+                              * cache_directory (str): The directory to store cache files.
                               * num_nodes (int): number of compute nodes used for the evaluation of the Python function.
                               * exclusive (bool): boolean flag to reserve exclusive access to selected compute nodes -
                                                   do not allow other tasks to use the same compute node.
@@ -539,7 +544,7 @@ def create_flux_executor(
                                                       allowed to take in seconds.
                               * priority (int): the queuing system priority assigned to a given Python function to
                                                 influence the scheduling.
-                              *`slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
+                              * slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
         pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None
         flux_executor (flux.job.FluxExecutor): Flux Python interface to submit the workers to flux
         flux_executor_nesting (bool): Provide hierarchically nested Flux job scheduler inside the submitted function.

{executorlib-1.9.0 → executorlib-1.9.2}/src/executorlib/executor/single.py

@@ -44,6 +44,7 @@ class SingleNodeExecutor(BaseExecutor):
                               * cwd (str): current working directory where the parallel python task is executed
                               * cache_key (str): Rather than using the internal hashing of executorlib the user can
                                                  provide an external cache_key to identify tasks on the file system.
+                              * cache_directory (str): The directory to store cache files.
                               * num_nodes (int): number of compute nodes used for the evaluation of the Python function.
                               * exclusive (bool): boolean flag to reserve exclusive access to selected compute nodes -
                                                   do not allow other tasks to use the same compute node.
@@ -137,6 +138,7 @@ class SingleNodeExecutor(BaseExecutor):
                                   * cwd (str): current working directory where the parallel python task is executed
                                   * cache_key (str): Rather than using the internal hashing of executorlib the user can
                                                       provide an external cache_key to identify tasks on the file system.
+                                  * cache_directory (str): The directory to store cache files.
                                   * num_nodes (int): number of compute nodes used for the evaluation of the Python
                                                      function.
                                   * exclusive (bool): boolean flag to reserve exclusive access to selected compute nodes
@@ -251,6 +253,7 @@ class TestClusterExecutor(BaseExecutor):
                               * cwd (str): current working directory where the parallel python task is executed
                               * cache_key (str): Rather than using the internal hashing of executorlib the user can
                                                  provide an external cache_key to identify tasks on the file system.
+                              * cache_directory (str): The directory to store cache files.
                               * num_nodes (int): number of compute nodes used for the evaluation of the Python function.
                               * exclusive (bool): boolean flag to reserve exclusive access to selected compute nodes -
                                                   do not allow other tasks to use the same compute node.
@@ -340,6 +343,7 @@ class TestClusterExecutor(BaseExecutor):
                                   * cwd (str): current working directory where the parallel python task is executed
                                   * cache_key (str): Rather than using the internal hashing of executorlib the user can
                                                       provide an external cache_key to identify tasks on the file system.
+                                  * cache_directory (str): The directory to store cache files.
                                   * num_nodes (int): number of compute nodes used for the evaluation of the Python
                                                      function.
                                   * exclusive (bool): boolean flag to reserve exclusive access to selected compute nodes
@@ -469,6 +473,7 @@ def create_single_node_executor(
                               * cwd (str): current working directory where the parallel python task is executed
                               * cache_key (str): Rather than using the internal hashing of executorlib the user can
                                                  provide an external cache_key to identify tasks on the file system.
+                              * cache_directory (str): The directory to store cache files.
                               * num_nodes (int): number of compute nodes used for the evaluation of the Python function.
                               * exclusive (bool): boolean flag to reserve exclusive access to selected compute nodes -
                                                   do not allow other tasks to use the same compute node.

{executorlib-1.9.0 → executorlib-1.9.2}/src/executorlib/executor/slurm.py

@@ -46,6 +46,7 @@ class SlurmClusterExecutor(BaseExecutor):
                               * cwd (str): current working directory where the parallel python task is executed
                               * cache_key (str): Rather than using the internal hashing of executorlib the user can
                                                  provide an external cache_key to identify tasks on the file system.
+                              * cache_directory (str): The directory to store cache files.
                               * num_nodes (int): number of compute nodes used for the evaluation of the Python function.
                               * exclusive (bool): boolean flag to reserve exclusive access to selected compute nodes -
                                                   do not allow other tasks to use the same compute node.
@@ -55,7 +56,7 @@ class SlurmClusterExecutor(BaseExecutor):
                                                       allowed to take in seconds.
                               * priority (int): the queuing system priority assigned to a given Python function to
                                                 influence the scheduling.
-                              *`slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
+                              * slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
         pysqa_config_directory (str, optional): path to the pysqa config directory (only for pysqa based backend).
         pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None
         hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the
@@ -141,6 +142,7 @@ class SlurmClusterExecutor(BaseExecutor):
                                   * cwd (str): current working directory where the parallel python task is executed
                                   * cache_key (str): Rather than using the internal hashing of executorlib the user can
                                                       provide an external cache_key to identify tasks on the file system.
+                                  * cache_directory (str): The directory to store cache files.
                                   * num_nodes (int): number of compute nodes used for the evaluation of the Python
                                                      function.
                                   * exclusive (bool): boolean flag to reserve exclusive access to selected compute nodes
@@ -285,6 +287,7 @@ class SlurmJobExecutor(BaseExecutor):
                               * cwd (str): current working directory where the parallel python task is executed
                               * cache_key (str): Rather than using the internal hashing of executorlib the user can
                                                  provide an external cache_key to identify tasks on the file system.
+                              * cache_directory (str): The directory to store cache files.
                               * num_nodes (int): number of compute nodes used for the evaluation of the Python function.
                               * exclusive (bool): boolean flag to reserve exclusive access to selected compute nodes -
                                                   do not allow other tasks to use the same compute node.
@@ -294,7 +297,7 @@ class SlurmJobExecutor(BaseExecutor):
                                                       allowed to take in seconds.
                               * priority (int): the queuing system priority assigned to a given Python function to
                                                 influence the scheduling.
-                              *`slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
+                              * slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
         pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None
         hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the
                                       context of an HPC cluster this essential to be able to communicate to an
@@ -380,6 +383,7 @@ class SlurmJobExecutor(BaseExecutor):
                                   * cwd (str): current working directory where the parallel python task is executed
                                   * cache_key (str): Rather than using the internal hashing of executorlib the user can
                                                       provide an external cache_key to identify tasks on the file system.
+                                  * cache_directory (str): The directory to store cache files.
                                   * num_nodes (int): number of compute nodes used for the evaluation of the Python
                                                      function.
                                   * exclusive (bool): boolean flag to reserve exclusive access to selected compute nodes
@@ -508,6 +512,7 @@ def create_slurm_executor(
                               * cwd (str): current working directory where the parallel python task is executed
                               * cache_key (str): Rather than using the internal hashing of executorlib the user can
                                                  provide an external cache_key to identify tasks on the file system.
+                              * cache_directory (str): The directory to store cache files.
                               * num_nodes (int): number of compute nodes used for the evaluation of the Python function.
                               * exclusive (bool): boolean flag to reserve exclusive access to selected compute nodes -
                                                   do not allow other tasks to use the same compute node.

{executorlib-1.9.0 → executorlib-1.9.2}/src/executorlib/standalone/validate.py

@@ -19,12 +19,14 @@ class ResourceDictValidation(BaseModel):
     gpus_per_core: Optional[int] = None
     cwd: Optional[str] = None
     cache_key: Optional[str] = None
+    cache_directory: Optional[str] = None
     num_nodes: Optional[int] = None
     exclusive: Optional[bool] = None
     error_log_file: Optional[str] = None
     run_time_limit: Optional[int] = None
     priority: Optional[int] = None
     slurm_cmd_args: Optional[list[str]] = None
+    submission_template: Optional[str] = None
 
     if HAS_PYDANTIC:
 

{executorlib-1.9.0 → executorlib-1.9.2}/src/executorlib/task_scheduler/interactive/blockallocation.py

@@ -1,7 +1,7 @@
 import queue
 import random
 from concurrent.futures import Future
-from threading import Event, Thread
+from threading import Event, Lock, Thread
 from typing import Callable, Optional
 
 from executorlib.standalone.command import get_interactive_execute_command
@@ -83,6 +83,8 @@ class BlockAllocationTaskScheduler(TaskSchedulerBase):
         self_id = random.getrandbits(128)
         self._self_id = self_id
         _interrupt_bootup_dict[self._self_id] = False
+        alive_workers = [max_workers]
+        alive_workers_lock = Lock()
         bootup_events = [Event() for _ in range(self._max_workers)]
         bootup_events[0].set()
         self._set_process(
@@ -99,6 +101,8 @@ class BlockAllocationTaskScheduler(TaskSchedulerBase):
                             if worker_id + 1 < self._max_workers
                             else None
                         ),
+                        "alive_workers": alive_workers,
+                        "alive_workers_lock": alive_workers_lock,
                     },
                 )
                 for worker_id in range(self._max_workers)
@@ -227,6 +231,8 @@ def _execute_multiple_tasks(
     restart_limit: int = 0,
     bootup_event: Optional[Event] = None,
     next_bootup_event: Optional[Event] = None,
+    alive_workers: Optional[list] = None,
+    alive_workers_lock: Optional[Lock] = None,
     **kwargs,
 ) -> None:
     """
@@ -258,6 +264,9 @@ def _execute_multiple_tasks(
         bootup_event (Event): Event to wait on before submitting the job to the scheduler, ensuring workers are
                               submitted in worker_id order.
         next_bootup_event (Event): Event to signal after job submission, unblocking the next worker.
+        alive_workers (list): Single-element list [N] tracking how many worker threads are still alive. Shared across
+                              all worker threads; decremented when a worker is permanently dead.
+        alive_workers_lock (Lock): Lock protecting alive_workers from concurrent modification.
     """
     if bootup_event is not None:
         bootup_event.wait()
@@ -279,11 +288,13 @@ def _execute_multiple_tasks(
     )
     restart_counter = 0
     while True:
-        if not interface.status and restart_counter > restart_limit:
-            interface.status = True  # no more restarts
-            interface_initialization_exception = ExecutorlibSocketError(
-                "SocketInterface crashed during execution."
+        if not interface.status and restart_counter >= restart_limit:
+            _drain_dead_worker(
+                future_queue=future_queue,
+                alive_workers=alive_workers,
+                alive_workers_lock=alive_workers_lock,
             )
+            break
         elif not interface.status:
             interface.bootup()
             interface_initialization_exception = _set_init_function(
@@ -321,6 +332,47 @@ def _execute_multiple_tasks(
                 task_done(future_queue=future_queue)
 
 
+def _drain_dead_worker(
+    future_queue: queue.Queue,
+    alive_workers: Optional[list] = None,
+    alive_workers_lock: Optional[Lock] = None,
+) -> None:
+    """Handle a permanently dead worker by recycling or failing its tasks.
+
+    If healthy workers remain, tasks are recycled back into the shared queue
+    so they can be picked up. If all workers are dead, tasks are failed
+    immediately with ExecutorlibSocketError. In both cases, the worker's
+    shutdown message is consumed to prevent hangs in shutdown().
+    """
+    if alive_workers is not None and alive_workers_lock is not None:
+        with alive_workers_lock:
+            if alive_workers[0] > 0:
+                alive_workers[0] -= 1
+    while True:
+        try:
+            task_dict = future_queue.get(timeout=1)
+        except queue.Empty:
+            continue
+        if "shutdown" in task_dict and task_dict["shutdown"]:
+            task_done(future_queue=future_queue)
+            break
+        elif "fn" in task_dict and "future" in task_dict:
+            if alive_workers is not None and alive_workers_lock is not None:
+                with alive_workers_lock:
+                    has_healthy_workers = alive_workers[0] > 0
+            else:
+                has_healthy_workers = False
+            if has_healthy_workers:
+                future_queue.put(task_dict)
+                task_done(future_queue=future_queue)
+            else:
+                f = task_dict.pop("future")
+                f.set_exception(
+                    ExecutorlibSocketError("SocketInterface crashed during execution.")
+                )
+                task_done(future_queue=future_queue)
+
+
 def _set_init_function(
     interface: SocketInterface,
     init_function: Optional[Callable] = None,

{executorlib-1.9.0 → executorlib-1.9.2}/src/executorlib/task_scheduler/interactive/spawner_pysqa.py

@@ -238,6 +238,8 @@ def create_pysqa_block_allocation_scheduler(
     cores_per_worker = executor_kwargs.get("cores", 1)
     if "cwd" in executor_kwargs and executor_kwargs["cwd"] is not None:
         executor_kwargs["cwd"] = os.path.abspath(executor_kwargs["cwd"])
+    elif cache_directory is not None:
+        executor_kwargs["cwd"] = os.path.abspath(cache_directory)
     if cache_directory is not None:
         executor_kwargs["cache_directory"] = os.path.abspath(cache_directory)
     else: