executorlib 1.9.1__py3-none-any.whl → 1.9.2__py3-none-any.whl

executorlib/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '1.9.1'
-__version_tuple__ = version_tuple = (1, 9, 1)
+__version__ = version = '1.9.2'
+__version_tuple__ = version_tuple = (1, 9, 2)
 
 __commit_id__ = commit_id = None

executorlib/executor/flux.py

@@ -55,7 +55,7 @@ class FluxJobExecutor(BaseExecutor):
                                                       allowed to take in seconds.
                               * priority (int): the queuing system priority assigned to a given Python function to
                                                 influence the scheduling.
-                              *`slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
+                              * slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
         pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None
         flux_executor (flux.job.FluxExecutor): Flux Python interface to submit the workers to flux
         flux_executor_nesting (bool): Provide hierarchically nested Flux job scheduler inside the submitted function.
@@ -287,7 +287,7 @@ class FluxClusterExecutor(BaseExecutor):
                                                       allowed to take in seconds.
                               * priority (int): the queuing system priority assigned to a given Python function to
                                                 influence the scheduling.
-                              *`slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
+                              * slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
         pysqa_config_directory (str, optional): path to the pysqa config directory (only for pysqa based backend).
         pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None
         hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the
@@ -544,7 +544,7 @@ def create_flux_executor(
                                                       allowed to take in seconds.
                               * priority (int): the queuing system priority assigned to a given Python function to
                                                 influence the scheduling.
-                              *`slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
+                              * slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
         pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None
         flux_executor (flux.job.FluxExecutor): Flux Python interface to submit the workers to flux
         flux_executor_nesting (bool): Provide hierarchically nested Flux job scheduler inside the submitted function.

executorlib/executor/slurm.py

@@ -56,7 +56,7 @@ class SlurmClusterExecutor(BaseExecutor):
                                                       allowed to take in seconds.
                               * priority (int): the queuing system priority assigned to a given Python function to
                                                 influence the scheduling.
-                              *`slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
+                              * slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
         pysqa_config_directory (str, optional): path to the pysqa config directory (only for pysqa based backend).
         pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None
         hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the
@@ -297,7 +297,7 @@ class SlurmJobExecutor(BaseExecutor):
                                                       allowed to take in seconds.
                               * priority (int): the queuing system priority assigned to a given Python function to
                                                 influence the scheduling.
-                              *`slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
+                              * slurm_cmd_args (list): Additional command line arguments for the srun call (SLURM only)
         pmi_mode (str): PMI interface to use (OpenMPI v5 requires pmix) default is None
         hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the
                                       context of an HPC cluster this essential to be able to communicate to an

executorlib/standalone/validate.py

@@ -26,6 +26,7 @@ class ResourceDictValidation(BaseModel):
     run_time_limit: Optional[int] = None
     priority: Optional[int] = None
     slurm_cmd_args: Optional[list[str]] = None
+    submission_template: Optional[str] = None
 
     if HAS_PYDANTIC:
 

executorlib/task_scheduler/interactive/blockallocation.py

@@ -1,7 +1,7 @@
 import queue
 import random
 from concurrent.futures import Future
-from threading import Event, Thread
+from threading import Event, Lock, Thread
 from typing import Callable, Optional
 
 from executorlib.standalone.command import get_interactive_execute_command
@@ -83,6 +83,8 @@ class BlockAllocationTaskScheduler(TaskSchedulerBase):
         self_id = random.getrandbits(128)
         self._self_id = self_id
         _interrupt_bootup_dict[self._self_id] = False
+        alive_workers = [max_workers]
+        alive_workers_lock = Lock()
         bootup_events = [Event() for _ in range(self._max_workers)]
         bootup_events[0].set()
         self._set_process(
@@ -99,6 +101,8 @@ class BlockAllocationTaskScheduler(TaskSchedulerBase):
                             if worker_id + 1 < self._max_workers
                             else None
                         ),
+                        "alive_workers": alive_workers,
+                        "alive_workers_lock": alive_workers_lock,
                     },
                 )
                 for worker_id in range(self._max_workers)
@@ -227,6 +231,8 @@ def _execute_multiple_tasks(
     restart_limit: int = 0,
     bootup_event: Optional[Event] = None,
     next_bootup_event: Optional[Event] = None,
+    alive_workers: Optional[list] = None,
+    alive_workers_lock: Optional[Lock] = None,
     **kwargs,
 ) -> None:
     """
@@ -258,6 +264,9 @@ def _execute_multiple_tasks(
         bootup_event (Event): Event to wait on before submitting the job to the scheduler, ensuring workers are
                               submitted in worker_id order.
         next_bootup_event (Event): Event to signal after job submission, unblocking the next worker.
+        alive_workers (list): Single-element list [N] tracking how many worker threads are still alive. Shared across
+                              all worker threads; decremented when a worker is permanently dead.
+        alive_workers_lock (Lock): Lock protecting alive_workers from concurrent modification.
     """
     if bootup_event is not None:
         bootup_event.wait()
@@ -279,11 +288,13 @@ def _execute_multiple_tasks(
     )
     restart_counter = 0
     while True:
-        if not interface.status and restart_counter > restart_limit:
-            interface.status = True  # no more restarts
-            interface_initialization_exception = ExecutorlibSocketError(
-                "SocketInterface crashed during execution."
+        if not interface.status and restart_counter >= restart_limit:
+            _drain_dead_worker(
+                future_queue=future_queue,
+                alive_workers=alive_workers,
+                alive_workers_lock=alive_workers_lock,
             )
+            break
         elif not interface.status:
             interface.bootup()
             interface_initialization_exception = _set_init_function(
@@ -321,6 +332,47 @@ def _execute_multiple_tasks(
                 task_done(future_queue=future_queue)
 
 
+def _drain_dead_worker(
+    future_queue: queue.Queue,
+    alive_workers: Optional[list] = None,
+    alive_workers_lock: Optional[Lock] = None,
+) -> None:
+    """Handle a permanently dead worker by recycling or failing its tasks.
+
+    If healthy workers remain, tasks are recycled back into the shared queue
+    so they can be picked up. If all workers are dead, tasks are failed
+    immediately with ExecutorlibSocketError. In both cases, the worker's
+    shutdown message is consumed to prevent hangs in shutdown().
+    """
+    if alive_workers is not None and alive_workers_lock is not None:
+        with alive_workers_lock:
+            if alive_workers[0] > 0:
+                alive_workers[0] -= 1
+    while True:
+        try:
+            task_dict = future_queue.get(timeout=1)
+        except queue.Empty:
+            continue
+        if "shutdown" in task_dict and task_dict["shutdown"]:
+            task_done(future_queue=future_queue)
+            break
+        elif "fn" in task_dict and "future" in task_dict:
+            if alive_workers is not None and alive_workers_lock is not None:
+                with alive_workers_lock:
+                    has_healthy_workers = alive_workers[0] > 0
+            else:
+                has_healthy_workers = False
+            if has_healthy_workers:
+                future_queue.put(task_dict)
+                task_done(future_queue=future_queue)
+            else:
+                f = task_dict.pop("future")
+                f.set_exception(
+                    ExecutorlibSocketError("SocketInterface crashed during execution.")
+                )
+                task_done(future_queue=future_queue)
+
+
 def _set_init_function(
     interface: SocketInterface,
     init_function: Optional[Callable] = None,

executorlib/task_scheduler/interactive/spawner_pysqa.py

@@ -238,6 +238,8 @@ def create_pysqa_block_allocation_scheduler(
     cores_per_worker = executor_kwargs.get("cores", 1)
     if "cwd" in executor_kwargs and executor_kwargs["cwd"] is not None:
         executor_kwargs["cwd"] = os.path.abspath(executor_kwargs["cwd"])
+    elif cache_directory is not None:
+        executor_kwargs["cwd"] = os.path.abspath(cache_directory)
     if cache_directory is not None:
         executor_kwargs["cache_directory"] = os.path.abspath(cache_directory)
     else:

{executorlib-1.9.1.dist-info → executorlib-1.9.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: executorlib
-Version: 1.9.1
+Version: 1.9.2
 Summary: Up-scale python functions for high performance computing (HPC) with executorlib.
 Project-URL: Homepage, https://github.com/pyiron/executorlib
 Project-URL: Documentation, https://executorlib.readthedocs.io

{executorlib-1.9.1.dist-info → executorlib-1.9.2.dist-info}/RECORD

@@ -1,5 +1,5 @@
 executorlib/__init__.py,sha256=IY_DkT706LPIv2_iIe3whrQNGoDzaDFYBvXyUepoZpw,4780
-executorlib/_version.py,sha256=WXrO_kPNiaVQjOl9RfdHJJdWkCctrYwvbkQ2oSaBTfw,704
+executorlib/_version.py,sha256=ZIdFbOeqiC-Wqy70vtAf_MO4asXmiASZByj2W0lijtI,704
 executorlib/api.py,sha256=F6ibDJnVcPGxxPe-UScFJHprWHiVsETGhgAvS1T-9ZI,1365
 executorlib/backend/__init__.py,sha256=2b_iwds4VUrJe92nKtQXkoGxkJiXeEDSlbvCRN_-de8,456
 executorlib/backend/cache_parallel.py,sha256=ldb2iYzrhcXJZ0LcH75FpTLO_6F51mleU9JGAilF00k,2020
@@ -8,9 +8,9 @@ executorlib/backend/interactive_parallel.py,sha256=HimGnWOdTXInC3o7W4wMUSTTlj6Ms
 executorlib/backend/interactive_serial.py,sha256=NxlSX566rQM-pHiy2rQja2rr3lFwdnv0EvIyXk8H7co,2982
 executorlib/executor/__init__.py,sha256=oR2ABLAMYV_TNs1eG-3-MAKHFS7OKB-1KOzCqKNbtWA,310
 executorlib/executor/base.py,sha256=vqNlVCjmQAsFQkSyjZ3gGTyQeN0khu4DkZBactr2Qz0,6559
-executorlib/executor/flux.py,sha256=yRrCBSsgfjw3r9PYMMeLChfvj4FrLiVkuJCDTC74Eik,40797
+executorlib/executor/flux.py,sha256=1uJGrHK3hswKdySVD2ESqOFjL2Xg1T4Ue_8pLUC2x6E,40797
 executorlib/executor/single.py,sha256=QJMJtitTRU4Mo2TabFYWZCdnqE7iDiVNKhnyU6prMt8,36037
-executorlib/executor/slurm.py,sha256=vCLxXH-uHV7Au7A_OLA2aCk78woYCmHYWHiiSfzt5JU,38353
+executorlib/executor/slurm.py,sha256=DIBJ1elrQibi-JWg2Cj6HieFgIILBMuSUo-v136LFwU,38353
 executorlib/standalone/__init__.py,sha256=n_Q4NUFu35Fn0qlWIhmoC_FyUkmFGlWn1cNc_7EscF0,290
 executorlib/standalone/batched.py,sha256=Q2GPy4hPD_t5PflqF-m8ADjcHgeb56eJF31-XbsMMwM,990
 executorlib/standalone/command.py,sha256=b4z2ODx0lVKg-SG-EtpnBYWH1lmA9OzyKE4iQUOGDis,6555
@@ -21,7 +21,7 @@ executorlib/standalone/queue.py,sha256=KFEBFQXGvcaf5qcuCjT5kaZg7ucWOTGOtrD96tIBl
 executorlib/standalone/scheduler.py,sha256=LhCuIdaw9BjrP3GerbHUIqwsvxERdtH5JQoeyAqASIU,2325
 executorlib/standalone/select.py,sha256=gn6N0E01qTfGFm8nDNE9bilXXmy4rk1Wb0P7rAk4y54,1770
 executorlib/standalone/serialize.py,sha256=OEXQKYQ3P-pvbLHmOgF86oPgJkzR5vRWmL4gwnJwy94,4292
-executorlib/standalone/validate.py,sha256=wQS6-S7X973WnXIRcvAbjxqel03ewlAMGnb-W8GFGkc,1925
+executorlib/standalone/validate.py,sha256=mjlcLZTTFmbXNMkKGzLT82q1iokg7clWgVRlakA3cOw,1971
 executorlib/standalone/interactive/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 executorlib/standalone/interactive/arguments.py,sha256=058jZ-4GVZo-YC5sYeZBSbIOzekdaK-PSp65WmA85Dg,2944
 executorlib/standalone/interactive/backend.py,sha256=gGgcvER8w75bM_p1uplSsr6yCtBBlOS_6wR1iQNsntc,3006
@@ -36,15 +36,15 @@ executorlib/task_scheduler/file/spawner_pysqa.py,sha256=N1JIAvNXnmBqltTneyEfiYdR
 executorlib/task_scheduler/file/spawner_subprocess.py,sha256=BejOMi5zLkIVUUywv5hl0YdnXkGujGo4nXMpmmuHq2k,3089
 executorlib/task_scheduler/file/task_scheduler.py,sha256=c57K8d3Sfh4p--xvAy5LsCtTq9GOdymaUeN0h4POyS8,6019
 executorlib/task_scheduler/interactive/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-executorlib/task_scheduler/interactive/blockallocation.py,sha256=68E562It-XJx90GUuOqxaXcgp8gmkX65f4aeSEO-9Ao,15480
+executorlib/task_scheduler/interactive/blockallocation.py,sha256=HcfrYEuopD7wLUoxuoawRRNdaH4dTsPnFTb7luR7G5Y,17738
 executorlib/task_scheduler/interactive/dependency.py,sha256=fh402wEkXMhKxzFzsUGn1qQgNTB_KgPWIzevI89kLLg,13964
 executorlib/task_scheduler/interactive/dependency_plot.py,sha256=iZBZ9UMDeYOMFjD2-kYOxOdKeodj7TuyshI4HbH4XCU,11788
 executorlib/task_scheduler/interactive/onetoone.py,sha256=cTXgTXR9kKzzlx6EfGumMsEm7DUa7BD9JSeBpdPiupI,13312
 executorlib/task_scheduler/interactive/shared.py,sha256=-krntmZFOOE5U0HdFwuU8JZPqZQmPW4-cePDWuvkTgM,6305
 executorlib/task_scheduler/interactive/spawner_flux.py,sha256=WCN0t1VrXWdE5gl1P8cCxgK2NSQPvs-9g5uc-1m8P1E,7087
-executorlib/task_scheduler/interactive/spawner_pysqa.py,sha256=uKbTstXfjiVvhog2slgcXh2ZYhWXjInuvGqFw4-bVe0,10299
+executorlib/task_scheduler/interactive/spawner_pysqa.py,sha256=fMATiEpX358z6GAr7JKalgnR8Ox9EUjMA71Uvx6bRis,10403
 executorlib/task_scheduler/interactive/spawner_slurm.py,sha256=90qCLBdAohT3CimAmNC1MXz8cF17kAx4Ci3HDWEtg4k,3725
-executorlib-1.9.1.dist-info/METADATA,sha256=texMflm0Zb7Pj_SHoV9o51FnX25bnKIKcyjqesCcPhs,15826
-executorlib-1.9.1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-executorlib-1.9.1.dist-info/licenses/LICENSE,sha256=NkrhccIxPXCbbvp_upyVojt7DNTwZIco3z9wsYpSLqU,1511
-executorlib-1.9.1.dist-info/RECORD,,
+executorlib-1.9.2.dist-info/METADATA,sha256=8DCjL9RTk6WV7XL7RROijiJ62mn1BYcNYD-b8mcEP6k,15826
+executorlib-1.9.2.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+executorlib-1.9.2.dist-info/licenses/LICENSE,sha256=NkrhccIxPXCbbvp_upyVojt7DNTwZIco3z9wsYpSLqU,1511
+executorlib-1.9.2.dist-info/RECORD,,

{executorlib-1.9.1.dist-info → executorlib-1.9.2.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.28.0
+Generator: hatchling 1.29.0
 Root-Is-Purelib: true
 Tag: py3-none-any