executorlib 1.10.1__py3-none-any.whl → 1.10.2__py3-none-any.whl

executorlib/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.10.1'
-__version_tuple__ = version_tuple = (1, 10, 1)
+__version__ = version = '1.10.2'
+__version_tuple__ = version_tuple = (1, 10, 2)
 
 __commit_id__ = commit_id = None

executorlib/executor/single.py

@@ -393,7 +393,7 @@ class TestClusterExecutor(BaseExecutor):
         )
         if not plot_dependency_graph:
             from executorlib.task_scheduler.file.spawner_subprocess import (
-                execute_in_subprocess,
+                subprocess_execute,
             )
             from executorlib.task_scheduler.file.task_scheduler import (
                 create_file_executor,
@@ -415,7 +415,7 @@ class TestClusterExecutor(BaseExecutor):
                     block_allocation=block_allocation,
                     init_function=init_function,
                     disable_dependencies=disable_dependencies,
-                    execute_function=execute_in_subprocess,
+                    execute_function=subprocess_execute,
                     wait=wait,
                     refresh_rate=refresh_rate,
                     validator=validate_resource_dict_with_optional_keys,

executorlib/standalone/command_pysqa.py

@@ -0,0 +1,148 @@
+import contextlib
+import os
+import subprocess
+from time import monotonic
+from typing import Optional, Union
+
+from pysqa import QueueAdapter
+
+# Minimum time between two queries of the queuing system for the status of a task whose output
+# file has not appeared yet. Detecting a dead job (timeout, OOM, node failure, scancel, ...) relies
+# on this status query, but it must not be issued on every poll of the (much faster) refresh_rate
+# loop, as that would flood the queuing system commands (e.g. squeue/sacct) with requests.
+_JOB_STATUS_CHECK_INTERVAL = 30.0
+
+
+def pysqa_job_output_validation(
+    task_key: str,
+    file_name: str,
+    queue_id: int,
+    status_check_dict: Optional[dict],
+    pysqa_config_directory: Optional[str] = None,
+    backend: Optional[str] = None,
+    job_status_check_interval: float = _JOB_STATUS_CHECK_INTERVAL,
+) -> bool:
+    """
+    Check whether the queuing system job backing a task has died without ever writing its output
+    file. Only applies to queuing system backends (pysqa) and is throttled to at most once every
+    ``job_status_check_interval`` seconds per task, to avoid flooding the queuing system with
+    status queries on every poll of the (much faster) refresh_rate loop.
+
+    A dead job is recognized in two ways, since queuing systems differ in whether they drop
+    terminated jobs from their listing: slurm's squeue removes a job as soon as it is gone
+    (status None), while flux's "flux jobs -a" keeps listing inactive jobs and instead reports
+    pysqa's terminal-failure status "error" (the same status pysqa_terminate already treats as
+    not alive).
+
+    Args:
+        task_key (str): The key of the task.
+        file_name (str): Path of the expected output HDF5 file.
+        queue_id (int, optional): The queuing system ID of the task.
+        pysqa_config_directory (str, optional): path to the pysqa config directory.
+        backend (str, optional): name of the backend used to spawn tasks ["slurm", "flux"].
+        status_check_dict (dict): Dictionary tracking when each task's job status was last queried.
+        job_status_check_interval (float): Minimum time interval between job status checks for the same task.
+
+    Returns:
+        bool: True if the job is no longer known to the queuing system, or is reported as having
+            errored out, and still has no output.
+    """
+    now = monotonic()
+    last_checked = (
+        status_check_dict.get(task_key, 0.0) if status_check_dict is not None else 0.0
+    )
+    if now - last_checked < job_status_check_interval:
+        return False
+    if status_check_dict is not None:
+        status_check_dict[task_key] = now
+    status = pysqa_get_status_of_job(
+        queue_id=queue_id,
+        config_directory=pysqa_config_directory,
+        backend=backend,
+    )
+    return (status is None or status == "error") and not os.path.exists(file_name)
+
+
+def pysqa_terminate(
+    queue_id: int,
+    config_directory: Optional[str] = None,
+    backend: Optional[str] = None,
+):
+    """
+    Delete job from queuing system
+
+    Args:
+        queue_id (int): Queuing system ID of the job to delete.
+        config_directory (str, optional): path to the config directory.
+        backend (str, optional): name of the backend used to spawn tasks ["slurm", "flux"].
+    """
+    qa = QueueAdapter(
+        directory=config_directory,
+        queue_type=backend,
+        execute_command=pysqa_execute_command,
+    )
+    status = qa.get_status_of_job(process_id=queue_id)
+    if status is not None and status not in ["finished", "error"]:
+        with contextlib.suppress(subprocess.CalledProcessError):
+            qa.delete_job(process_id=queue_id)
+
+
+def pysqa_get_status_of_job(
+    queue_id: int,
+    config_directory: Optional[str] = None,
+    backend: Optional[str] = None,
+) -> Optional[str]:
+    """
+    Query the status of a job from the queuing system.
+
+    Args:
+        queue_id (int): Queuing system ID of the job to check.
+        config_directory (str, optional): path to the config directory.
+        backend (str, optional): name of the backend used to spawn tasks ["slurm", "flux"].
+
+    Returns:
+        str: status of the job as reported by the queuing system, None if the queuing system no
+             longer knows about the job (e.g. it timed out, was killed or already finished).
+    """
+    qa = QueueAdapter(
+        directory=config_directory,
+        queue_type=backend,
+        execute_command=pysqa_execute_command,
+    )
+    return qa.get_status_of_job(process_id=queue_id)
+
+
+def pysqa_execute_command(
+    commands: str,
+    working_directory: Optional[str] = None,
+    split_output: bool = True,
+    shell: bool = False,
+    error_filename: str = "pysqa.err",
+) -> Union[str, list[str]]:
+    """
+    A wrapper around the subprocess.check_output function. Modified from pysqa to raise an exception if the subprocess
+    fails to submit the job to the queue.
+
+    Args:
+        commands (str): The command(s) to be executed on the command line
+        working_directory (str, optional): The directory where the command is executed. Defaults to None.
+        split_output (bool, optional): Boolean flag to split newlines in the output. Defaults to True.
+        shell (bool, optional): Additional switch to convert commands to a single string. Defaults to False.
+        error_filename (str, optional): In case the execution fails, the output is written to this file. Defaults to "pysqa.err".
+
+    Returns:
+        Union[str, List[str]]: Output of the shell command either as a string or as a list of strings
+    """
+    if shell and isinstance(commands, list):
+        commands = " ".join(commands)
+    out = subprocess.check_output(
+        commands,
+        cwd=working_directory,
+        stderr=subprocess.STDOUT,
+        universal_newlines=True,
+        shell=not isinstance(commands, list),
+    )
+    if out is not None and split_output:
+        return out.split("\n")
+    else:
+        return out

executorlib/standalone/interactive/communication.py

@@ -78,19 +78,17 @@ class SocketInterface:
         while len(response_lst) == 0:
             response_lst = self._poller.poll(self._time_out_ms)
             if not self._spawner.poll():
-                raise ExecutorlibSocketError(
-                    "SocketInterface crashed during execution."
-                )
+                return {
+                    "error": ExecutorlibSocketError(
+                        "SocketInterface crashed during execution."
+                    ),
+                }
         data = self._socket.recv(zmq.NOBLOCK)
         if self._logger is not None:
             self._logger.warning(
                 "Received dictionary of size: " + str(sys.getsizeof(data))
             )
-        output = cloudpickle.loads(data)
-        if "result" in output:
-            return output["result"]
-        else:
-            raise output["error"]
+        return cloudpickle.loads(data)
 
     def send_and_receive_dict(self, input_dict: dict) -> dict:
         """
@@ -154,7 +152,7 @@ class SocketInterface:
         if self._spawner.poll():
             result = self.send_and_receive_dict(
                 input_dict={"shutdown": True, "wait": wait}
-            )
+            )["result"]
             self._spawner.shutdown(wait=wait)
         self._reset_socket()
         return result

executorlib/task_scheduler/file/shared.py

@@ -8,7 +8,7 @@ from typing import Any, Callable, Optional
 from executorlib.standalone.command import get_cache_execute_command
 from executorlib.standalone.hdf import get_cache_files, get_output, get_queue_id
 from executorlib.standalone.serialize import serialize_funct
-from executorlib.task_scheduler.file.spawner_subprocess import terminate_subprocess
+from executorlib.task_scheduler.file.spawner_subprocess import subprocess_terminate
 
 
 class FutureItem:
@@ -59,6 +59,7 @@ def execute_tasks_h5(
     execute_function: Callable,
     executor_kwargs: dict,
     terminate_function: Optional[Callable] = None,
+    validate_function: Optional[Callable] = None,
     pysqa_config_directory: Optional[str] = None,
     backend: Optional[str] = None,
     disable_dependencies: bool = False,
@@ -76,6 +77,7 @@ def execute_tasks_h5(
                               - cwd (str/None): current working directory where the parallel python task is executed
         execute_function (Callable): The function to execute the tasks.
         terminate_function (Callable): The function to terminate the tasks.
+        validate_function (Callable): The function to validate the tasks.
         pysqa_config_directory (str, optional): path to the pysqa config directory (only for pysqa based backend).
         backend (str, optional): name of the backend used to spawn tasks.
         disable_dependencies (boolean): Disable resolving future objects during the submission.
@@ -92,6 +94,7 @@ def execute_tasks_h5(
     cache_dir_dict: dict = {}
     file_name_dict: dict = {}
     duplicate_dict: dict = {}
+    status_check_dict: dict = {}
     while True:
         task_dict = None
         with contextlib.suppress(queue.Empty):
@@ -104,7 +107,9 @@ def execute_tasks_h5(
                 process_dict=process_dict,
                 duplicate_dict=duplicate_dict,
                 cache_dir_dict=cache_dir_dict,
+                status_check_dict=status_check_dict,
                 terminate_function=terminate_function,
+                validate_function=validate_function,
                 pysqa_config_directory=pysqa_config_directory,
                 backend=backend,
                 refresh_rate=refresh_rate,
@@ -188,7 +193,9 @@ def execute_tasks_h5(
                 cache_dir_dict=cache_dir_dict,
                 process_dict=process_dict,
                 duplicate_dict=duplicate_dict,
+                status_check_dict=status_check_dict,
                 terminate_function=terminate_function,
+                validate_function=validate_function,
                 pysqa_config_directory=pysqa_config_directory,
                 backend=backend,
                 refresh_rate=refresh_rate,
@@ -199,15 +206,30 @@ def _check_task_output(
     task_key: str,
     future_obj: Future,
     cache_directory: str,
+    queue_id: Optional[int] = None,
+    pysqa_config_directory: Optional[str] = None,
+    backend: Optional[str] = None,
+    validate_function: Optional[Callable] = None,
+    status_check_dict: Optional[dict] = None,
     duplicate_dict: Optional[dict] = None,
 ) -> Future:
     """
     Check the output of a task and set the result of the future object if available.
 
+    If the output file is missing and the task is running on a queuing system backend, this also
+    detects jobs which died without producing output (e.g. walltime TIMEOUT, OOM, NODE_FAIL or an
+    external scancel) by periodically querying the job status via pysqa and fails the future
+    instead of leaving it pending forever.
+
     Args:
         task_key (str): The key of the task.
         future_obj (Future): The future object associated with the task.
         cache_directory (str): The directory where the HDF5 files are stored.
+        queue_id (int, optional): The queuing system ID of the task, if submitted via pysqa.
+        pysqa_config_directory (str, optional): path to the pysqa config directory.
+        backend (str, optional): name of the backend used to spawn tasks ["slurm", "flux"].
+        status_check_dict (dict): Dictionary tracking when each task's job status was last queried,
+            used to throttle calls to the queuing system.
         duplicate_dict (dict): The dictionary mapping task keys to their associated duplicate future objects.
     Returns:
         Future: The updated future object.
@@ -215,8 +237,32 @@ def _check_task_output(
     """
     file_name = os.path.join(cache_directory, task_key + "_o.h5")
     if not os.path.exists(file_name):
-        return future_obj
-    exec_flag, no_error_flag, result = get_output(file_name=file_name)
+        if (
+            backend is None
+            or queue_id is None
+            or validate_function is None
+            or not validate_function(
+                task_key=task_key,
+                file_name=file_name,
+                queue_id=queue_id,
+                pysqa_config_directory=pysqa_config_directory,
+                backend=backend,
+                status_check_dict=status_check_dict,
+            )
+        ):
+            return future_obj
+        exec_flag, no_error_flag, result = (
+            True,
+            False,
+            RuntimeError(
+                f"executorlib: queue job {queue_id} for task {task_key} terminated without "
+                "producing output (timeout, out-of-memory, node failure or cancellation)."
+            ),
+        )
+    else:
+        exec_flag, no_error_flag, result = get_output(file_name=file_name)
+    if status_check_dict is not None:
+        status_check_dict.pop(task_key, None)
     _update_future(
         future_obj=future_obj,
         exec_flag=exec_flag,
@@ -321,7 +367,9 @@ def _refresh_memory_dict(
     cache_dir_dict: dict,
     process_dict: dict,
     duplicate_dict: Optional[dict] = None,
+    status_check_dict: Optional[dict] = None,
     terminate_function: Optional[Callable] = None,
+    validate_function: Optional[Callable] = None,
     pysqa_config_directory: Optional[str] = None,
     backend: Optional[str] = None,
     refresh_rate: float = 0.01,
@@ -334,7 +382,10 @@ def _refresh_memory_dict(
         cache_dir_dict (dict): dictionary with task keys and cache directories
         process_dict (dict): dictionary with task keys and process reference.
         duplicate_dict (dict): dictionary with task keys and duplicate future objects.
+        status_check_dict (dict): dictionary with task keys and the last time their queuing system
+            job status was queried, used to throttle detection of jobs that died without output.
         terminate_function (callable): The function to terminate the tasks.
+        validate_function (callable): The function to validate the tasks.
         pysqa_config_directory (str): path to the pysqa config directory (only for pysqa based backend).
         backend (str): name of the backend used to spawn tasks.
         refresh_rate (float): The rate at which to refresh the result. Defaults to 0.01.
@@ -351,11 +402,19 @@ def _refresh_memory_dict(
         pysqa_config_directory=pysqa_config_directory,
         backend=backend,
     )
+    if status_check_dict is not None:
+        for key in cancelled_lst:
+            status_check_dict.pop(key, None)
     memory_updated_dict = {
         key: _check_task_output(
             task_key=key,
             future_obj=value,
             cache_directory=cache_dir_dict[key],
+            queue_id=process_dict.get(key),
+            pysqa_config_directory=pysqa_config_directory,
+            backend=backend,
+            validate_function=validate_function,
+            status_check_dict=status_check_dict,
             duplicate_dict=duplicate_dict,
         )
         for key, value in memory_dict.items()
@@ -381,7 +440,7 @@ def _cancel_processes(
         pysqa_config_directory (str): path to the pysqa config directory (only for pysqa based backend).
         backend (str): name of the backend used to spawn tasks.
     """
-    if terminate_function is not None and terminate_function == terminate_subprocess:
+    if terminate_function is not None and terminate_function == subprocess_terminate:
         for task in process_dict.values():
             terminate_function(task=task)
     elif terminate_function is not None and backend is not None:
@@ -443,7 +502,9 @@ def _shutdown_executor(
     process_dict: dict,
     cache_dir_dict: dict,
     duplicate_dict: Optional[dict] = None,
+    status_check_dict: Optional[dict] = None,
     terminate_function: Optional[Callable] = None,
+    validate_function: Optional[Callable] = None,
     pysqa_config_directory: Optional[str] = None,
     backend: Optional[str] = None,
     refresh_rate: float = 0.01,
@@ -466,6 +527,9 @@ def _shutdown_executor(
         process_dict (dict): Mapping of task keys to process handles or queue IDs.
         duplicate_dict (dict): Mapping of task keys to lists of duplicate Future objects.
         cache_dir_dict (dict): Mapping of task keys to the cache directory for each task.
+        status_check_dict (dict): Mapping of task keys to the last time their queuing system job
+            status was queried, used to throttle detection of jobs that died without output.
+        validate_function (Callable, optional): Function used to validate the tasks.
         terminate_function (Callable, optional): Function used to terminate running processes.
         pysqa_config_directory (str, optional): Path to the pysqa config directory.
         backend (str, optional): Name of the backend ("slurm", "flux", or None for subprocess).
@@ -478,7 +542,9 @@ def _shutdown_executor(
                 cache_dir_dict=cache_dir_dict,
                 process_dict=process_dict,
                 duplicate_dict=duplicate_dict,
+                status_check_dict=status_check_dict,
                 terminate_function=terminate_function,
+                validate_function=validate_function,
                 pysqa_config_directory=pysqa_config_directory,
                 backend=backend,
                 refresh_rate=refresh_rate,
@@ -493,7 +559,9 @@ def _shutdown_executor(
                 cache_dir_dict=cache_dir_dict,
                 process_dict=process_dict,
                 duplicate_dict=duplicate_dict,
+                status_check_dict=status_check_dict,
                 terminate_function=terminate_function,
+                validate_function=validate_function,
                 pysqa_config_directory=pysqa_config_directory,
                 backend=backend,
                 refresh_rate=refresh_rate,
@@ -512,7 +580,9 @@ def _shutdown_executor(
             cache_dir_dict=cache_dir_dict,
             process_dict=process_dict,
             duplicate_dict=duplicate_dict,
+            status_check_dict=status_check_dict,
             terminate_function=terminate_function,
+            validate_function=validate_function,
             pysqa_config_directory=pysqa_config_directory,
             backend=backend,
             refresh_rate=refresh_rate,

executorlib/task_scheduler/file/spawner_pysqa.py

@@ -4,12 +4,12 @@ from typing import Optional
 
 from pysqa import QueueAdapter
 
+from executorlib.standalone.command_pysqa import pysqa_execute_command, pysqa_terminate
 from executorlib.standalone.hdf import dump, get_queue_id
 from executorlib.standalone.inputcheck import check_file_exists
 from executorlib.standalone.interactive.spawner import (
     set_current_directory_in_environment,
 )
-from executorlib.standalone.scheduler import pysqa_execute_command, terminate_with_pysqa
 
 
 def execute_with_pysqa(
@@ -122,7 +122,7 @@ def terminate_tasks_in_cache(
     for f in hdf5_file_lst:
         queue_id = get_queue_id(f)
         if queue_id is not None:
-            terminate_with_pysqa(
+            pysqa_terminate(
                 queue_id=queue_id,
                 config_directory=pysqa_config_directory,
                 backend=backend,
@@ -148,7 +148,7 @@ def terminate_task_in_cache(
     file_name = os.path.join(cache_directory, cache_key + "_i.h5")
     queue_id = get_queue_id(file_name=file_name)
     if queue_id is not None:
-        terminate_with_pysqa(
+        pysqa_terminate(
             queue_id=queue_id,
             config_directory=pysqa_config_directory,
             backend=backend,

executorlib/task_scheduler/file/spawner_subprocess.py

@@ -10,7 +10,7 @@ from executorlib.standalone.interactive.spawner import (
 )
 
 
-def execute_in_subprocess(
+def subprocess_execute(
     command: list,
     file_name: str,
     data_dict: dict,
@@ -65,7 +65,7 @@ def execute_in_subprocess(
     return subprocess.Popen(command, universal_newlines=True, cwd=cwd)
 
 
-def terminate_subprocess(task):
+def subprocess_terminate(task):
     """
     Terminate a subprocess and wait for it to complete.
 

executorlib/task_scheduler/file/task_scheduler.py

@@ -12,17 +12,21 @@ from executorlib.standalone.inputcheck import (
 from executorlib.task_scheduler.base import TaskSchedulerBase, validate_resource_dict
 from executorlib.task_scheduler.file.shared import execute_tasks_h5
 from executorlib.task_scheduler.file.spawner_subprocess import (
-    execute_in_subprocess,
-    terminate_subprocess,
+    subprocess_execute,
+    subprocess_terminate,
 )
 
 try:
-    from executorlib.standalone.scheduler import terminate_with_pysqa
+    from executorlib.standalone.command_pysqa import (
+        pysqa_job_output_validation,
+        pysqa_terminate,
+    )
     from executorlib.task_scheduler.file.spawner_pysqa import execute_with_pysqa
 except ImportError:
     # If pysqa is not available fall back to executing tasks in a subprocess
-    execute_with_pysqa = execute_in_subprocess  # type: ignore
-    terminate_with_pysqa = None  # type: ignore
+    execute_with_pysqa = subprocess_execute  # type: ignore
+    pysqa_terminate = None  # type: ignore
+    pysqa_job_output_validation = None  # type: ignore
 
 
 class FileTaskScheduler(TaskSchedulerBase):
@@ -74,6 +78,7 @@ class FileTaskScheduler(TaskSchedulerBase):
             "future_queue": self._future_queue,
             "execute_function": execute_function,
             "terminate_function": terminate_function,
+            "validate_function": pysqa_job_output_validation,
             "pysqa_config_directory": pysqa_config_directory,
             "backend": backend,
             "disable_dependencies": disable_dependencies,
@@ -126,10 +131,10 @@ def create_file_executor(
     check_executor(executor=flux_executor)
     check_nested_flux_executor(nested_flux_executor=flux_executor_nesting)
     check_flux_log_files(flux_log_files=flux_log_files)
-    if execute_function != execute_in_subprocess:
-        terminate_function = terminate_with_pysqa  # type: ignore
+    if execute_function != subprocess_execute:
+        terminate_function = pysqa_terminate  # type: ignore
     else:
-        terminate_function = terminate_subprocess  # type: ignore
+        terminate_function = subprocess_terminate  # type: ignore
     return FileTaskScheduler(
         executor_kwargs=executor_kwargs,
         pysqa_config_directory=pysqa_config_directory,

executorlib/task_scheduler/interactive/blockallocation.py

@@ -379,10 +379,9 @@ def _set_init_function(
 ) -> Optional[Exception]:
     interface_initialization_exception = None
     if init_function is not None and interface.status:
-        try:
-            _ = interface.send_and_receive_dict(
-                input_dict={"init": True, "fn": init_function, "args": (), "kwargs": {}}
-            )
-        except Exception as init_exception:
-            interface_initialization_exception = init_exception
+        output = interface.send_and_receive_dict(
+            input_dict={"init": True, "fn": init_function, "args": (), "kwargs": {}}
+        )
+        if "error" in output:
+            interface_initialization_exception = output["error"]
     return interface_initialization_exception

executorlib/task_scheduler/interactive/shared.py

@@ -101,13 +101,13 @@ def _execute_task_without_cache(
     Returns:
         bool: True if the task was submitted successfully, False otherwise.
     """
-    try:
-        future_obj.set_result(interface.send_and_receive_dict(input_dict=task_dict))
-    except Exception as thread_exception:
-        if isinstance(thread_exception, ExecutorlibSocketError):
-            return False
-        else:
-            future_obj.set_exception(exception=thread_exception)
+    output = interface.send_and_receive_dict(input_dict=task_dict)
+    if "result" in output:
+        future_obj.set_result(output["result"])
+    elif isinstance(output["error"], ExecutorlibSocketError):
+        return False
+    else:
+        future_obj.set_exception(exception=output["error"])
     return True
 
 
@@ -141,18 +141,17 @@ def _execute_task_with_cache(
     )
     file_name = os.path.abspath(os.path.join(cache_directory, task_key + "_o.h5"))
     if file_name not in get_cache_files(cache_directory=cache_directory):
-        try:
-            time_start = time.time()
-            result = interface.send_and_receive_dict(input_dict=task_dict)
-            data_dict["output"] = result
+        time_start = time.time()
+        output = interface.send_and_receive_dict(input_dict=task_dict)
+        if "result" in output:
+            data_dict["output"] = output["result"]
             data_dict["runtime"] = time.time() - time_start
             dump(file_name=file_name, data_dict=data_dict)
-            future_obj.set_result(result)
-        except Exception as thread_exception:
-            if isinstance(thread_exception, ExecutorlibSocketError):
-                return False
-            else:
-                future_obj.set_exception(exception=thread_exception)
+            future_obj.set_result(output["result"])
+        elif isinstance(output["error"], ExecutorlibSocketError):
+            return False
+        else:
+            future_obj.set_exception(exception=output["error"])
     else:
         _, _, result = get_output(file_name=file_name)
         future_obj.set_result(result)

executorlib/task_scheduler/interactive/spawner_pysqa.py

@@ -5,12 +5,12 @@ from typing import Callable, Optional
 
 from pysqa import QueueAdapter
 
+from executorlib.standalone.command_pysqa import pysqa_execute_command, pysqa_terminate
 from executorlib.standalone.inputcheck import validate_number_of_cores
 from executorlib.standalone.interactive.spawner import (
     BaseSpawner,
     set_current_directory_in_environment,
 )
-from executorlib.standalone.scheduler import pysqa_execute_command, terminate_with_pysqa
 from executorlib.task_scheduler.base import validate_resource_dict
 from executorlib.task_scheduler.interactive.blockallocation import (
     BlockAllocationTaskScheduler,
@@ -162,7 +162,7 @@ class PysqaSpawner(BaseSpawner):
             wait (bool, optional): Whether to wait for the interface to shutdown. Defaults to True.
         """
         if self._process is not None:
-            terminate_with_pysqa(
+            pysqa_terminate(
                 queue_id=self._process,
                 config_directory=self._config_directory,
                 backend=self._backend,

{executorlib-1.10.1.dist-info → executorlib-1.10.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: executorlib
-Version: 1.10.1
+Version: 1.10.2
 Summary: Up-scale python functions for high performance computing (HPC) with executorlib.
 Project-URL: Homepage, https://github.com/pyiron/executorlib
 Project-URL: Documentation, https://executorlib.readthedocs.io
@@ -57,12 +57,12 @@ Requires-Dist: mpi4py<=4.1.2,>=3.1.4; extra == 'all'
 Requires-Dist: networkx<=3.6.1,>=2.8.8; extra == 'all'
 Requires-Dist: pydantic<=2.13.4,>=2.5.3; extra == 'all'
 Requires-Dist: pygraphviz<=1.14,>=1.10; extra == 'all'
-Requires-Dist: pysqa==0.4.3; extra == 'all'
+Requires-Dist: pysqa==0.4.4; extra == 'all'
 Provides-Extra: cache
 Requires-Dist: h5py<=3.16.0,>=3.6.0; extra == 'cache'
 Provides-Extra: cluster
 Requires-Dist: h5py<=3.16.0,>=3.6.0; extra == 'cluster'
-Requires-Dist: pysqa==0.4.3; extra == 'cluster'
+Requires-Dist: pysqa==0.4.4; extra == 'cluster'
 Provides-Extra: graph
 Requires-Dist: networkx<=3.6.1,>=2.8.8; extra == 'graph'
 Requires-Dist: pygraphviz<=1.14,>=1.10; extra == 'graph'

{executorlib-1.10.1.dist-info → executorlib-1.10.2.dist-info}/RECORD

@@ -1,5 +1,5 @@
 executorlib/__init__.py,sha256=IY_DkT706LPIv2_iIe3whrQNGoDzaDFYBvXyUepoZpw,4780
-executorlib/_version.py,sha256=9cYIW_mnApkxjl7hrsLPCkpAtl6R-GXOqmSgy2KFfGA,522
+executorlib/_version.py,sha256=aubWhgSag2TvKOWdccSUKUZIISdFObmOxMDtFhqzXC0,522
 executorlib/api.py,sha256=F6ibDJnVcPGxxPe-UScFJHprWHiVsETGhgAvS1T-9ZI,1365
 executorlib/backend/__init__.py,sha256=2b_iwds4VUrJe92nKtQXkoGxkJiXeEDSlbvCRN_-de8,456
 executorlib/backend/cache_parallel.py,sha256=ldb2iYzrhcXJZ0LcH75FpTLO_6F51mleU9JGAilF00k,2020
@@ -9,42 +9,42 @@ executorlib/backend/interactive_serial.py,sha256=ZV_VGcnCAk7N-VcoJKYEIG0OGUUwR4K
 executorlib/executor/__init__.py,sha256=oR2ABLAMYV_TNs1eG-3-MAKHFS7OKB-1KOzCqKNbtWA,310
 executorlib/executor/base.py,sha256=UKJViNoshthAgF2I4FoK0nZT0FL7WgkrzWFYhRiU1H4,6957
 executorlib/executor/flux.py,sha256=g8K1GMfn9KMM4udj3LdmDMahavPBBCOusYVG36W1hAY,40779
-executorlib/executor/single.py,sha256=U-mflJIQSBMS7atK7RQktx87LGkJhLSBbh2Uz9kOLlY,36021
+executorlib/executor/single.py,sha256=jX4BZCFdNksA3XhTpkxyLJfXGNh0j-5qq8jSgRsjwuk,36015
 executorlib/executor/slurm.py,sha256=Prvts9IgDHgCCMYoWpNdeoKMZ4h1ICQ05fkomh25Yl4,38337
 executorlib/standalone/__init__.py,sha256=n_Q4NUFu35Fn0qlWIhmoC_FyUkmFGlWn1cNc_7EscF0,290
 executorlib/standalone/batched.py,sha256=o3TtsFJir8nP1O_NZFfW0Q78CoZAFgHMyroWlg520nE,1446
 executorlib/standalone/command.py,sha256=4Xndog84zJdku1ke1DwdUamkWvzRrMz-VDCi5cRXi0M,6548
+executorlib/standalone/command_pysqa.py,sha256=eNTyHxtjUsn3950NOWHfNrFKgtxgD9GAVxCdN6Ewxcc,5948
 executorlib/standalone/error.py,sha256=aaMQoRsoLzLCbYT-E4LZxGM9pm_AhjMesRjVt0d_LDk,714
 executorlib/standalone/hdf.py,sha256=dRePOCYXbPG_i__ASUcD4qs5AAU2fPbP5o_5ytwaOjw,7805
 executorlib/standalone/inputcheck.py,sha256=kSWUzqdMOhILYXW4Isumn1EGH3SegxPUMWRkUWfEleU,10041
 executorlib/standalone/queue.py,sha256=KFEBFQXGvcaf5qcuCjT5kaZg7ucWOTGOtrD96tIBlq0,589
-executorlib/standalone/scheduler.py,sha256=LhCuIdaw9BjrP3GerbHUIqwsvxERdtH5JQoeyAqASIU,2325
 executorlib/standalone/select.py,sha256=FUSXYbhoc_uP_vPji8L4PcK4b2Gwcai7rXZ5dKRVjZM,3061
 executorlib/standalone/serialize.py,sha256=7DxcuCYZVqLf8wa7EXsN-_8fSpUJqyQ-5G6NT863SgM,4716
 executorlib/standalone/validate.py,sha256=hHG32xomLhbqnqd6Vn4AO05bQ7B3WZ1rngTkMsODwwk,4430
 executorlib/standalone/interactive/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 executorlib/standalone/interactive/arguments.py,sha256=iaVXf7Wq9MUOu3WeYSi2t-f8Si_pyAPyykcnq8bf8cY,3239
 executorlib/standalone/interactive/backend.py,sha256=gGgcvER8w75bM_p1uplSsr6yCtBBlOS_6wR1iQNsntc,3006
-executorlib/standalone/interactive/communication.py,sha256=ZYngnW-myohI5jjF1RDHfMEX8dVDN2Ow6kBjMVkAfxw,10448
+executorlib/standalone/interactive/communication.py,sha256=ctiVWkc7g1mfsYHJSJzg29nAvaEkJopXpajBmOpJOIc,10400
 executorlib/standalone/interactive/spawner.py,sha256=4TEPL4iI_rF25kOaW-eqY74jK93LFZ05cYOHUjIuo4M,6384
 executorlib/task_scheduler/__init__.py,sha256=-sw60rfnEvwFiOAONFHpE3izPA9CPtp5cQ7vRetw2Mw,249
 executorlib/task_scheduler/base.py,sha256=A9nQb8FYKbZGjQP1G5ccT75ODbEqqiM0OVybHAxjqzE,9718
 executorlib/task_scheduler/file/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 executorlib/task_scheduler/file/backend.py,sha256=tojXcYytAsfABugsOMfkx9oAh01do_DcSneVldMfeK0,2876
-executorlib/task_scheduler/file/shared.py,sha256=zRODWHzqZBzrEjmJenBNBwwM2mqMcnNIXpOfTGsyYBo,21128
-executorlib/task_scheduler/file/spawner_pysqa.py,sha256=j7GdUQgo6vZKqbXtdXL3doaCK8InrDdPJOn0kWGY7-4,6032
-executorlib/task_scheduler/file/spawner_subprocess.py,sha256=BejOMi5zLkIVUUywv5hl0YdnXkGujGo4nXMpmmuHq2k,3089
-executorlib/task_scheduler/file/task_scheduler.py,sha256=c57K8d3Sfh4p--xvAy5LsCtTq9GOdymaUeN0h4POyS8,6019
+executorlib/task_scheduler/file/shared.py,sha256=uJECqjLr_Mq9AQdVOAkEdN-gS2dryfwufmtrcgMs02M,24719
+executorlib/task_scheduler/file/spawner_pysqa.py,sha256=z6kEsyVnCFmpLmVlHL2LxumAgDU0t0zvo59snMv0sMw,6021
+executorlib/task_scheduler/file/spawner_subprocess.py,sha256=yMc4nHRUghgdR0ehihE2CYsaTn1RspRI3ddhIhkTgDw,3086
+executorlib/task_scheduler/file/task_scheduler.py,sha256=gfEpNVJ9u87_yaDYZ86kHt_mAEXfOqBwvxX4IlpKt6o,6170
 executorlib/task_scheduler/interactive/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-executorlib/task_scheduler/interactive/blockallocation.py,sha256=HcfrYEuopD7wLUoxuoawRRNdaH4dTsPnFTb7luR7G5Y,17738
+executorlib/task_scheduler/interactive/blockallocation.py,sha256=hsKTDScVxzLJ4sHhAWIoMjRxZxX2zzpO4r_z-7ksV_s,17705
 executorlib/task_scheduler/interactive/dependency.py,sha256=zQMfI6AjeUv3A1qyPzFah2BKYGWgGLRyEIdZ0XTjq1M,15222
 executorlib/task_scheduler/interactive/dependency_plot.py,sha256=iZBZ9UMDeYOMFjD2-kYOxOdKeodj7TuyshI4HbH4XCU,11788
 executorlib/task_scheduler/interactive/onetoone.py,sha256=cTXgTXR9kKzzlx6EfGumMsEm7DUa7BD9JSeBpdPiupI,13312
-executorlib/task_scheduler/interactive/shared.py,sha256=-krntmZFOOE5U0HdFwuU8JZPqZQmPW4-cePDWuvkTgM,6305
+executorlib/task_scheduler/interactive/shared.py,sha256=tGlhIsqOfkO7QwNmr3I71M8DFT3NNXHwyUoZ_BrjfWQ,6263
 executorlib/task_scheduler/interactive/spawner_flux.py,sha256=M7CPWvzY7MDkplJ4awymh7WbMYwKsDPTKxA2Op9wfNo,7075
-executorlib/task_scheduler/interactive/spawner_pysqa.py,sha256=i-E9MOK9_aiUigrrGFMLs_m0y3VhrEgu95_PXCOg-hg,10352
+executorlib/task_scheduler/interactive/spawner_pysqa.py,sha256=jQB8PQAH9cBo_ofpxHTzydg0ghwSOP51CtVL96YH_fc,10346
 executorlib/task_scheduler/interactive/spawner_slurm.py,sha256=K5mj63PE8FYbkw9WWm4uEjoA5q0HM42z1d2wjAmO5YA,3713
-executorlib-1.10.1.dist-info/METADATA,sha256=cK6UBcUFueR_PY8sOJC9D6xJ0gBNTJ7oQyVSLlFi4A8,20966
-executorlib-1.10.1.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
-executorlib-1.10.1.dist-info/licenses/LICENSE,sha256=NkrhccIxPXCbbvp_upyVojt7DNTwZIco3z9wsYpSLqU,1511
-executorlib-1.10.1.dist-info/RECORD,,
+executorlib-1.10.2.dist-info/METADATA,sha256=xt32x11FToSi84-Knrpr2Rx4OIUUBfWth9c7xSPf0Es,20966
+executorlib-1.10.2.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
+executorlib-1.10.2.dist-info/licenses/LICENSE,sha256=NkrhccIxPXCbbvp_upyVojt7DNTwZIco3z9wsYpSLqU,1511
+executorlib-1.10.2.dist-info/RECORD,,

{executorlib-1.10.1.dist-info → executorlib-1.10.2.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.30.1
+Generator: hatchling 1.31.0
 Root-Is-Purelib: true
 Tag: py3-none-any

executorlib/standalone/scheduler.py

@@ -1,65 +0,0 @@
-import contextlib
-import subprocess
-from typing import Optional, Union
-
-from pysqa import QueueAdapter
-
-
-def terminate_with_pysqa(
-    queue_id: int,
-    config_directory: Optional[str] = None,
-    backend: Optional[str] = None,
-):
-    """
-    Delete job from queuing system
-
-    Args:
-        queue_id (int): Queuing system ID of the job to delete.
-        config_directory (str, optional): path to the config directory.
-        backend (str, optional): name of the backend used to spawn tasks ["slurm", "flux"].
-    """
-    qa = QueueAdapter(
-        directory=config_directory,
-        queue_type=backend,
-        execute_command=pysqa_execute_command,
-    )
-    status = qa.get_status_of_job(process_id=queue_id)
-    if status is not None and status not in ["finished", "error"]:
-        with contextlib.suppress(subprocess.CalledProcessError):
-            qa.delete_job(process_id=queue_id)
-
-
-def pysqa_execute_command(
-    commands: str,
-    working_directory: Optional[str] = None,
-    split_output: bool = True,
-    shell: bool = False,
-    error_filename: str = "pysqa.err",
-) -> Union[str, list[str]]:
-    """
-    A wrapper around the subprocess.check_output function. Modified from pysqa to raise an exception if the subprocess
-    fails to submit the job to the queue.
-
-    Args:
-        commands (str): The command(s) to be executed on the command line
-        working_directory (str, optional): The directory where the command is executed. Defaults to None.
-        split_output (bool, optional): Boolean flag to split newlines in the output. Defaults to True.
-        shell (bool, optional): Additional switch to convert commands to a single string. Defaults to False.
-        error_filename (str, optional): In case the execution fails, the output is written to this file. Defaults to "pysqa.err".
-
-    Returns:
-        Union[str, List[str]]: Output of the shell command either as a string or as a list of strings
-    """
-    if shell and isinstance(commands, list):
-        commands = " ".join(commands)
-    out = subprocess.check_output(
-        commands,
-        cwd=working_directory,
-        stderr=subprocess.STDOUT,
-        universal_newlines=True,
-        shell=not isinstance(commands, list),
-    )
-    if out is not None and split_output:
-        return out.split("\n")
-    else:
-        return out