parsl 2025.9.8__py3-none-any.whl → 2025.11.10__py3-none-any.whl

parsl/app/bash.py

@@ -88,7 +88,7 @@ def remote_side_bash_executor(func, *args, **kwargs):
         raise pe.AppTimeout(f"App {func_name} exceeded walltime: {timeout} seconds")
 
     except Exception as e:
-        raise pe.AppException(f"App {func_name} caught exception with returncode: {returncode}", e)
+        raise pe.AppException(f"App {func_name} caught exception", e)
 
     if returncode != 0:
         raise pe.BashExitFailure(func_name, proc.returncode)

parsl/benchmark/perf.py

@@ -2,46 +2,65 @@ import argparse
 import concurrent.futures
 import importlib
 import time
+from typing import Any, Dict, Literal
 
 import parsl
+from parsl.dataflow.dflow import DataFlowKernel
+from parsl.errors import InternalConsistencyError
+
+VALID_NAMED_ITERATION_MODES = ("estimate", "exponential")
 
 min_iterations = 2
 
 
 # TODO: factor with conftest.py where this is copy/pasted from?
-def load_dfk_from_config(filename):
+def load_dfk_from_config(filename: str) -> DataFlowKernel:
     spec = importlib.util.spec_from_file_location('', filename)
+
+    if spec is None:
+        raise RuntimeError("Could not import configuration")
+
+    module = importlib.util.module_from_spec(spec)
+
+    if spec.loader is None:
+        raise RuntimeError("Could not load configuration")
+
+    spec.loader.exec_module(module)
+
     module = importlib.util.module_from_spec(spec)
     spec.loader.exec_module(module)
 
     if hasattr(module, 'config'):
-        parsl.load(module.config)
+        return parsl.load(module.config)
     elif hasattr(module, 'fresh_config'):
-        parsl.load(module.fresh_config())
+        return parsl.load(module.fresh_config())
     else:
         raise RuntimeError("Config module does not define config or fresh_config")
 
 
 @parsl.python_app
-def app(extra_payload, parsl_resource_specification={}):
+def app(extra_payload: Any, parsl_resource_specification: Dict = {}) -> int:
     return 7
 
 
-def performance(*, resources: dict, target_t: float, args_extra_size: int):
-    n = 10
+def performance(*, resources: dict, target_t: float, args_extra_size: int, iterate_mode: str | list[int]) -> None:
 
     delta_t: float
-    delta_t = 0
-
-    threshold_t = int(0.75 * target_t)
 
     iteration = 1
 
     args_extra_payload = "x" * args_extra_size
 
-    while delta_t < threshold_t or iteration <= min_iterations:
+    if isinstance(iterate_mode, list):
+        n = iterate_mode[0]
+    else:
+        n = 10
+
+    iterate = True
+
+    while iterate:
         print(f"==== Iteration {iteration} ====")
-        print(f"Will run {n} tasks to target {target_t} seconds runtime")
+        print(f"Will run {n} tasks")
         start_t = time.time()
 
         fs = []
@@ -65,10 +84,42 @@ def performance(*, resources: dict, target_t: float, args_extra_size: int):
         print(f"Runtime: actual {delta_t:.3f}s vs target {target_t}s")
         print(f"Tasks per second: {rate:.3f}")
 
-        n = max(1, int(target_t * rate))
-
         iteration += 1
 
+        # decide upon next iteration
+
+        match iterate_mode:
+            case "estimate":
+                n = max(1, int(target_t * rate))
+                iterate = delta_t < (0.75 * target_t) or iteration <= min_iterations
+            case "exponential":
+                n = int(n * 2)
+                iterate = delta_t < target_t or iteration <= min_iterations
+            case seq if isinstance(seq, list) and iteration <= len(seq):
+                n = seq[iteration - 1]
+                iterate = True
+            case seq if isinstance(seq, list):
+                iterate = False
+            case _:
+                raise InternalConsistencyError(f"Bad iterate mode {iterate_mode} - should have been validated at arg parse time")
+
+
+def validate_int_list(v: str) -> list[int] | Literal[False]:
+    try:
+        return list(map(int, v.split(",")))
+    except ValueError:
+        return False
+
+
+def iteration_mode(v: str) -> str | list[int]:
+    match v:
+        case s if s in VALID_NAMED_ITERATION_MODES:
+            return s
+        case _ if seq := validate_int_list(v):
+            return seq
+        case _:
+            raise argparse.ArgumentTypeError(f"Invalid iteration mode: {v}")
+
 
 def cli_run() -> None:
     parser = argparse.ArgumentParser(
@@ -82,6 +133,12 @@ Example usage: python -m parsl.benchmark.perf --config parsl/tests/configs/workq
     parser.add_argument("--resources", metavar="EXPR", help="parsl_resource_specification dictionary")
     parser.add_argument("--time", metavar="SECONDS", help="target number of seconds for an iteration", default=120, type=float)
     parser.add_argument("--argsize", metavar="BYTES", help="extra bytes to add into app invocation arguments", default=0, type=int)
+    parser.add_argument("--version", action="version", version=f"parsl-perf from Parsl {parsl.__version__}")
+    parser.add_argument("--iterate",
+                        metavar="MODE",
+                        help="Iteration mode: " + ", ".join(VALID_NAMED_ITERATION_MODES) + ", or sequence of explicit sizes",
+                        type=iteration_mode,
+                        default="estimate")
 
     args = parser.parse_args()
 
@@ -90,10 +147,9 @@ Example usage: python -m parsl.benchmark.perf --config parsl/tests/configs/workq
     else:
         resources = {}
 
-    load_dfk_from_config(args.config)
-    performance(resources=resources, target_t=args.time, args_extra_size=args.argsize)
-    print("Cleaning up DFK")
-    parsl.dfk().cleanup()
+    with load_dfk_from_config(args.config):
+        performance(resources=resources, target_t=args.time, args_extra_size=args.argsize, iterate_mode=args.iterate)
+        print("Tests complete - leaving DFK block")
     print("The end")
 
 

parsl/concurrent/__init__.py

@@ -1,42 +1,88 @@
 """Interfaces modeled after Python's `concurrent library <https://docs.python.org/3/library/concurrent.html>`_"""
 import time
 from concurrent.futures import Executor
-from typing import Callable, Dict, Iterable, Iterator, Optional
+from contextlib import AbstractContextManager
+from typing import Callable, Dict, Iterable, Iterator, Literal, Optional
 from warnings import warn
 
-from parsl import Config, DataFlowKernel
+from parsl import Config, DataFlowKernel, load
 from parsl.app.python import PythonApp
 
 
-class ParslPoolExecutor(Executor):
+class ParslPoolExecutor(Executor, AbstractContextManager):
     """An executor that uses a pool of workers managed by Parsl
 
     Works just like a :class:`~concurrent.futures.ProcessPoolExecutor` except that tasks
     are distributed across workers that can be on different machines.
-    Create a new executor by supplying a Parsl :class:`~parsl.Config` object to define
-    how to create new workers, Parsl will set up and tear down workers on your behalf.
 
-    Note: Parsl does not support canceling tasks. The :meth:`map` method does not cancel work
+    Create a new executor using one of two methods:
+
+    1. Supplying a Parsl :class:`~parsl.Config` that defines how to create new workers.
+       The executor will start a new Parsl Data Flow Kernel (DFK) when it is entered as a context manager.
+
+    2. Supplying an already-started Parsl :class:`~parsl.DataFlowKernel` (DFK).
+       The executor assumes you will start and stop the Parsl DFK outside the Executor.
+
+    The futures returned by :meth:`submit` and :meth:`map` are Parsl futures and will work
+    with the same function chaining mechanisms as when using Parsl with decorators.
+
+    .. code-block:: python
+
+        def f(x):
+            return x + 1
+
+        @python_app
+        def parity(x):
+            return 'odd' if x % 2 == 1 else 'even'
+
+        with ParslPoolExecutor(config=my_parsl_config) as executor:
+            future_1 = executor.submit(f, 1)
+            assert parity(future_1) == 'even'  # Function chaining, as expected
+
+            future_2 = executor.submit(f, future_1)
+            assert future_2.result() == 3  # Chaining works with `submit` too
+
+    Parsl does not support canceling tasks. The :meth:`map` method does not cancel work
     when one member of the run fails or a timeout is reached
     and :meth:`shutdown` does not cancel work on completion.
     """
 
-    def __init__(self, config: Config):
+    def __init__(self, config: Config | None = None, dfk: DataFlowKernel | None = None, executors: Literal['all'] | list[str] = 'all'):
         """Create the executor
 
         Args:
             config: Configuration for the Parsl Data Flow Kernel (DFK)
+            dfk: DataFlowKernel of an already-started parsl
+            executors: List of executors to use for supplied functions
         """
+        if (config is not None) and (dfk is not None):
+            raise ValueError('Specify only one of config or dfk')
+        if (config is None) and (dfk is None):
+            raise ValueError('Must specify one of config or dfk')
         self._config = config
-        self.dfk = DataFlowKernel(self._config)
         self._app_cache: Dict[Callable, PythonApp] = {}  # Cache specific to this instance: https://stackoverflow.com/questions/33672412
+        self._dfk = dfk
+        self.executors = executors
+
+        # Start workers immediately
+        if self._config is not None:
+            self._dfk = load(self._config)
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self._dfk is None:  # Nothing has been started, do nothing
+            return
+        elif self._config is not None:  # The executors are being managed by this class, shut them down
+            self.shutdown(wait=True)
+            return
+        else:  # The DFK is managed elsewhere, do nothing
+            return
 
     @property
     def app_count(self):
         """Number of functions currently registered with the executor"""
         return len(self._app_cache)
 
-    def _get_app(self, fn: Callable) -> PythonApp:
+    def get_app(self, fn: Callable) -> PythonApp:
         """Create a PythonApp for a function
 
         Args:
@@ -46,22 +92,53 @@ class ParslPoolExecutor(Executor):
         """
         if fn in self._app_cache:
             return self._app_cache[fn]
-        app = PythonApp(fn, data_flow_kernel=self.dfk)
+        app = PythonApp(fn, data_flow_kernel=self._dfk, executors=self.executors)
         self._app_cache[fn] = app
         return app
 
     def submit(self, fn, *args, **kwargs):
-        app = self._get_app(fn)
+        """Submits a callable to be executed with the given arguments.
+
+        Schedules the callable to be executed as ``fn(*args, **kwargs)`` and returns
+        a Future instance representing the execution of the callable.
+
+        Returns:
+            A Future representing the given call.
+        """
+
+        if self._dfk is None:
+            raise RuntimeError('Executor has been shut down.')
+        app = self.get_app(fn)
         return app(*args, **kwargs)
 
     # TODO (wardlt): This override can go away when Parsl supports cancel
     def map(self, fn: Callable, *iterables: Iterable, timeout: Optional[float] = None, chunksize: int = 1) -> Iterator:
+        """Returns an iterator equivalent to map(fn, iter).
+
+        Args:
+            fn: A callable that will take as many arguments as there are
+                passed iterables.
+            timeout: The maximum number of seconds to wait. If None, then there
+                is no limit on the wait time.
+            chunksize: If greater than one, the iterables will be chopped into
+                chunks of size chunksize and submitted to the process pool.
+                If set to one, the items in the list will be sent one at a time.
+
+        Returns:
+            An iterator equivalent to: map(func, ``*iterables``) but the calls may
+            be evaluated out-of-order.
+
+        Raises:
+            TimeoutError: If the entire result iterator could not be generated
+                before the given timeout.
+            Exception: If ``fn(*args)`` raises for any values.
+        """
         # This is a version of the CPython 3.9 `.map` implementation modified to not use `cancel`
         if timeout is not None:
             end_time = timeout + time.monotonic()
 
         # Submit the applications
-        app = self._get_app(fn)
+        app = self.get_app(fn)
         fs = [app(*args) for args in zip(*iterables)]
 
         # Yield the futures as completed
@@ -78,8 +155,12 @@ class ParslPoolExecutor(Executor):
         return result_iterator()
 
     def shutdown(self, wait: bool = True, *, cancel_futures: bool = False) -> None:
+        if self._dfk is None:
+            return  # Do nothing. Nothing is active
         if cancel_futures:
             warn(message="Canceling on-going tasks is not supported in Parsl")
         if wait:
-            self.dfk.wait_for_current_tasks()
-        self.dfk.cleanup()
+            self._dfk.wait_for_current_tasks()
+        if self._config is not None:  # The executors are being managed
+            self._dfk.cleanup()  # Shutdown the DFK
+        self._dfk = None

parsl/curvezmq.py

@@ -101,17 +101,6 @@ class BaseContext(metaclass=ABCMeta):
         """
         self._ctx.destroy(linger)
 
-    def recreate(self, linger: Optional[int] = None):
-        """Destroy then recreate the context.
-
-        Parameters
-        ----------
-        linger : int, optional
-            If specified, set LINGER on sockets prior to closing them.
-        """
-        self.destroy(linger)
-        self._ctx = zmq.Context()
-
 
 class ServerContext(BaseContext):
     """CurveZMQ server context
@@ -175,11 +164,6 @@ class ServerContext(BaseContext):
             self.auth_thread.stop()
         super().destroy(linger)
 
-    def recreate(self, linger: Optional[int] = None):
-        super().recreate(linger)
-        if self.auth_thread:
-            self.auth_thread = self._start_auth_thread()
-
 
 class ClientContext(BaseContext):
     """CurveZMQ client context

parsl/data_provider/globus.py

@@ -4,7 +4,6 @@ import os
 from functools import partial
 from typing import Optional
 
-import globus_sdk
 import typeguard
 
 import parsl
@@ -79,6 +78,7 @@ class Globus:
 
     @classmethod
     def transfer_file(cls, src_ep, dst_ep, src_path, dst_path):
+        import globus_sdk
         tc = globus_sdk.TransferClient(authorizer=cls.authorizer)
         td = globus_sdk.TransferData(tc, src_ep, dst_ep)
         td.add_item(src_path, dst_path)
@@ -140,6 +140,7 @@ class Globus:
     def _do_native_app_authentication(cls, client_id, redirect_uri,
                                       requested_scopes=None):
 
+        import globus_sdk
         client = globus_sdk.NativeAppAuthClient(client_id=client_id)
         client.oauth2_start_flow(
             requested_scopes=requested_scopes,
@@ -154,6 +155,7 @@ class Globus:
 
     @classmethod
     def _get_native_app_authorizer(cls, client_id):
+        import globus_sdk
         tokens = None
         try:
             tokens = cls._load_tokens_from_file(cls.TOKEN_FILE)