parsl 2025.9.1__tar.gz → 2025.11.10__tar.gz

parsl-2025.11.10/PKG-INFO

@@ -0,0 +1,114 @@
+Metadata-Version: 2.4
+Name: parsl
+Version: 2025.11.10
+Summary: Simple data dependent workflows in Python
+Home-page: https://github.com/Parsl/parsl
+Download-URL: https://github.com/Parsl/parsl/archive/2025.11.10.tar.gz
+Author: The Parsl Team
+Author-email: parsl@googlegroups.com
+License: Apache 2.0
+Keywords: Workflows,Scientific computing
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.10.0
+License-File: LICENSE
+Requires-Dist: pyzmq>=17.1.2
+Requires-Dist: typeguard!=3.*,<5,>=2.10
+Requires-Dist: typing-extensions<5,>=4.6
+Requires-Dist: dill
+Requires-Dist: tblib
+Requires-Dist: requests
+Requires-Dist: sortedcontainers
+Requires-Dist: psutil>=5.5.1
+Requires-Dist: setproctitle
+Requires-Dist: filelock<4,>=3.13
+Provides-Extra: monitoring
+Requires-Dist: sqlalchemy<2.1,>=2; extra == "monitoring"
+Provides-Extra: visualization
+Requires-Dist: pydot>=1.4.2; extra == "visualization"
+Requires-Dist: networkx<3.3,>=3.2; extra == "visualization"
+Requires-Dist: Flask>=1.0.2; extra == "visualization"
+Requires-Dist: flask_sqlalchemy; extra == "visualization"
+Requires-Dist: pandas<3,>=2.2; extra == "visualization"
+Requires-Dist: plotly; extra == "visualization"
+Requires-Dist: python-daemon; extra == "visualization"
+Provides-Extra: aws
+Requires-Dist: boto3; extra == "aws"
+Provides-Extra: kubernetes
+Requires-Dist: kubernetes; extra == "kubernetes"
+Provides-Extra: docs
+Requires-Dist: ipython<=8.6.0; extra == "docs"
+Requires-Dist: nbsphinx; extra == "docs"
+Requires-Dist: sphinx<8,>=7.4; extra == "docs"
+Requires-Dist: sphinx_rtd_theme; extra == "docs"
+Provides-Extra: google-cloud
+Requires-Dist: google-auth; extra == "google-cloud"
+Requires-Dist: google-api-python-client; extra == "google-cloud"
+Provides-Extra: gssapi
+Requires-Dist: python-gssapi; extra == "gssapi"
+Provides-Extra: azure
+Requires-Dist: azure<=4; extra == "azure"
+Requires-Dist: msrestazure; extra == "azure"
+Provides-Extra: workqueue
+Requires-Dist: work_queue; extra == "workqueue"
+Provides-Extra: flux
+Requires-Dist: pyyaml; extra == "flux"
+Requires-Dist: cffi; extra == "flux"
+Requires-Dist: jsonschema; extra == "flux"
+Provides-Extra: proxystore
+Requires-Dist: proxystore; extra == "proxystore"
+Provides-Extra: radical-pilot
+Requires-Dist: radical.pilot==1.90; extra == "radical-pilot"
+Requires-Dist: radical.utils==1.90; extra == "radical-pilot"
+Provides-Extra: globus-compute
+Requires-Dist: globus_compute_sdk>=2.34.0; extra == "globus-compute"
+Provides-Extra: globus-transfer
+Requires-Dist: globus-sdk; extra == "globus-transfer"
+Provides-Extra: all
+Requires-Dist: sqlalchemy<2.1,>=2; extra == "all"
+Requires-Dist: pydot>=1.4.2; extra == "all"
+Requires-Dist: networkx<3.3,>=3.2; extra == "all"
+Requires-Dist: Flask>=1.0.2; extra == "all"
+Requires-Dist: flask_sqlalchemy; extra == "all"
+Requires-Dist: pandas<3,>=2.2; extra == "all"
+Requires-Dist: plotly; extra == "all"
+Requires-Dist: python-daemon; extra == "all"
+Requires-Dist: boto3; extra == "all"
+Requires-Dist: kubernetes; extra == "all"
+Requires-Dist: ipython<=8.6.0; extra == "all"
+Requires-Dist: nbsphinx; extra == "all"
+Requires-Dist: sphinx<8,>=7.4; extra == "all"
+Requires-Dist: sphinx_rtd_theme; extra == "all"
+Requires-Dist: google-auth; extra == "all"
+Requires-Dist: google-api-python-client; extra == "all"
+Requires-Dist: python-gssapi; extra == "all"
+Requires-Dist: azure<=4; extra == "all"
+Requires-Dist: msrestazure; extra == "all"
+Requires-Dist: work_queue; extra == "all"
+Requires-Dist: pyyaml; extra == "all"
+Requires-Dist: cffi; extra == "all"
+Requires-Dist: jsonschema; extra == "all"
+Requires-Dist: proxystore; extra == "all"
+Requires-Dist: radical.pilot==1.90; extra == "all"
+Requires-Dist: radical.utils==1.90; extra == "all"
+Requires-Dist: globus_compute_sdk>=2.34.0; extra == "all"
+Requires-Dist: globus-sdk; extra == "all"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: download-url
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+Simple parallel workflows system for Python

{parsl-2025.9.1 → parsl-2025.11.10}/README.rst

@@ -1,6 +1,6 @@
 Parsl - Parallel Scripting Library
 ==================================
-|licence| |docs| |NSF-1550588| |NSF-1550476| |NSF-1550562| |NSF-1550528| |NumFOCUS| |CZI-EOSS| 
+|licence| |docs| |NSF-1550588| |NSF-1550476| |NSF-1550562| |NSF-1550528| |NumFOCUS| |CZI-EOSS| |paper| 
 
 Parsl extends parallelism in Python beyond a single computer.
 
@@ -67,6 +67,9 @@ then explore the `parallel computing patterns <https://parsl.readthedocs.io/en/s
 .. |NumFOCUS| image:: https://img.shields.io/badge/powered%20by-NumFOCUS-orange.svg?style=flat&colorA=E1523D&colorB=007D8A
     :target: https://numfocus.org
     :alt: Powered by NumFOCUS
+.. |paper| image:: https://img.shields.io/badge/Software_DOI-10.1145/3307681.3325400-blue
+    :target: https://doi.org/10.1145/3307681.3325400
+    :alt: Conference proceedings paper
 
    
 Quickstart
@@ -120,7 +123,7 @@ For Developers
 Requirements
 ============
 
-Parsl is supported in Python 3.9+. Requirements can be found `here <requirements.txt>`_. Requirements for running tests can be found `here <test-requirements.txt>`_.
+Parsl is supported in Python 3.10+. Requirements can be found `here <requirements.txt>`_. Requirements for running tests can be found `here <test-requirements.txt>`_.
 
 Code of Conduct
 ===============

{parsl-2025.9.1 → parsl-2025.11.10}/parsl/__init__.py

@@ -15,7 +15,6 @@ AUTO_LOGNAME
 
 """
 import logging
-import multiprocessing as _multiprocessing
 import os
 import platform
 
@@ -32,9 +31,6 @@ from parsl.log_utils import set_file_logger, set_stream_logger
 from parsl.monitoring import MonitoringHub
 from parsl.version import VERSION
 
-if platform.system() == 'Darwin':
-    _multiprocessing.set_start_method('fork', force=True)
-
 __author__ = 'The Parsl Team'
 __version__ = VERSION
 

{parsl-2025.9.1 → parsl-2025.11.10}/parsl/app/bash.py

@@ -88,7 +88,7 @@ def remote_side_bash_executor(func, *args, **kwargs):
         raise pe.AppTimeout(f"App {func_name} exceeded walltime: {timeout} seconds")
 
     except Exception as e:
-        raise pe.AppException(f"App {func_name} caught exception with returncode: {returncode}", e)
+        raise pe.AppException(f"App {func_name} caught exception", e)
 
     if returncode != 0:
         raise pe.BashExitFailure(func_name, proc.returncode)

parsl-2025.11.10/parsl/benchmark/perf.py

@@ -0,0 +1,157 @@
+import argparse
+import concurrent.futures
+import importlib
+import time
+from typing import Any, Dict, Literal
+
+import parsl
+from parsl.dataflow.dflow import DataFlowKernel
+from parsl.errors import InternalConsistencyError
+
+VALID_NAMED_ITERATION_MODES = ("estimate", "exponential")
+
+min_iterations = 2
+
+
+# TODO: factor with conftest.py where this is copy/pasted from?
+def load_dfk_from_config(filename: str) -> DataFlowKernel:
+    spec = importlib.util.spec_from_file_location('', filename)
+
+    if spec is None:
+        raise RuntimeError("Could not import configuration")
+
+    module = importlib.util.module_from_spec(spec)
+
+    if spec.loader is None:
+        raise RuntimeError("Could not load configuration")
+
+    spec.loader.exec_module(module)
+
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+
+    if hasattr(module, 'config'):
+        return parsl.load(module.config)
+    elif hasattr(module, 'fresh_config'):
+        return parsl.load(module.fresh_config())
+    else:
+        raise RuntimeError("Config module does not define config or fresh_config")
+
+
+@parsl.python_app
+def app(extra_payload: Any, parsl_resource_specification: Dict = {}) -> int:
+    return 7
+
+
+def performance(*, resources: dict, target_t: float, args_extra_size: int, iterate_mode: str | list[int]) -> None:
+
+    delta_t: float
+
+    iteration = 1
+
+    args_extra_payload = "x" * args_extra_size
+
+    if isinstance(iterate_mode, list):
+        n = iterate_mode[0]
+    else:
+        n = 10
+
+    iterate = True
+
+    while iterate:
+        print(f"==== Iteration {iteration} ====")
+        print(f"Will run {n} tasks")
+        start_t = time.time()
+
+        fs = []
+        print("Submitting tasks / invoking apps")
+        for _ in range(n):
+            fs.append(app(args_extra_payload, parsl_resource_specification=resources))
+
+        submitted_t = time.time()
+        print(f"All {n} tasks submitted ... waiting for completion")
+        print(f"Submission took {submitted_t - start_t:.3f} seconds = {n / (submitted_t - start_t):.3f} tasks/second")
+
+        for f in concurrent.futures.as_completed(fs):
+            assert f.result() == 7
+
+        end_t = time.time()
+
+        delta_t = end_t - start_t
+
+        rate = n / delta_t
+
+        print(f"Runtime: actual {delta_t:.3f}s vs target {target_t}s")
+        print(f"Tasks per second: {rate:.3f}")
+
+        iteration += 1
+
+        # decide upon next iteration
+
+        match iterate_mode:
+            case "estimate":
+                n = max(1, int(target_t * rate))
+                iterate = delta_t < (0.75 * target_t) or iteration <= min_iterations
+            case "exponential":
+                n = int(n * 2)
+                iterate = delta_t < target_t or iteration <= min_iterations
+            case seq if isinstance(seq, list) and iteration <= len(seq):
+                n = seq[iteration - 1]
+                iterate = True
+            case seq if isinstance(seq, list):
+                iterate = False
+            case _:
+                raise InternalConsistencyError(f"Bad iterate mode {iterate_mode} - should have been validated at arg parse time")
+
+
+def validate_int_list(v: str) -> list[int] | Literal[False]:
+    try:
+        return list(map(int, v.split(",")))
+    except ValueError:
+        return False
+
+
+def iteration_mode(v: str) -> str | list[int]:
+    match v:
+        case s if s in VALID_NAMED_ITERATION_MODES:
+            return s
+        case _ if seq := validate_int_list(v):
+            return seq
+        case _:
+            raise argparse.ArgumentTypeError(f"Invalid iteration mode: {v}")
+
+
+def cli_run() -> None:
+    parser = argparse.ArgumentParser(
+        prog="parsl-perf",
+        description="Measure performance of Parsl configurations",
+        epilog="""
+Example usage: python -m parsl.benchmark.perf --config parsl/tests/configs/workqueue_blocks.py  --resources '{"cores":1, "memory":0, "disk":0}'
+        """)
+
+    parser.add_argument("--config", required=True, help="path to Python file that defines a configuration")
+    parser.add_argument("--resources", metavar="EXPR", help="parsl_resource_specification dictionary")
+    parser.add_argument("--time", metavar="SECONDS", help="target number of seconds for an iteration", default=120, type=float)
+    parser.add_argument("--argsize", metavar="BYTES", help="extra bytes to add into app invocation arguments", default=0, type=int)
+    parser.add_argument("--version", action="version", version=f"parsl-perf from Parsl {parsl.__version__}")
+    parser.add_argument("--iterate",
+                        metavar="MODE",
+                        help="Iteration mode: " + ", ".join(VALID_NAMED_ITERATION_MODES) + ", or sequence of explicit sizes",
+                        type=iteration_mode,
+                        default="estimate")
+
+    args = parser.parse_args()
+
+    if args.resources:
+        resources = eval(args.resources)
+    else:
+        resources = {}
+
+    with load_dfk_from_config(args.config):
+        performance(resources=resources, target_t=args.time, args_extra_size=args.argsize, iterate_mode=args.iterate)
+        print("Tests complete - leaving DFK block")
+    print("The end")
+
+
+if __name__ == "__main__":
+    cli_run()

parsl-2025.11.10/parsl/concurrent/__init__.py

@@ -0,0 +1,166 @@
+"""Interfaces modeled after Python's `concurrent library <https://docs.python.org/3/library/concurrent.html>`_"""
+import time
+from concurrent.futures import Executor
+from contextlib import AbstractContextManager
+from typing import Callable, Dict, Iterable, Iterator, Literal, Optional
+from warnings import warn
+
+from parsl import Config, DataFlowKernel, load
+from parsl.app.python import PythonApp
+
+
+class ParslPoolExecutor(Executor, AbstractContextManager):
+    """An executor that uses a pool of workers managed by Parsl
+
+    Works just like a :class:`~concurrent.futures.ProcessPoolExecutor` except that tasks
+    are distributed across workers that can be on different machines.
+
+    Create a new executor using one of two methods:
+
+    1. Supplying a Parsl :class:`~parsl.Config` that defines how to create new workers.
+       The executor will start a new Parsl Data Flow Kernel (DFK) when it is entered as a context manager.
+
+    2. Supplying an already-started Parsl :class:`~parsl.DataFlowKernel` (DFK).
+       The executor assumes you will start and stop the Parsl DFK outside the Executor.
+
+    The futures returned by :meth:`submit` and :meth:`map` are Parsl futures and will work
+    with the same function chaining mechanisms as when using Parsl with decorators.
+
+    .. code-block:: python
+
+        def f(x):
+            return x + 1
+
+        @python_app
+        def parity(x):
+            return 'odd' if x % 2 == 1 else 'even'
+
+        with ParslPoolExecutor(config=my_parsl_config) as executor:
+            future_1 = executor.submit(f, 1)
+            assert parity(future_1) == 'even'  # Function chaining, as expected
+
+            future_2 = executor.submit(f, future_1)
+            assert future_2.result() == 3  # Chaining works with `submit` too
+
+    Parsl does not support canceling tasks. The :meth:`map` method does not cancel work
+    when one member of the run fails or a timeout is reached
+    and :meth:`shutdown` does not cancel work on completion.
+    """
+
+    def __init__(self, config: Config | None = None, dfk: DataFlowKernel | None = None, executors: Literal['all'] | list[str] = 'all'):
+        """Create the executor
+
+        Args:
+            config: Configuration for the Parsl Data Flow Kernel (DFK)
+            dfk: DataFlowKernel of an already-started parsl
+            executors: List of executors to use for supplied functions
+        """
+        if (config is not None) and (dfk is not None):
+            raise ValueError('Specify only one of config or dfk')
+        if (config is None) and (dfk is None):
+            raise ValueError('Must specify one of config or dfk')
+        self._config = config
+        self._app_cache: Dict[Callable, PythonApp] = {}  # Cache specific to this instance: https://stackoverflow.com/questions/33672412
+        self._dfk = dfk
+        self.executors = executors
+
+        # Start workers immediately
+        if self._config is not None:
+            self._dfk = load(self._config)
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self._dfk is None:  # Nothing has been started, do nothing
+            return
+        elif self._config is not None:  # The executors are being managed by this class, shut them down
+            self.shutdown(wait=True)
+            return
+        else:  # The DFK is managed elsewhere, do nothing
+            return
+
+    @property
+    def app_count(self):
+        """Number of functions currently registered with the executor"""
+        return len(self._app_cache)
+
+    def get_app(self, fn: Callable) -> PythonApp:
+        """Create a PythonApp for a function
+
+        Args:
+            fn: Function to be turned into a Parsl app
+        Returns:
+            PythonApp version of that function
+        """
+        if fn in self._app_cache:
+            return self._app_cache[fn]
+        app = PythonApp(fn, data_flow_kernel=self._dfk, executors=self.executors)
+        self._app_cache[fn] = app
+        return app
+
+    def submit(self, fn, *args, **kwargs):
+        """Submits a callable to be executed with the given arguments.
+
+        Schedules the callable to be executed as ``fn(*args, **kwargs)`` and returns
+        a Future instance representing the execution of the callable.
+
+        Returns:
+            A Future representing the given call.
+        """
+
+        if self._dfk is None:
+            raise RuntimeError('Executor has been shut down.')
+        app = self.get_app(fn)
+        return app(*args, **kwargs)
+
+    # TODO (wardlt): This override can go away when Parsl supports cancel
+    def map(self, fn: Callable, *iterables: Iterable, timeout: Optional[float] = None, chunksize: int = 1) -> Iterator:
+        """Returns an iterator equivalent to map(fn, iter).
+
+        Args:
+            fn: A callable that will take as many arguments as there are
+                passed iterables.
+            timeout: The maximum number of seconds to wait. If None, then there
+                is no limit on the wait time.
+            chunksize: If greater than one, the iterables will be chopped into
+                chunks of size chunksize and submitted to the process pool.
+                If set to one, the items in the list will be sent one at a time.
+
+        Returns:
+            An iterator equivalent to: map(func, ``*iterables``) but the calls may
+            be evaluated out-of-order.
+
+        Raises:
+            TimeoutError: If the entire result iterator could not be generated
+                before the given timeout.
+            Exception: If ``fn(*args)`` raises for any values.
+        """
+        # This is a version of the CPython 3.9 `.map` implementation modified to not use `cancel`
+        if timeout is not None:
+            end_time = timeout + time.monotonic()
+
+        # Submit the applications
+        app = self.get_app(fn)
+        fs = [app(*args) for args in zip(*iterables)]
+
+        # Yield the futures as completed
+        def result_iterator():
+            # reverse to keep finishing order
+            fs.reverse()
+            while fs:
+                # Careful not to keep a reference to the popped future
+                if timeout is None:
+                    yield fs.pop().result()
+                else:
+                    yield fs.pop().result(end_time - time.monotonic())
+
+        return result_iterator()
+
+    def shutdown(self, wait: bool = True, *, cancel_futures: bool = False) -> None:
+        if self._dfk is None:
+            return  # Do nothing. Nothing is active
+        if cancel_futures:
+            warn(message="Canceling on-going tasks is not supported in Parsl")
+        if wait:
+            self._dfk.wait_for_current_tasks()
+        if self._config is not None:  # The executors are being managed
+            self._dfk.cleanup()  # Shutdown the DFK
+        self._dfk = None

{parsl-2025.9.1 → parsl-2025.11.10}/parsl/curvezmq.py

@@ -101,17 +101,6 @@ class BaseContext(metaclass=ABCMeta):
         """
         self._ctx.destroy(linger)
 
-    def recreate(self, linger: Optional[int] = None):
-        """Destroy then recreate the context.
-
-        Parameters
-        ----------
-        linger : int, optional
-            If specified, set LINGER on sockets prior to closing them.
-        """
-        self.destroy(linger)
-        self._ctx = zmq.Context()
-
 
 class ServerContext(BaseContext):
     """CurveZMQ server context
@@ -175,11 +164,6 @@ class ServerContext(BaseContext):
             self.auth_thread.stop()
         super().destroy(linger)
 
-    def recreate(self, linger: Optional[int] = None):
-        super().recreate(linger)
-        if self.auth_thread:
-            self.auth_thread = self._start_auth_thread()
-
 
 class ClientContext(BaseContext):
     """CurveZMQ client context

{parsl-2025.9.1 → parsl-2025.11.10}/parsl/data_provider/globus.py

@@ -4,7 +4,6 @@ import os
 from functools import partial
 from typing import Optional
 
-import globus_sdk
 import typeguard
 
 import parsl
@@ -79,6 +78,7 @@ class Globus:
 
     @classmethod
     def transfer_file(cls, src_ep, dst_ep, src_path, dst_path):
+        import globus_sdk
         tc = globus_sdk.TransferClient(authorizer=cls.authorizer)
         td = globus_sdk.TransferData(tc, src_ep, dst_ep)
         td.add_item(src_path, dst_path)
@@ -140,6 +140,7 @@ class Globus:
     def _do_native_app_authentication(cls, client_id, redirect_uri,
                                       requested_scopes=None):
 
+        import globus_sdk
         client = globus_sdk.NativeAppAuthClient(client_id=client_id)
         client.oauth2_start_flow(
             requested_scopes=requested_scopes,
@@ -154,6 +155,7 @@ class Globus:
 
     @classmethod
     def _get_native_app_authorizer(cls, client_id):
+        import globus_sdk
         tokens = None
         try:
             tokens = cls._load_tokens_from_file(cls.TOKEN_FILE)