parsl 2025.10.20__tar.gz → 2025.10.27__tar.gz

{parsl-2025.10.20/parsl.egg-info → parsl-2025.10.27}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2025.10.20
+Version: 2025.10.27
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2025.10.20.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2025.10.27.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0
@@ -28,6 +28,7 @@ Provides-Extra: flux
 Provides-Extra: proxystore
 Provides-Extra: radical-pilot
 Provides-Extra: globus_compute
+Provides-Extra: globus_transfer
 Provides-Extra: all
 License-File: LICENSE
 

parsl-2025.10.27/parsl/concurrent/__init__.py

@@ -0,0 +1,166 @@
+"""Interfaces modeled after Python's `concurrent library <https://docs.python.org/3/library/concurrent.html>`_"""
+import time
+from concurrent.futures import Executor
+from contextlib import AbstractContextManager
+from typing import Callable, Dict, Iterable, Iterator, Literal, Optional
+from warnings import warn
+
+from parsl import Config, DataFlowKernel, load
+from parsl.app.python import PythonApp
+
+
+class ParslPoolExecutor(Executor, AbstractContextManager):
+    """An executor that uses a pool of workers managed by Parsl
+
+    Works just like a :class:`~concurrent.futures.ProcessPoolExecutor` except that tasks
+    are distributed across workers that can be on different machines.
+
+    Create a new executor using one of two methods:
+
+    1. Supplying a Parsl :class:`~parsl.Config` that defines how to create new workers.
+       The executor will start a new Parsl Data Flow Kernel (DFK) when it is entered as a context manager.
+
+    2. Supplying an already-started Parsl :class:`~parsl.DataFlowKernel` (DFK).
+       The executor assumes you will start and stop the Parsl DFK outside the Executor.
+
+    The futures returned by :meth:`submit` and :meth:`map` are Parsl futures and will work
+    with the same function chaining mechanisms as when using Parsl with decorators.
+
+    .. code-block:: python
+
+        def f(x):
+            return x + 1
+
+        @python_app
+        def parity(x):
+            return 'odd' if x % 2 == 1 else 'even'
+
+        with ParslPoolExecutor(config=my_parsl_config) as executor:
+            future_1 = executor.submit(f, 1)
+            assert parity(future_1) == 'even'  # Function chaining, as expected
+
+            future_2 = executor.submit(f, future_1)
+            assert future_2.result() == 3  # Chaining works with `submit` too
+
+    Parsl does not support canceling tasks. The :meth:`map` method does not cancel work
+    when one member of the run fails or a timeout is reached
+    and :meth:`shutdown` does not cancel work on completion.
+    """
+
+    def __init__(self, config: Config | None = None, dfk: DataFlowKernel | None = None, executors: Literal['all'] | list[str] = 'all'):
+        """Create the executor
+
+        Args:
+            config: Configuration for the Parsl Data Flow Kernel (DFK)
+            dfk: DataFlowKernel of an already-started parsl
+            executors: List of executors to use for supplied functions
+        """
+        if (config is not None) and (dfk is not None):
+            raise ValueError('Specify only one of config or dfk')
+        if (config is None) and (dfk is None):
+            raise ValueError('Must specify one of config or dfk')
+        self._config = config
+        self._app_cache: Dict[Callable, PythonApp] = {}  # Cache specific to this instance: https://stackoverflow.com/questions/33672412
+        self._dfk = dfk
+        self.executors = executors
+
+        # Start workers immediately
+        if self._config is not None:
+            self._dfk = load(self._config)
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self._dfk is None:  # Nothing has been started, do nothing
+            return
+        elif self._config is not None:  # The executors are being managed by this class, shut them down
+            self.shutdown(wait=True)
+            return
+        else:  # The DFK is managed elsewhere, do nothing
+            return
+
+    @property
+    def app_count(self):
+        """Number of functions currently registered with the executor"""
+        return len(self._app_cache)
+
+    def get_app(self, fn: Callable) -> PythonApp:
+        """Create a PythonApp for a function
+
+        Args:
+            fn: Function to be turned into a Parsl app
+        Returns:
+            PythonApp version of that function
+        """
+        if fn in self._app_cache:
+            return self._app_cache[fn]
+        app = PythonApp(fn, data_flow_kernel=self._dfk, executors=self.executors)
+        self._app_cache[fn] = app
+        return app
+
+    def submit(self, fn, *args, **kwargs):
+        """Submits a callable to be executed with the given arguments.
+
+        Schedules the callable to be executed as ``fn(*args, **kwargs)`` and returns
+        a Future instance representing the execution of the callable.
+
+        Returns:
+            A Future representing the given call.
+        """
+
+        if self._dfk is None:
+            raise RuntimeError('Executor has been shut down.')
+        app = self.get_app(fn)
+        return app(*args, **kwargs)
+
+    # TODO (wardlt): This override can go away when Parsl supports cancel
+    def map(self, fn: Callable, *iterables: Iterable, timeout: Optional[float] = None, chunksize: int = 1) -> Iterator:
+        """Returns an iterator equivalent to map(fn, iter).
+
+        Args:
+            fn: A callable that will take as many arguments as there are
+                passed iterables.
+            timeout: The maximum number of seconds to wait. If None, then there
+                is no limit on the wait time.
+            chunksize: If greater than one, the iterables will be chopped into
+                chunks of size chunksize and submitted to the process pool.
+                If set to one, the items in the list will be sent one at a time.
+
+        Returns:
+            An iterator equivalent to: map(func, ``*iterables``) but the calls may
+            be evaluated out-of-order.
+
+        Raises:
+            TimeoutError: If the entire result iterator could not be generated
+                before the given timeout.
+            Exception: If ``fn(*args)`` raises for any values.
+        """
+        # This is a version of the CPython 3.9 `.map` implementation modified to not use `cancel`
+        if timeout is not None:
+            end_time = timeout + time.monotonic()
+
+        # Submit the applications
+        app = self.get_app(fn)
+        fs = [app(*args) for args in zip(*iterables)]
+
+        # Yield the futures as completed
+        def result_iterator():
+            # reverse to keep finishing order
+            fs.reverse()
+            while fs:
+                # Careful not to keep a reference to the popped future
+                if timeout is None:
+                    yield fs.pop().result()
+                else:
+                    yield fs.pop().result(end_time - time.monotonic())
+
+        return result_iterator()
+
+    def shutdown(self, wait: bool = True, *, cancel_futures: bool = False) -> None:
+        if self._dfk is None:
+            return  # Do nothing. Nothing is active
+        if cancel_futures:
+            warn(message="Canceling on-going tasks is not supported in Parsl")
+        if wait:
+            self._dfk.wait_for_current_tasks()
+        if self._config is not None:  # The executors are being managed
+            self._dfk.cleanup()  # Shutdown the DFK
+        self._dfk = None

{parsl-2025.10.20 → parsl-2025.10.27}/parsl/data_provider/globus.py

@@ -4,7 +4,6 @@ import os
 from functools import partial
 from typing import Optional
 
-import globus_sdk
 import typeguard
 
 import parsl
@@ -79,6 +78,7 @@ class Globus:
 
     @classmethod
     def transfer_file(cls, src_ep, dst_ep, src_path, dst_path):
+        import globus_sdk
         tc = globus_sdk.TransferClient(authorizer=cls.authorizer)
         td = globus_sdk.TransferData(tc, src_ep, dst_ep)
         td.add_item(src_path, dst_path)
@@ -140,6 +140,7 @@ class Globus:
     def _do_native_app_authentication(cls, client_id, redirect_uri,
                                       requested_scopes=None):
 
+        import globus_sdk
         client = globus_sdk.NativeAppAuthClient(client_id=client_id)
         client.oauth2_start_flow(
             requested_scopes=requested_scopes,
@@ -154,6 +155,7 @@ class Globus:
 
     @classmethod
     def _get_native_app_authorizer(cls, client_id):
+        import globus_sdk
         tokens = None
         try:
             tokens = cls._load_tokens_from_file(cls.TOKEN_FILE)

{parsl-2025.10.20 → parsl-2025.10.27}/parsl/dataflow/dflow.py

@@ -190,7 +190,7 @@ class DataFlowKernel:
         self.tasks: Dict[int, TaskRecord] = {}
         self.submitter_lock = threading.Lock()
 
-        self.dependency_launch_pool = cf.ThreadPoolExecutor(max_workers=1, thread_name_prefix="Dependency-Launch")
+        self._task_launch_pool = cf.ThreadPoolExecutor(max_workers=1, thread_name_prefix="Task-Launch")
 
         self.dependency_resolver = self.config.dependency_resolver if self.config.dependency_resolver is not None \
             else SHALLOW_DEPENDENCY_RESOLVER
@@ -608,7 +608,7 @@ class DataFlowKernel:
         launch_if_ready is thread safe, so may be called from any thread
         or callback.
         """
-        self.dependency_launch_pool.submit(self._launch_if_ready_async, task_record)
+        self._task_launch_pool.submit(self._launch_if_ready_async, task_record)
 
     @wrap_with_logs
     def _launch_if_ready_async(self, task_record: TaskRecord) -> None:
@@ -1203,9 +1203,9 @@ class DataFlowKernel:
             self.monitoring.close()
             logger.info("Terminated monitoring")
 
-        logger.info("Terminating dependency launch pool")
-        self.dependency_launch_pool.shutdown()
-        logger.info("Terminated dependency launch pool")
+        logger.info("Terminating task launch pool")
+        self._task_launch_pool.shutdown()
+        logger.info("Terminated task launch pool")
 
         logger.info("Unregistering atexit hook")
         atexit.unregister(self.atexit_cleanup)

{parsl-2025.10.20 → parsl-2025.10.27}/parsl/executors/high_throughput/interchange.py

@@ -138,9 +138,6 @@ class Interchange:
 
         self.pending_task_queue: SortedList[Any] = SortedList(key=lambda tup: (tup[0], tup[1]))
 
-        # count of tasks that have been received from the submit side
-        self.task_counter = 0
-
         # count of tasks that have been sent out to worker pools
         self.count = 0
 
@@ -332,15 +329,15 @@ class Interchange:
             msg = self.task_incoming.recv_pyobj()
 
             # Process priority, higher number = lower priority
+            task_id = msg['task_id']
             resource_spec = msg['context'].get('resource_spec', {})
             priority = resource_spec.get('priority', float('inf'))
-            queue_entry = (-priority, -self.task_counter, msg)
+            queue_entry = (-priority, -task_id, msg)
 
-            logger.debug("putting message onto pending_task_queue")
+            logger.debug("Putting task %s onto pending_task_queue", task_id)
 
             self.pending_task_queue.add(queue_entry)
-            self.task_counter += 1
-            logger.debug(f"Fetched {self.task_counter} tasks so far")
+            logger.debug("Put task %s onto pending_task_queue", task_id)
 
     def process_manager_socket_message(
         self,

{parsl-2025.10.20 → parsl-2025.10.27}/parsl/tests/configs/local_threads_globus.py

@@ -1,6 +1,5 @@
 from parsl.config import Config
 from parsl.data_provider.data_manager import default_staging
-from parsl.data_provider.globus import GlobusStaging
 from parsl.executors.threads import ThreadPoolExecutor
 
 # If you are a developer running tests, make sure to update parsl/tests/configs/user_opts.py
@@ -10,19 +9,24 @@ from parsl.executors.threads import ThreadPoolExecutor
 #          (i.e., user_opts['swan']['username'] -> 'your_username')
 from .user_opts import user_opts
 
-storage_access = default_staging + [GlobusStaging(
-                endpoint_uuid=user_opts['globus']['endpoint'],
-                endpoint_path=user_opts['globus']['path']
-            )]
 
-config = Config(
-    executors=[
-        ThreadPoolExecutor(
-            label='local_threads_globus',
-            working_dir=user_opts['globus']['path'],
-            storage_access=storage_access
-        )
-    ]
-)
+def fresh_config():
+    from parsl.data_provider.globus import GlobusStaging
+
+    storage_access = default_staging + [GlobusStaging(
+                    endpoint_uuid=user_opts['globus']['endpoint'],
+                    endpoint_path=user_opts['globus']['path']
+                )]
+
+    return Config(
+        executors=[
+            ThreadPoolExecutor(
+                label='local_threads_globus',
+                working_dir=user_opts['globus']['path'],
+                storage_access=storage_access
+            )
+        ]
+    )
+
 
 remote_writeable = user_opts['globus']['remote_writeable']

parsl-2025.10.27/parsl/tests/sites/test_concurrent.py

@@ -0,0 +1,96 @@
+"""Tests of the interfaces to Python's concurrent library"""
+from pytest import mark, raises, warns
+
+from parsl import Config, HighThroughputExecutor, load, python_app
+from parsl.concurrent import ParslPoolExecutor
+
+
+def f(x):
+    return x + 1
+
+
+def g(x):
+    return 2 * x
+
+
+@python_app
+def is_odd(x):
+    if x % 2 == 1:
+        return 1
+    else:
+        return 0
+
+
+def make_config():
+    return Config(
+        executors=[
+            HighThroughputExecutor(
+                label='test_executor',
+                address="127.0.0.1",
+                max_workers_per_node=2,
+                heartbeat_period=2,
+                heartbeat_threshold=4,
+                encrypted=False,
+            )
+        ],
+        strategy='none',
+    )
+
+
+@mark.local
+def test_init_errors():
+    with load(make_config()) as dfk, raises(ValueError, match='Specify only one of config or dfk'):
+        ParslPoolExecutor(config=make_config(), dfk=dfk)
+
+    with raises(ValueError, match='Must specify one of config or dfk'):
+        ParslPoolExecutor()
+
+
+@mark.local
+def test_executor():
+    my_config = make_config()
+
+    with ParslPoolExecutor(my_config) as exc:
+        # Test a single submit
+        future = exc.submit(f, 1)
+        assert future.result() == 2
+
+        # Make sure the map works
+        results = list(exc.map(f, [1, 2, 3]))
+        assert results == [2, 3, 4]
+
+        # Make sure map works with a timeout
+        results = list(exc.map(f, [1, 2, 3], timeout=5))
+        assert results == [2, 3, 4]
+
+        # Make sure only one function was registered
+        assert exc.app_count == 1
+
+    with raises(RuntimeError, match='shut down'):
+        exc.submit(f, 1)
+
+    with warns(UserWarning):
+        ParslPoolExecutor(make_config()).shutdown(False, cancel_futures=True)
+
+
+@mark.local
+def test_with_dfk():
+    config = make_config()
+
+    with load(config) as dfk, ParslPoolExecutor(dfk=dfk, executors=['test_executor']) as exc:
+        future = exc.submit(f, 1)
+        assert future.result() == 2
+        assert exc.get_app(f).executors == ['test_executor']
+
+
+@mark.local
+def test_chaining():
+    """Make sure the executor functions can be chained together"""
+    config = make_config()
+
+    with ParslPoolExecutor(config) as exc:
+        future_odd = exc.submit(f, 10)
+        assert is_odd(future_odd).result()
+
+        future_even = exc.submit(g, future_odd)
+        assert not is_odd(future_even).result()

{parsl-2025.10.20 → parsl-2025.10.27}/parsl/tests/test_htex/test_priority_queue.py

@@ -61,7 +61,7 @@ def test_priority_queue(try_assert):
             with open(htex.worker_logdir + "/interchange.log", "r") as f:
                 lines = f.readlines()
                 for line in lines:
-                    if f"Fetched {n} tasks so far" in line:
+                    if f"Put task {n} onto pending_task_queue" in line:
                         return True
             return False
 

{parsl-2025.10.20 → parsl-2025.10.27}/parsl/tests/test_python_apps/test_basic.py

@@ -14,12 +14,6 @@ def import_square(x):
     return math.pow(x, 2)
 
 
-@python_app
-def custom_exception():
-    from globus_sdk import GlobusError
-    raise GlobusError('foobar')
-
-
 def test_simple(n=2):
     x = double(n)
     assert x.result() == n * 2
@@ -38,11 +32,3 @@ def test_parallel_for(n):
 
     for i in d:
         assert d[i].result() == 2 * i
-
-
-def test_custom_exception():
-    from globus_sdk import GlobusError
-
-    x = custom_exception()
-    with pytest.raises(GlobusError):
-        x.result()

parsl-2025.10.27/parsl/tests/test_python_apps/test_exception.py

@@ -0,0 +1,19 @@
+import pytest
+
+from parsl.app.app import python_app
+
+
+class CustomException(Exception):
+    pass
+
+
+@python_app
+def custom_exception():
+    from parsl.tests.test_python_apps.test_exception import CustomException
+    raise CustomException('foobar')
+
+
+def test_custom_exception():
+    x = custom_exception()
+    with pytest.raises(CustomException):
+        x.result()

{parsl-2025.10.20 → parsl-2025.10.27}/parsl/tests/test_staging/test_staging_globus.py

@@ -3,9 +3,9 @@ import pytest
 import parsl
 from parsl.app.app import python_app
 from parsl.data_provider.files import File
-from parsl.tests.configs.local_threads_globus import config, remote_writeable
+from parsl.tests.configs.local_threads_globus import fresh_config, remote_writeable
 
-local_config = config
+local_config = fresh_config
 
 
 @python_app

{parsl-2025.10.20 → parsl-2025.10.27}/parsl/tests/unit/test_globus_compute_executor.py

@@ -2,18 +2,21 @@ import random
 from unittest import mock
 
 import pytest
-from globus_compute_sdk import Executor
 
 from parsl.executors import GlobusComputeExecutor
 
 
 @pytest.fixture
 def mock_ex():
-    # Not Parsl's job to test GC's Executor
+    # Not Parsl's job to test GC's Executor, although it
+    # still needs to be importable for these test cases.
+    from globus_compute_sdk import Executor
+
     yield mock.Mock(spec=Executor)
 
 
 @pytest.mark.local
+@pytest.mark.globus_compute
 def test_gc_executor_mock_spec(mock_ex):
     # a test of tests -- make sure we're using spec= in the mock
     with pytest.raises(AttributeError):
@@ -21,12 +24,14 @@ def test_gc_executor_mock_spec(mock_ex):
 
 
 @pytest.mark.local
+@pytest.mark.globus_compute
 def test_gc_executor_label_default(mock_ex):
     gce = GlobusComputeExecutor(mock_ex)
     assert gce.label == type(gce).__name__, "Expect reasonable default label"
 
 
 @pytest.mark.local
+@pytest.mark.globus_compute
 def test_gc_executor_label(mock_ex, randomstring):
     exp_label = randomstring()
     gce = GlobusComputeExecutor(mock_ex, label=exp_label)
@@ -34,6 +39,7 @@ def test_gc_executor_label(mock_ex, randomstring):
 
 
 @pytest.mark.local
+@pytest.mark.globus_compute
 def test_gc_executor_resets_spec_after_submit(mock_ex, randomstring):
     submit_res = {randomstring(): "some submit res"}
     res = {"some": randomstring(), "spec": randomstring()}
@@ -57,6 +63,7 @@ def test_gc_executor_resets_spec_after_submit(mock_ex, randomstring):
 
 
 @pytest.mark.local
+@pytest.mark.globus_compute
 def test_gc_executor_resets_uep_after_submit(mock_ex, randomstring):
     uep_conf = randomstring()
     res = {"some": randomstring()}
@@ -79,6 +86,7 @@ def test_gc_executor_resets_uep_after_submit(mock_ex, randomstring):
 
 
 @pytest.mark.local
+@pytest.mark.globus_compute
 def test_gc_executor_happy_path(mock_ex, randomstring):
     mock_fn = mock.Mock()
     args = tuple(randomstring() for _ in range(random.randint(0, 3)))
@@ -95,6 +103,7 @@ def test_gc_executor_happy_path(mock_ex, randomstring):
 
 
 @pytest.mark.local
+@pytest.mark.globus_compute
 def test_gc_executor_shuts_down_asynchronously(mock_ex):
     gce = GlobusComputeExecutor(mock_ex)
     gce.shutdown()

{parsl-2025.10.20 → parsl-2025.10.27}/parsl/version.py

@@ -3,4 +3,4 @@
 Year.Month.Day[alpha/beta/..]
 Alphas will be numbered like this -> 2024.12.10a0
 """
-VERSION = '2025.10.20'
+VERSION = '2025.10.27'

{parsl-2025.10.20 → parsl-2025.10.27/parsl.egg-info}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2025.10.20
+Version: 2025.10.27
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2025.10.20.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2025.10.27.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0
@@ -28,6 +28,7 @@ Provides-Extra: flux
 Provides-Extra: proxystore
 Provides-Extra: radical-pilot
 Provides-Extra: globus_compute
+Provides-Extra: globus_transfer
 Provides-Extra: all
 License-File: LICENSE
 

{parsl-2025.10.20 → parsl-2025.10.27}/parsl.egg-info/SOURCES.txt

@@ -370,6 +370,7 @@ parsl/tests/test_python_apps/test_dep_standard_futures.py
 parsl/tests/test_python_apps/test_dependencies.py
 parsl/tests/test_python_apps/test_dependencies_deep.py
 parsl/tests/test_python_apps/test_depfail_propagation.py
+parsl/tests/test_python_apps/test_exception.py
 parsl/tests/test_python_apps/test_fail.py
 parsl/tests/test_python_apps/test_fibonacci_iterative.py
 parsl/tests/test_python_apps/test_fibonacci_recursive.py

{parsl-2025.10.20 → parsl-2025.10.27}/parsl.egg-info/requires.txt

@@ -1,7 +1,6 @@
 pyzmq>=17.1.2
 typeguard!=3.*,<5,>=2.10
 typing-extensions<5,>=4.6
-globus-sdk
 dill
 tblib
 requests
@@ -38,6 +37,7 @@ proxystore
 radical.pilot==1.90
 radical.utils==1.90
 globus_compute_sdk>=2.34.0
+globus-sdk
 
 [aws]
 boto3
@@ -60,6 +60,9 @@ jsonschema
 [globus_compute]
 globus_compute_sdk>=2.34.0
 
+[globus_transfer]
+globus-sdk
+
 [google_cloud]
 google-auth
 google-api-python-client

{parsl-2025.10.20 → parsl-2025.10.27}/requirements.txt

@@ -8,7 +8,6 @@ typeguard>=2.10,!=3.*,<5
 
 
 typing-extensions>=4.6,<5
-globus-sdk
 dill
 tblib
 requests

{parsl-2025.10.20 → parsl-2025.10.27}/setup.py

@@ -46,6 +46,7 @@ extras_require = {
     'proxystore': ['proxystore'],
     'radical-pilot': ['radical.pilot==1.90', 'radical.utils==1.90'],
     'globus_compute': ['globus_compute_sdk>=2.34.0'],
+    'globus_transfer': ['globus-sdk'],
     # Disabling psi-j since github direct links are not allowed by pypi
     # 'psij': ['psi-j-parsl@git+https://github.com/ExaWorks/psi-j-parsl']
 }