parsl 2025.1.13__tar.gz → 2025.1.20__tar.gz

{parsl-2025.1.13/parsl.egg-info → parsl-2025.1.20}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2025.1.13
+Version: 2025.1.20
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2025.01.13.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2025.01.20.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0
@@ -28,6 +28,7 @@ Provides-Extra: workqueue
 Provides-Extra: flux
 Provides-Extra: proxystore
 Provides-Extra: radical-pilot
+Provides-Extra: globus_compute
 Provides-Extra: all
 License-File: LICENSE
 

parsl-2025.1.20/parsl/configs/gc_multisite.py

@@ -0,0 +1,27 @@
+from globus_compute_sdk import Executor
+
+from parsl.config import Config
+from parsl.executors import GlobusComputeExecutor
+from parsl.usage_tracking.levels import LEVEL_1
+
+# Please start your own endpoint on perlmutter following instructions below to use this config:
+# https://globus-compute.readthedocs.io/en/stable/endpoints/endpoint_examples.html#perlmutter-nersc
+perlmutter_endpoint = 'YOUR_PERLMUTTER_ENDPOINT_UUID'
+
+# Please start your own endpoint on expanse following instructions below to use this config:
+# https://globus-compute.readthedocs.io/en/stable/endpoints/endpoint_examples.html#expanse-sdsc
+expanse_endpoint = 'YOUR_EXPANSE_ENDPOINT_UUID'
+
+config = Config(
+    executors=[
+        GlobusComputeExecutor(
+            executor=Executor(endpoint_id=perlmutter_endpoint),
+            label="Perlmutter",
+        ),
+        GlobusComputeExecutor(
+            executor=Executor(endpoint_id=expanse_endpoint),
+            label="Expanse",
+        ),
+    ],
+    usage_tracking=LEVEL_1,
+)

parsl-2025.1.20/parsl/configs/gc_tutorial.py

@@ -0,0 +1,18 @@
+from globus_compute_sdk import Executor
+
+from parsl.config import Config
+from parsl.executors import GlobusComputeExecutor
+from parsl.usage_tracking.levels import LEVEL_1
+
+# Public tutorial endpoint
+tutorial_endpoint = '4b116d3c-1703-4f8f-9f6f-39921e5864df'
+
+config = Config(
+    executors=[
+        GlobusComputeExecutor(
+            executor=Executor(endpoint_id=tutorial_endpoint),
+            label="Tutorial_Endpoint_py3.11",
+        )
+    ],
+    usage_tracking=LEVEL_1,
+)

{parsl-2025.1.13 → parsl-2025.1.20}/parsl/dataflow/dflow.py

@@ -484,24 +484,18 @@ class DataFlowKernel:
 
             # now we know each joinable Future is done
             # so now look for any exceptions
-            exceptions_tids: List[Tuple[BaseException, Optional[str]]]
+            exceptions_tids: List[Tuple[BaseException, str]]
             exceptions_tids = []
             if isinstance(joinable, Future):
                 je = joinable.exception()
                 if je is not None:
-                    if hasattr(joinable, 'task_record'):
-                        tid = joinable.task_record['id']
-                    else:
-                        tid = None
+                    tid = self.render_future_description(joinable)
                     exceptions_tids = [(je, tid)]
             elif isinstance(joinable, list):
                 for future in joinable:
                     je = future.exception()
                     if je is not None:
-                        if hasattr(joinable, 'task_record'):
-                            tid = joinable.task_record['id']
-                        else:
-                            tid = None
+                        tid = self.render_future_description(future)
                         exceptions_tids.append((je, tid))
             else:
                 raise TypeError(f"Unknown joinable type {type(joinable)}")
@@ -918,13 +912,7 @@ class DataFlowKernel:
         dep_failures = []
 
         def append_failure(e: Exception, dep: Future) -> None:
-            # If this Future is associated with a task inside this DFK,
-            # then refer to the task ID.
-            # Otherwise make a repr of the Future object.
-            if hasattr(dep, 'task_record') and dep.task_record['dfk'] == self:
-                tid = "task " + repr(dep.task_record['id'])
-            else:
-                tid = repr(dep)
+            tid = self.render_future_description(dep)
             dep_failures.extend([(e, tid)])
 
         # Replace item in args
@@ -1076,10 +1064,7 @@ class DataFlowKernel:
 
         depend_descs = []
         for d in depends:
-            if isinstance(d, AppFuture) or isinstance(d, DataFuture):
-                depend_descs.append("task {}".format(d.tid))
-            else:
-                depend_descs.append(repr(d))
+            depend_descs.append(self.render_future_description(d))
 
         if depend_descs != []:
             waiting_message = "waiting on {}".format(", ".join(depend_descs))
@@ -1215,10 +1200,8 @@ class DataFlowKernel:
                 self._checkpoint_timer.close()
 
         # Send final stats
-        logger.info("Sending end message for usage tracking")
         self.usage_tracker.send_end_message()
         self.usage_tracker.close()
-        logger.info("Closed usage tracking")
 
         logger.info("Closing job status poller")
         self.job_status_poller.close()
@@ -1438,6 +1421,18 @@ class DataFlowKernel:
                 '' if label is None else '_{}'.format(label),
                 kw))
 
+    def render_future_description(self, dep: Future) -> str:
+        """Renders a description of the future in the context of the
+        current DFK.
+        """
+        if isinstance(dep, AppFuture) and dep.task_record['dfk'] == self:
+            tid = "task " + repr(dep.task_record['id'])
+        elif isinstance(dep, DataFuture):
+            tid = "DataFuture from task " + repr(dep.tid)
+        else:
+            tid = repr(dep)
+        return tid
+
 
 class DataFlowKernelLoader:
     """Manage which DataFlowKernel is active.

{parsl-2025.1.13 → parsl-2025.1.20}/parsl/dataflow/taskrecord.py

@@ -43,12 +43,11 @@ class TaskRecord(TypedDict, total=False):
     executed on.
     """
 
-    retries_left: int
     fail_count: int
     fail_cost: float
     fail_history: List[str]
 
-    checkpoint: bool  # this change is also in #1516
+    checkpoint: bool
     """Should this task be checkpointed?
     """
 
@@ -68,7 +67,6 @@ class TaskRecord(TypedDict, total=False):
 
     # these three could be more strongly typed perhaps but I'm not thinking about that now
     func: Callable
-    fn_hash: str
     args: Sequence[Any]
     # in some places we uses a Tuple[Any, ...] and in some places a List[Any].
     # This is an attempt to correctly type both of those.

{parsl-2025.1.13 → parsl-2025.1.20}/parsl/executors/__init__.py

@@ -1,4 +1,5 @@
 from parsl.executors.flux.executor import FluxExecutor
+from parsl.executors.globus_compute import GlobusComputeExecutor
 from parsl.executors.high_throughput.executor import HighThroughputExecutor
 from parsl.executors.high_throughput.mpi_executor import MPIExecutor
 from parsl.executors.threads import ThreadPoolExecutor
@@ -8,4 +9,5 @@ __all__ = ['ThreadPoolExecutor',
            'HighThroughputExecutor',
            'MPIExecutor',
            'WorkQueueExecutor',
-           'FluxExecutor']
+           'FluxExecutor',
+           'GlobusComputeExecutor']

parsl-2025.1.20/parsl/executors/globus_compute.py

@@ -0,0 +1,125 @@
+from __future__ import annotations
+
+import copy
+from concurrent.futures import Future
+from typing import Any, Callable, Dict
+
+import typeguard
+
+from parsl.errors import OptionalModuleMissing
+from parsl.executors.base import ParslExecutor
+from parsl.utils import RepresentationMixin
+
+try:
+    from globus_compute_sdk import Executor
+    _globus_compute_enabled = True
+except ImportError:
+    _globus_compute_enabled = False
+
+
+class GlobusComputeExecutor(ParslExecutor, RepresentationMixin):
+    """ GlobusComputeExecutor enables remote execution on Globus Compute endpoints
+
+    GlobusComputeExecutor is a thin wrapper over globus_compute_sdk.Executor
+    Refer to `globus-compute user documentation <https://globus-compute.readthedocs.io/en/latest/executor.html>`_
+    and `reference documentation <https://globus-compute.readthedocs.io/en/latest/reference/executor.html>`_
+    for more details.
+
+    .. note::
+       As a remote execution system, Globus Compute relies on serialization to ship
+       tasks and results between the Parsl client side and the remote Globus Compute
+       Endpoint side. Serialization is unreliable across python versions, and
+       wrappers used by Parsl assume identical Parsl versions across on both sides.
+       We recommend using matching Python, Parsl and Globus Compute version on both
+       the client side and the endpoint side for stable behavior.
+
+    """
+
+    @typeguard.typechecked
+    def __init__(
+        self,
+        executor: Executor,
+        label: str = 'GlobusComputeExecutor',
+    ):
+        """
+        Parameters
+        ----------
+
+        executor: globus_compute_sdk.Executor
+            Pass a globus_compute_sdk Executor that will be used to execute
+            tasks on a globus_compute endpoint. Refer to `globus-compute docs
+            <https://globus-compute.readthedocs.io/en/latest/reference/executor.html#globus-compute-executor>`_
+
+        label:
+            a label to name the executor
+        """
+        if not _globus_compute_enabled:
+            raise OptionalModuleMissing(
+                ['globus-compute-sdk'],
+                "GlobusComputeExecutor requires globus-compute-sdk installed"
+            )
+
+        super().__init__()
+        self.executor: Executor = executor
+        self.resource_specification = self.executor.resource_specification
+        self.user_endpoint_config = self.executor.user_endpoint_config
+        self.label = label
+
+    def start(self) -> None:
+        """ Start the Globus Compute Executor """
+        pass
+
+    def submit(self, func: Callable, resource_specification: Dict[str, Any], *args: Any, **kwargs: Any) -> Future:
+        """ Submit func to globus-compute
+
+
+        Parameters
+        ----------
+
+        func: Callable
+            Python function to execute remotely
+
+        resource_specification: Dict[str, Any]
+            Resource specification can be used specify MPI resources required by MPI applications on
+            Endpoints configured to use globus compute's MPIEngine. GCE also accepts *user_endpoint_config*
+            to configure endpoints when the endpoint is a `Multi-User Endpoint
+            <https://globus-compute.readthedocs.io/en/latest/endpoints/endpoints.html#templating-endpoint-configuration>`_
+
+        args:
+            Args to pass to the function
+
+        kwargs:
+            kwargs to pass to the function
+
+        Returns
+        -------
+
+        Future
+        """
+        res_spec = copy.deepcopy(resource_specification or self.resource_specification)
+        # Pop user_endpoint_config since it is illegal in resource_spec for globus_compute
+        if res_spec:
+            user_endpoint_config = res_spec.pop('user_endpoint_config', self.user_endpoint_config)
+        else:
+            user_endpoint_config = self.user_endpoint_config
+
+        try:
+            self.executor.resource_specification = res_spec
+            self.executor.user_endpoint_config = user_endpoint_config
+            return self.executor.submit(func, *args, **kwargs)
+        finally:
+            # Reset executor state to defaults set at configuration time
+            self.executor.resource_specification = self.resource_specification
+            self.executor.user_endpoint_config = self.user_endpoint_config
+
+    def shutdown(self):
+        """Clean-up the resources associated with the Executor.
+
+        GCE.shutdown will cancel all futures that have not yet registered with
+        Globus Compute and will not wait for the launched futures to complete.
+        This method explicitly shutsdown the result_watcher thread to avoid
+        it waiting for outstanding futures at thread exit.
+        """
+        self.executor.shutdown(wait=False, cancel_futures=True)
+        result_watcher = self.executor._get_result_watcher()
+        result_watcher.shutdown(wait=False, cancel_futures=True)

{parsl-2025.1.13 → parsl-2025.1.20}/parsl/executors/high_throughput/errors.py

@@ -27,7 +27,7 @@ class VersionMismatch(Exception):
     def __str__(self) -> str:
         return (
             f"Manager version info {self.manager_version} does not match interchange"
-            f" version info {self.interchange_version}, causing a critical failure"
+            f" version info {self.interchange_version}"
         )
 
 

{parsl-2025.1.13 → parsl-2025.1.20}/parsl/executors/high_throughput/executor.py

@@ -8,7 +8,7 @@ import warnings
 from collections import defaultdict
 from concurrent.futures import Future
 from dataclasses import dataclass
-from typing import Callable, Dict, List, Optional, Sequence, Tuple, Union
+from typing import Callable, Dict, List, Optional, Sequence, Set, Tuple, Union
 
 import typeguard
 
@@ -357,10 +357,8 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
         return self.logdir
 
     def validate_resource_spec(self, resource_specification: dict):
-        """HTEX supports the following *Optional* resource specifications:
-        priority: lower value is higher priority"""
         if resource_specification:
-            acceptable_fields = {'priority'}
+            acceptable_fields: Set[str] = set()  # add new resource spec field names here to make htex accept them
             keys = set(resource_specification.keys())
             invalid_keys = keys - acceptable_fields
             if invalid_keys:

{parsl-2025.1.13 → parsl-2025.1.20}/parsl/executors/high_throughput/zmq_pipes.py

@@ -213,7 +213,6 @@ class ResultsIncoming:
         """Get a message from the queue, returning None if timeout expires
         without a message. timeout is measured in milliseconds.
         """
-        logger.debug("Waiting for ResultsIncoming message")
         socks = dict(self.poller.poll(timeout=timeout_ms))
         if self.results_receiver in socks and socks[self.results_receiver] == zmq.POLLIN:
             m = self.results_receiver.recv_multipart()

parsl-2025.1.20/parsl/tests/configs/globus_compute.py

@@ -0,0 +1,20 @@
+import os
+
+from globus_compute_sdk import Executor
+
+from parsl.config import Config
+from parsl.executors import GlobusComputeExecutor
+
+
+def fresh_config():
+
+    endpoint_id = os.environ["GLOBUS_COMPUTE_ENDPOINT"]
+
+    return Config(
+        executors=[
+            GlobusComputeExecutor(
+                executor=Executor(endpoint_id=endpoint_id),
+                label="globus_compute",
+            )
+        ]
+    )

{parsl-2025.1.13 → parsl-2025.1.20}/parsl/tests/conftest.py

@@ -163,6 +163,10 @@ def pytest_configure(config):
         'markers',
         'shared_fs: Marks tests that require a shared_fs between the workers are the test client'
     )
+    config.addinivalue_line(
+        'markers',
+        'issue_3620: Marks tests that do not work correctly on GlobusComputeExecutor (ref: issue 3620)'
+    )
 
 
 @pytest.fixture(autouse=True, scope='session')

{parsl-2025.1.13 → parsl-2025.1.20}/parsl/tests/test_error_handling/test_resource_spec.py

@@ -1,3 +1,5 @@
+import pytest
+
 import parsl
 from parsl.app.app import python_app
 from parsl.executors import WorkQueueExecutor
@@ -11,6 +13,7 @@ def double(x, parsl_resource_specification={}):
     return x * 2
 
 
+@pytest.mark.issue_3620
 def test_resource(n=2):
     executors = parsl.dfk().executors
     executor = None

parsl-2025.1.20/parsl/tests/test_htex/test_interchange_exit_bad_registration.py

@@ -0,0 +1,120 @@
+import json
+import logging
+import os
+import pickle
+import platform
+import subprocess
+import time
+
+import psutil
+import pytest
+import zmq
+
+import parsl.executors.high_throughput.zmq_pipes as zmq_pipes
+from parsl.executors.high_throughput.executor import DEFAULT_INTERCHANGE_LAUNCH_CMD
+from parsl.executors.high_throughput.manager_selector import RandomManagerSelector
+from parsl.version import VERSION as PARSL_VERSION
+
+P_ms = 10
+
+
+@pytest.mark.local
+def test_exit_with_bad_registration(tmpd_cwd, try_assert):
+    """Test that the interchange exits when it receives a bad registration message.
+    This complements parsl/tests/test_scaling/test_worker_interchange_bad_messages_3262.py
+    which tests that the interchange is resistent to other forms of bad message.
+    """
+
+    outgoing_q = zmq_pipes.TasksOutgoing(
+        "127.0.0.1", (49152, 65535), None
+    )
+    incoming_q = zmq_pipes.ResultsIncoming(
+        "127.0.0.1", (49152, 65535), None
+    )
+    command_client = zmq_pipes.CommandClient(
+        "127.0.0.1", (49152, 65535), None
+    )
+
+    interchange_config = {"client_address": "127.0.0.1",
+                          "client_ports": (outgoing_q.port,
+                                           incoming_q.port,
+                                           command_client.port),
+                          "interchange_address": "127.0.0.1",
+                          "worker_ports": None,
+                          "worker_port_range": (50000, 60000),
+                          "hub_address": None,
+                          "hub_zmq_port": None,
+                          "logdir": tmpd_cwd,
+                          "heartbeat_threshold": 120,
+                          "poll_period": P_ms,
+                          "logging_level": logging.DEBUG,
+                          "cert_dir": None,
+                          "manager_selector": RandomManagerSelector(),
+                          "run_id": "test"
+                          }
+
+    config_pickle = pickle.dumps(interchange_config)
+
+    interchange_proc = subprocess.Popen(DEFAULT_INTERCHANGE_LAUNCH_CMD, stdin=subprocess.PIPE)
+    stdin = interchange_proc.stdin
+    assert stdin is not None, "Popen should have created an IO object (vs default None) because of PIPE mode"
+
+    stdin.write(config_pickle)
+    stdin.flush()
+    stdin.close()
+
+    # wait for interchange to be alive, by waiting for the command thread to become
+    # responsive. if the interchange process didn't start enough to get the command
+    # thread running, this will time out.
+
+    (task_port, result_port) = command_client.run("WORKER_PORTS", timeout_s=120)
+
+    # now we'll assume that if the interchange command thread is responding,
+    # then the worker polling code is also running and that the interchange has
+    # started successfully.
+
+    # send bad registration message as if from a new worker pool. The badness here
+    # is that the Python version does not match the real Python version - which
+    # unlike some other bad interchange messages, should cause the interchange
+    # to shut down.
+
+    msg = {'type': 'registration',
+           'parsl_v': PARSL_VERSION,
+           'python_v': "{}.{}.{}".format(1, 1, 1),  # this is the bad bit
+           'worker_count': 1,
+           'uid': 'testuid',
+           'block_id': 0,
+           'start_time': time.time(),
+           'prefetch_capacity': 0,
+           'max_capacity': 1,
+           'os': platform.system(),
+           'hostname': platform.node(),
+           'dir': os.getcwd(),
+           'cpu_count': psutil.cpu_count(logical=False),
+           'total_memory': psutil.virtual_memory().total,
+           }
+
+    # connect to worker port and send this message.
+
+    context = zmq.Context()
+    channel_timeout = 10000  # in milliseconds
+    task_channel = context.socket(zmq.DEALER)
+    task_channel.setsockopt(zmq.LINGER, 0)
+    task_channel.setsockopt(zmq.IDENTITY, b'testid')
+
+    task_channel.set_hwm(0)
+    task_channel.setsockopt(zmq.SNDTIMEO, channel_timeout)
+    task_channel.connect(f"tcp://127.0.0.1:{task_port}")
+
+    b_msg = json.dumps(msg).encode('utf-8')
+
+    task_channel.send(b_msg)
+
+    # check that the interchange exits within some reasonable time
+    try_assert(lambda: interchange_proc.poll() is not None, "Interchange did not exit after killing watched client process", timeout_ms=5000)
+
+    # See issue #3697 - ideally the interchange would exit cleanly, but it does not.
+    # assert interchange_proc.poll() == 0, "Interchange exited with an error code, not 0"
+
+    task_channel.close()
+    context.term()

{parsl-2025.1.13 → parsl-2025.1.20}/parsl/tests/test_htex/test_resource_spec_validation.py

@@ -30,13 +30,6 @@ def test_resource_spec_validation():
     assert ret_val is None
 
 
-@pytest.mark.local
-def test_resource_spec_validation_one_key():
-    htex = HighThroughputExecutor()
-    ret_val = htex.validate_resource_spec({"priority": 2})
-    assert ret_val is None
-
-
 @pytest.mark.local
 def test_resource_spec_validation_bad_keys():
     htex = HighThroughputExecutor()

{parsl-2025.1.13 → parsl-2025.1.20}/parsl/tests/test_python_apps/test_dep_standard_futures.py

@@ -43,3 +43,6 @@ def test_future_fail_dependency():
     # Future, plain_fut, somewhere in its str
 
     assert repr(plain_fut) in str(ex)
+    assert len(ex.dependent_exceptions_tids) == 1
+    assert isinstance(ex.dependent_exceptions_tids[0][0], ValueError)
+    assert ex.dependent_exceptions_tids[0][1].startswith("<Future ")

parsl-2025.1.20/parsl/tests/test_python_apps/test_fail.py

@@ -0,0 +1,52 @@
+import pytest
+
+from parsl.app.app import python_app
+from parsl.dataflow.errors import DependencyError
+
+
+class ManufacturedTestFailure(Exception):
+    pass
+
+
+@python_app
+def random_fail(fail_prob: float, inputs=()):
+    import random
+    if random.random() < fail_prob:
+        raise ManufacturedTestFailure("App failure")
+
+
+def test_no_deps():
+    """Test basic error handling, with no dependent failures
+    """
+    futs = [random_fail(1), random_fail(0), random_fail(0)]
+
+    for f in futs:
+        try:
+            f.result()
+        except ManufacturedTestFailure:
+            pass
+
+
+def test_fail_sequence_first():
+    t1 = random_fail(fail_prob=1)
+    t2 = random_fail(fail_prob=0, inputs=[t1])
+    t_final = random_fail(fail_prob=0, inputs=[t2])
+
+    with pytest.raises(DependencyError):
+        t_final.result()
+
+    assert len(t_final.exception().dependent_exceptions_tids) == 1
+    assert isinstance(t_final.exception().dependent_exceptions_tids[0][0], DependencyError)
+    assert t_final.exception().dependent_exceptions_tids[0][1].startswith("task ")
+
+
+def test_fail_sequence_middle():
+    t1 = random_fail(fail_prob=0)
+    t2 = random_fail(fail_prob=1, inputs=[t1])
+    t_final = random_fail(fail_prob=0, inputs=[t2])
+
+    with pytest.raises(DependencyError):
+        t_final.result()
+
+    assert len(t_final.exception().dependent_exceptions_tids) == 1
+    assert isinstance(t_final.exception().dependent_exceptions_tids[0][0], ManufacturedTestFailure)

{parsl-2025.1.13 → parsl-2025.1.20}/parsl/tests/test_python_apps/test_join.py

@@ -97,7 +97,10 @@ def test_error():
     f = outer_error()
     e = f.exception()
     assert isinstance(e, JoinError)
+
+    assert len(e.dependent_exceptions_tids) == 1
     assert isinstance(e.dependent_exceptions_tids[0][0], InnerError)
+    assert e.dependent_exceptions_tids[0][1].startswith("task ")
 
 
 def test_two_errors():
@@ -109,10 +112,12 @@ def test_two_errors():
     de0 = e.dependent_exceptions_tids[0][0]
     assert isinstance(de0, InnerError)
     assert de0.args[0] == "Error A"
+    assert e.dependent_exceptions_tids[0][1].startswith("task ")
 
     de1 = e.dependent_exceptions_tids[1][0]
     assert isinstance(de1, InnerError)
     assert de1.args[0] == "Error B"
+    assert e.dependent_exceptions_tids[1][1].startswith("task ")
 
 
 def test_one_error_one_result():
@@ -125,6 +130,7 @@ def test_one_error_one_result():
     de0 = e.dependent_exceptions_tids[0][0]
     assert isinstance(de0, InnerError)
     assert de0.args[0] == "Error A"
+    assert e.dependent_exceptions_tids[0][1].startswith("task ")
 
 
 @join_app