parsl 2025.1.6__tar.gz → 2025.1.20__tar.gz

{parsl-2025.1.6/parsl.egg-info → parsl-2025.1.20}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2025.1.6
+Version: 2025.1.20
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2025.01.06.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2025.01.20.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0
@@ -28,6 +28,7 @@ Provides-Extra: workqueue
 Provides-Extra: flux
 Provides-Extra: proxystore
 Provides-Extra: radical-pilot
+Provides-Extra: globus_compute
 Provides-Extra: all
 License-File: LICENSE
 

{parsl-2025.1.6 → parsl-2025.1.20}/parsl/app/errors.py

@@ -70,8 +70,8 @@ class MissingOutputs(ParslError):
         self.reason = reason
         self.outputs = outputs
 
-    def __repr__(self) -> str:
-        return "Missing Outputs: {0}, Reason:{1}".format(self.outputs, self.reason)
+    def __str__(self) -> str:
+        return "Missing Outputs: {0}, Reason: {1}".format(self.outputs, self.reason)
 
 
 class BadStdStreamFile(ParslError):
@@ -85,11 +85,8 @@ class BadStdStreamFile(ParslError):
         super().__init__(reason)
         self._reason = reason
 
-    def __repr__(self) -> str:
-        return "Bad Stream File: {}".format(self._reason)
-
     def __str__(self) -> str:
-        return self.__repr__()
+        return "Bad Stream File: {}".format(self._reason)
 
 
 class RemoteExceptionWrapper:

{parsl-2025.1.6 → parsl-2025.1.20}/parsl/app/futures.py

@@ -2,7 +2,6 @@
 """
 import logging
 from concurrent.futures import Future
-from typing import Optional
 
 import typeguard
 
@@ -39,24 +38,23 @@ class DataFuture(Future):
             self.set_result(self.file_obj)
 
     @typeguard.typechecked
-    def __init__(self, fut: Future, file_obj: File, tid: Optional[int] = None) -> None:
+    def __init__(self, fut: Future, file_obj: File, tid: int) -> None:
         """Construct the DataFuture object.
 
         If the file_obj is a string convert to a File.
 
         Args:
-            - fut (AppFuture) : AppFuture that this DataFuture will track
-            - file_obj (string/File obj) : Something representing file(s)
+            - fut (Future) : Future that this DataFuture will track.
+                             Completion of ``fut`` indicates that the data is
+                             ready.
+            - file_obj (File) : File that this DataFuture represents the availability of
 
         Kwargs:
             - tid (task_id) : Task id that this DataFuture tracks
         """
         super().__init__()
         self._tid = tid
-        if isinstance(file_obj, File):
-            self.file_obj = file_obj
-        else:
-            raise ValueError("DataFuture must be initialized with a File, not {}".format(type(file_obj)))
+        self.file_obj = file_obj
         self.parent = fut
 
         self.parent.add_done_callback(self.parent_callback)

parsl-2025.1.20/parsl/configs/gc_multisite.py

@@ -0,0 +1,27 @@
+from globus_compute_sdk import Executor
+
+from parsl.config import Config
+from parsl.executors import GlobusComputeExecutor
+from parsl.usage_tracking.levels import LEVEL_1
+
+# Please start your own endpoint on perlmutter following instructions below to use this config:
+# https://globus-compute.readthedocs.io/en/stable/endpoints/endpoint_examples.html#perlmutter-nersc
+perlmutter_endpoint = 'YOUR_PERLMUTTER_ENDPOINT_UUID'
+
+# Please start your own endpoint on expanse following instructions below to use this config:
+# https://globus-compute.readthedocs.io/en/stable/endpoints/endpoint_examples.html#expanse-sdsc
+expanse_endpoint = 'YOUR_EXPANSE_ENDPOINT_UUID'
+
+config = Config(
+    executors=[
+        GlobusComputeExecutor(
+            executor=Executor(endpoint_id=perlmutter_endpoint),
+            label="Perlmutter",
+        ),
+        GlobusComputeExecutor(
+            executor=Executor(endpoint_id=expanse_endpoint),
+            label="Expanse",
+        ),
+    ],
+    usage_tracking=LEVEL_1,
+)

parsl-2025.1.20/parsl/configs/gc_tutorial.py

@@ -0,0 +1,18 @@
+from globus_compute_sdk import Executor
+
+from parsl.config import Config
+from parsl.executors import GlobusComputeExecutor
+from parsl.usage_tracking.levels import LEVEL_1
+
+# Public tutorial endpoint
+tutorial_endpoint = '4b116d3c-1703-4f8f-9f6f-39921e5864df'
+
+config = Config(
+    executors=[
+        GlobusComputeExecutor(
+            executor=Executor(endpoint_id=tutorial_endpoint),
+            label="Tutorial_Endpoint_py3.11",
+        )
+    ],
+    usage_tracking=LEVEL_1,
+)

{parsl-2025.1.6 → parsl-2025.1.20}/parsl/dataflow/dflow.py

@@ -484,24 +484,18 @@ class DataFlowKernel:
 
             # now we know each joinable Future is done
             # so now look for any exceptions
-            exceptions_tids: List[Tuple[BaseException, Optional[str]]]
+            exceptions_tids: List[Tuple[BaseException, str]]
             exceptions_tids = []
             if isinstance(joinable, Future):
                 je = joinable.exception()
                 if je is not None:
-                    if hasattr(joinable, 'task_record'):
-                        tid = joinable.task_record['id']
-                    else:
-                        tid = None
+                    tid = self.render_future_description(joinable)
                     exceptions_tids = [(je, tid)]
             elif isinstance(joinable, list):
                 for future in joinable:
                     je = future.exception()
                     if je is not None:
-                        if hasattr(joinable, 'task_record'):
-                            tid = joinable.task_record['id']
-                        else:
-                            tid = None
+                        tid = self.render_future_description(future)
                         exceptions_tids.append((je, tid))
             else:
                 raise TypeError(f"Unknown joinable type {type(joinable)}")
@@ -918,13 +912,7 @@ class DataFlowKernel:
         dep_failures = []
 
         def append_failure(e: Exception, dep: Future) -> None:
-            # If this Future is associated with a task inside this DFK,
-            # then refer to the task ID.
-            # Otherwise make a repr of the Future object.
-            if hasattr(dep, 'task_record') and dep.task_record['dfk'] == self:
-                tid = "task " + repr(dep.task_record['id'])
-            else:
-                tid = repr(dep)
+            tid = self.render_future_description(dep)
             dep_failures.extend([(e, tid)])
 
         # Replace item in args
@@ -1076,10 +1064,7 @@ class DataFlowKernel:
 
         depend_descs = []
         for d in depends:
-            if isinstance(d, AppFuture) or isinstance(d, DataFuture):
-                depend_descs.append("task {}".format(d.tid))
-            else:
-                depend_descs.append(repr(d))
+            depend_descs.append(self.render_future_description(d))
 
         if depend_descs != []:
             waiting_message = "waiting on {}".format(", ".join(depend_descs))
@@ -1215,10 +1200,8 @@ class DataFlowKernel:
                 self._checkpoint_timer.close()
 
         # Send final stats
-        logger.info("Sending end message for usage tracking")
         self.usage_tracker.send_end_message()
         self.usage_tracker.close()
-        logger.info("Closed usage tracking")
 
         logger.info("Closing job status poller")
         self.job_status_poller.close()
@@ -1438,6 +1421,18 @@ class DataFlowKernel:
                 '' if label is None else '_{}'.format(label),
                 kw))
 
+    def render_future_description(self, dep: Future) -> str:
+        """Renders a description of the future in the context of the
+        current DFK.
+        """
+        if isinstance(dep, AppFuture) and dep.task_record['dfk'] == self:
+            tid = "task " + repr(dep.task_record['id'])
+        elif isinstance(dep, DataFuture):
+            tid = "DataFuture from task " + repr(dep.tid)
+        else:
+            tid = repr(dep)
+        return tid
+
 
 class DataFlowKernelLoader:
     """Manage which DataFlowKernel is active.

{parsl-2025.1.6 → parsl-2025.1.20}/parsl/dataflow/errors.py

@@ -25,11 +25,8 @@ class BadCheckpoint(DataFlowException):
     def __init__(self, reason: str) -> None:
         self.reason = reason
 
-    def __repr__(self) -> str:
-        return self.reason
-
     def __str__(self) -> str:
-        return self.__repr__()
+        return self.reason
 
 
 class DependencyError(DataFlowException):

{parsl-2025.1.6 → parsl-2025.1.20}/parsl/dataflow/rundirs.py

@@ -6,17 +6,16 @@ logger = logging.getLogger(__name__)
 
 
 def make_rundir(path: str) -> str:
-    """When a path has not been specified, make the run directory.
+    """Create a numbered run directory under the specified path.
 
-    Creates a rundir with the following hierarchy:
-        ./runinfo <- Home of all run directories
+        ./runinfo <- specified path
           |----000
           |----001 <- Directories for each run
           | ....
           |----NNN
 
-    Kwargs:
-        - path (str): String path to a specific run dir
+    Args:
+        - path (str): String path to root of all rundirs
     """
     try:
         if not os.path.exists(path):

{parsl-2025.1.6 → parsl-2025.1.20}/parsl/dataflow/taskrecord.py

@@ -43,12 +43,11 @@ class TaskRecord(TypedDict, total=False):
     executed on.
     """
 
-    retries_left: int
     fail_count: int
     fail_cost: float
     fail_history: List[str]
 
-    checkpoint: bool  # this change is also in #1516
+    checkpoint: bool
     """Should this task be checkpointed?
     """
 
@@ -68,7 +67,6 @@ class TaskRecord(TypedDict, total=False):
 
     # these three could be more strongly typed perhaps but I'm not thinking about that now
     func: Callable
-    fn_hash: str
     args: Sequence[Any]
     # in some places we uses a Tuple[Any, ...] and in some places a List[Any].
     # This is an attempt to correctly type both of those.

{parsl-2025.1.6 → parsl-2025.1.20}/parsl/executors/__init__.py

@@ -1,4 +1,5 @@
 from parsl.executors.flux.executor import FluxExecutor
+from parsl.executors.globus_compute import GlobusComputeExecutor
 from parsl.executors.high_throughput.executor import HighThroughputExecutor
 from parsl.executors.high_throughput.mpi_executor import MPIExecutor
 from parsl.executors.threads import ThreadPoolExecutor
@@ -8,4 +9,5 @@ __all__ = ['ThreadPoolExecutor',
            'HighThroughputExecutor',
            'MPIExecutor',
            'WorkQueueExecutor',
-           'FluxExecutor']
+           'FluxExecutor',
+           'GlobusComputeExecutor']

parsl-2025.1.20/parsl/executors/globus_compute.py

@@ -0,0 +1,125 @@
+from __future__ import annotations
+
+import copy
+from concurrent.futures import Future
+from typing import Any, Callable, Dict
+
+import typeguard
+
+from parsl.errors import OptionalModuleMissing
+from parsl.executors.base import ParslExecutor
+from parsl.utils import RepresentationMixin
+
+try:
+    from globus_compute_sdk import Executor
+    _globus_compute_enabled = True
+except ImportError:
+    _globus_compute_enabled = False
+
+
+class GlobusComputeExecutor(ParslExecutor, RepresentationMixin):
+    """ GlobusComputeExecutor enables remote execution on Globus Compute endpoints
+
+    GlobusComputeExecutor is a thin wrapper over globus_compute_sdk.Executor
+    Refer to `globus-compute user documentation <https://globus-compute.readthedocs.io/en/latest/executor.html>`_
+    and `reference documentation <https://globus-compute.readthedocs.io/en/latest/reference/executor.html>`_
+    for more details.
+
+    .. note::
+       As a remote execution system, Globus Compute relies on serialization to ship
+       tasks and results between the Parsl client side and the remote Globus Compute
+       Endpoint side. Serialization is unreliable across python versions, and
+       wrappers used by Parsl assume identical Parsl versions across on both sides.
+       We recommend using matching Python, Parsl and Globus Compute version on both
+       the client side and the endpoint side for stable behavior.
+
+    """
+
+    @typeguard.typechecked
+    def __init__(
+        self,
+        executor: Executor,
+        label: str = 'GlobusComputeExecutor',
+    ):
+        """
+        Parameters
+        ----------
+
+        executor: globus_compute_sdk.Executor
+            Pass a globus_compute_sdk Executor that will be used to execute
+            tasks on a globus_compute endpoint. Refer to `globus-compute docs
+            <https://globus-compute.readthedocs.io/en/latest/reference/executor.html#globus-compute-executor>`_
+
+        label:
+            a label to name the executor
+        """
+        if not _globus_compute_enabled:
+            raise OptionalModuleMissing(
+                ['globus-compute-sdk'],
+                "GlobusComputeExecutor requires globus-compute-sdk installed"
+            )
+
+        super().__init__()
+        self.executor: Executor = executor
+        self.resource_specification = self.executor.resource_specification
+        self.user_endpoint_config = self.executor.user_endpoint_config
+        self.label = label
+
+    def start(self) -> None:
+        """ Start the Globus Compute Executor """
+        pass
+
+    def submit(self, func: Callable, resource_specification: Dict[str, Any], *args: Any, **kwargs: Any) -> Future:
+        """ Submit func to globus-compute
+
+
+        Parameters
+        ----------
+
+        func: Callable
+            Python function to execute remotely
+
+        resource_specification: Dict[str, Any]
+            Resource specification can be used specify MPI resources required by MPI applications on
+            Endpoints configured to use globus compute's MPIEngine. GCE also accepts *user_endpoint_config*
+            to configure endpoints when the endpoint is a `Multi-User Endpoint
+            <https://globus-compute.readthedocs.io/en/latest/endpoints/endpoints.html#templating-endpoint-configuration>`_
+
+        args:
+            Args to pass to the function
+
+        kwargs:
+            kwargs to pass to the function
+
+        Returns
+        -------
+
+        Future
+        """
+        res_spec = copy.deepcopy(resource_specification or self.resource_specification)
+        # Pop user_endpoint_config since it is illegal in resource_spec for globus_compute
+        if res_spec:
+            user_endpoint_config = res_spec.pop('user_endpoint_config', self.user_endpoint_config)
+        else:
+            user_endpoint_config = self.user_endpoint_config
+
+        try:
+            self.executor.resource_specification = res_spec
+            self.executor.user_endpoint_config = user_endpoint_config
+            return self.executor.submit(func, *args, **kwargs)
+        finally:
+            # Reset executor state to defaults set at configuration time
+            self.executor.resource_specification = self.resource_specification
+            self.executor.user_endpoint_config = self.user_endpoint_config
+
+    def shutdown(self):
+        """Clean-up the resources associated with the Executor.
+
+        GCE.shutdown will cancel all futures that have not yet registered with
+        Globus Compute and will not wait for the launched futures to complete.
+        This method explicitly shutsdown the result_watcher thread to avoid
+        it waiting for outstanding futures at thread exit.
+        """
+        self.executor.shutdown(wait=False, cancel_futures=True)
+        result_watcher = self.executor._get_result_watcher()
+        result_watcher.shutdown(wait=False, cancel_futures=True)

{parsl-2025.1.6 → parsl-2025.1.20}/parsl/executors/high_throughput/errors.py

@@ -27,7 +27,7 @@ class VersionMismatch(Exception):
     def __str__(self) -> str:
         return (
             f"Manager version info {self.manager_version} does not match interchange"
-            f" version info {self.interchange_version}, causing a critical failure"
+            f" version info {self.interchange_version}"
         )
 
 
@@ -38,11 +38,8 @@ class WorkerLost(Exception):
         self.worker_id = worker_id
         self.hostname = hostname
 
-    def __repr__(self):
-        return "Task failure due to loss of worker {} on host {}".format(self.worker_id, self.hostname)
-
     def __str__(self):
-        return self.__repr__()
+        return "Task failure due to loss of worker {} on host {}".format(self.worker_id, self.hostname)
 
 
 class CommandClientTimeoutError(Exception):

{parsl-2025.1.6 → parsl-2025.1.20}/parsl/executors/high_throughput/executor.py

@@ -8,7 +8,7 @@ import warnings
 from collections import defaultdict
 from concurrent.futures import Future
 from dataclasses import dataclass
-from typing import Callable, Dict, List, Optional, Sequence, Tuple, Union
+from typing import Callable, Dict, List, Optional, Sequence, Set, Tuple, Union
 
 import typeguard
 
@@ -357,10 +357,8 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
         return self.logdir
 
     def validate_resource_spec(self, resource_specification: dict):
-        """HTEX supports the following *Optional* resource specifications:
-        priority: lower value is higher priority"""
         if resource_specification:
-            acceptable_fields = {'priority'}
+            acceptable_fields: Set[str] = set()  # add new resource spec field names here to make htex accept them
             keys = set(resource_specification.keys())
             invalid_keys = keys - acceptable_fields
             if invalid_keys:

{parsl-2025.1.6 → parsl-2025.1.20}/parsl/executors/high_throughput/zmq_pipes.py

@@ -213,7 +213,6 @@ class ResultsIncoming:
         """Get a message from the queue, returning None if timeout expires
         without a message. timeout is measured in milliseconds.
         """
-        logger.debug("Waiting for ResultsIncoming message")
         socks = dict(self.poller.poll(timeout=timeout_ms))
         if self.results_receiver in socks and socks[self.results_receiver] == zmq.POLLIN:
             m = self.results_receiver.recv_multipart()

{parsl-2025.1.6 → parsl-2025.1.20}/parsl/executors/taskvine/executor.py

@@ -582,12 +582,6 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
         logger.debug("TaskVine shutdown started")
         self._should_stop.set()
 
-        # Remove the workers that are still going
-        kill_ids = [self.blocks_to_job_id[block] for block in self.blocks_to_job_id.keys()]
-        if self.provider:
-            logger.debug("Cancelling blocks")
-            self.provider.cancel(kill_ids)
-
         # Join all processes before exiting
         logger.debug("Joining on submit process")
         self._submit_process.join()

{parsl-2025.1.6 → parsl-2025.1.20}/parsl/executors/taskvine/factory_config.py

@@ -32,7 +32,7 @@ class TaskVineFactoryConfig:
 
     worker_options: Optional[str]
         Additional options to pass to workers. Run
-        `vine_worker --help` for more details.
+        ``vine_worker --help`` for more details.
         Default is None.
 
     worker_executable: str

{parsl-2025.1.6 → parsl-2025.1.20}/parsl/executors/taskvine/manager_config.py

@@ -24,7 +24,7 @@ class TaskVineManagerConfig:
 
     address: Optional[str]
         Address of the local machine.
-        If None, socket.gethostname() will be used to determine the address.
+        If None, :py:func:`parsl.addresses.get_any_address` will be used to determine the address.
 
     project_name: Optional[str]
         If given, TaskVine will periodically report its status and performance
@@ -56,7 +56,7 @@ class TaskVineManagerConfig:
         environment name is given, TaskVine will package the conda
         environment in a tarball and send it along with tasks to be
         executed in a replicated conda environment.
-        If a tarball of packages (*.tar.gz) is given, TaskVine
+        If a tarball of packages (``*.tar.gz``) is given, TaskVine
         skips the packaging step and sends the tarball along with
         tasks to be executed in a replicated conda environment.
 

{parsl-2025.1.6 → parsl-2025.1.20}/parsl/executors/workqueue/executor.py

@@ -697,12 +697,6 @@ class WorkQueueExecutor(BlockProviderExecutor, putils.RepresentationMixin):
         logger.debug("Work Queue shutdown started")
         self.should_stop.value = True
 
-        # Remove the workers that are still going
-        kill_ids = [self.blocks_to_job_id[block] for block in self.blocks_to_job_id.keys()]
-        if self.provider:
-            logger.debug("Cancelling blocks")
-            self.provider.cancel(kill_ids)
-
         logger.debug("Joining on submit process")
         self.submit_process.join()
         self.submit_process.close()

parsl-2025.1.20/parsl/tests/configs/globus_compute.py

@@ -0,0 +1,20 @@
+import os
+
+from globus_compute_sdk import Executor
+
+from parsl.config import Config
+from parsl.executors import GlobusComputeExecutor
+
+
+def fresh_config():
+
+    endpoint_id = os.environ["GLOBUS_COMPUTE_ENDPOINT"]
+
+    return Config(
+        executors=[
+            GlobusComputeExecutor(
+                executor=Executor(endpoint_id=endpoint_id),
+                label="globus_compute",
+            )
+        ]
+    )

{parsl-2025.1.6 → parsl-2025.1.20}/parsl/tests/conftest.py

@@ -163,6 +163,10 @@ def pytest_configure(config):
         'markers',
         'shared_fs: Marks tests that require a shared_fs between the workers are the test client'
     )
+    config.addinivalue_line(
+        'markers',
+        'issue_3620: Marks tests that do not work correctly on GlobusComputeExecutor (ref: issue 3620)'
+    )
 
 
 @pytest.fixture(autouse=True, scope='session')

{parsl-2025.1.6 → parsl-2025.1.20}/parsl/tests/test_error_handling/test_resource_spec.py

@@ -1,3 +1,5 @@
+import pytest
+
 import parsl
 from parsl.app.app import python_app
 from parsl.executors import WorkQueueExecutor
@@ -11,6 +13,7 @@ def double(x, parsl_resource_specification={}):
     return x * 2
 
 
+@pytest.mark.issue_3620
 def test_resource(n=2):
     executors = parsl.dfk().executors
     executor = None