parsl 2025.1.13__py3-none-any.whl → 2025.1.27__py3-none-any.whl

parsl/configs/gc_multisite.py

@@ -0,0 +1,27 @@
+from globus_compute_sdk import Executor
+
+from parsl.config import Config
+from parsl.executors import GlobusComputeExecutor
+from parsl.usage_tracking.levels import LEVEL_1
+
+# Please start your own endpoint on perlmutter following instructions below to use this config:
+# https://globus-compute.readthedocs.io/en/stable/endpoints/endpoint_examples.html#perlmutter-nersc
+perlmutter_endpoint = 'YOUR_PERLMUTTER_ENDPOINT_UUID'
+
+# Please start your own endpoint on expanse following instructions below to use this config:
+# https://globus-compute.readthedocs.io/en/stable/endpoints/endpoint_examples.html#expanse-sdsc
+expanse_endpoint = 'YOUR_EXPANSE_ENDPOINT_UUID'
+
+config = Config(
+    executors=[
+        GlobusComputeExecutor(
+            executor=Executor(endpoint_id=perlmutter_endpoint),
+            label="Perlmutter",
+        ),
+        GlobusComputeExecutor(
+            executor=Executor(endpoint_id=expanse_endpoint),
+            label="Expanse",
+        ),
+    ],
+    usage_tracking=LEVEL_1,
+)

parsl/configs/gc_tutorial.py

@@ -0,0 +1,18 @@
+from globus_compute_sdk import Executor
+
+from parsl.config import Config
+from parsl.executors import GlobusComputeExecutor
+from parsl.usage_tracking.levels import LEVEL_1
+
+# Public tutorial endpoint
+tutorial_endpoint = '4b116d3c-1703-4f8f-9f6f-39921e5864df'
+
+config = Config(
+    executors=[
+        GlobusComputeExecutor(
+            executor=Executor(endpoint_id=tutorial_endpoint),
+            label="Tutorial_Endpoint_py3.11",
+        )
+    ],
+    usage_tracking=LEVEL_1,
+)

parsl/dataflow/dflow.py

@@ -28,7 +28,7 @@ from parsl.config import Config
 from parsl.data_provider.data_manager import DataManager
 from parsl.data_provider.files import File
 from parsl.dataflow.dependency_resolvers import SHALLOW_DEPENDENCY_RESOLVER
-from parsl.dataflow.errors import BadCheckpoint, DependencyError, JoinError
+from parsl.dataflow.errors import DependencyError, JoinError
 from parsl.dataflow.futures import AppFuture
 from parsl.dataflow.memoization import Memoizer
 from parsl.dataflow.rundirs import make_rundir
@@ -161,13 +161,13 @@ class DataFlowKernel:
                                  workflow_info))
 
         if config.checkpoint_files is not None:
-            checkpoints = self.load_checkpoints(config.checkpoint_files)
+            checkpoint_files = config.checkpoint_files
         elif config.checkpoint_files is None and config.checkpoint_mode is not None:
-            checkpoints = self.load_checkpoints(get_all_checkpoints(self.run_dir))
+            checkpoint_files = get_all_checkpoints(self.run_dir)
         else:
-            checkpoints = {}
+            checkpoint_files = []
 
-        self.memoizer = Memoizer(self, memoize=config.app_cache, checkpoint=checkpoints)
+        self.memoizer = Memoizer(self, memoize=config.app_cache, checkpoint_files=checkpoint_files)
         self.checkpointed_tasks = 0
         self._checkpoint_timer = None
         self.checkpoint_mode = config.checkpoint_mode
@@ -484,24 +484,18 @@ class DataFlowKernel:
 
             # now we know each joinable Future is done
             # so now look for any exceptions
-            exceptions_tids: List[Tuple[BaseException, Optional[str]]]
+            exceptions_tids: List[Tuple[BaseException, str]]
             exceptions_tids = []
             if isinstance(joinable, Future):
                 je = joinable.exception()
                 if je is not None:
-                    if hasattr(joinable, 'task_record'):
-                        tid = joinable.task_record['id']
-                    else:
-                        tid = None
+                    tid = self.render_future_description(joinable)
                     exceptions_tids = [(je, tid)]
             elif isinstance(joinable, list):
                 for future in joinable:
                     je = future.exception()
                     if je is not None:
-                        if hasattr(joinable, 'task_record'):
-                            tid = joinable.task_record['id']
-                        else:
-                            tid = None
+                        tid = self.render_future_description(future)
                         exceptions_tids.append((je, tid))
             else:
                 raise TypeError(f"Unknown joinable type {type(joinable)}")
@@ -918,13 +912,7 @@ class DataFlowKernel:
         dep_failures = []
 
         def append_failure(e: Exception, dep: Future) -> None:
-            # If this Future is associated with a task inside this DFK,
-            # then refer to the task ID.
-            # Otherwise make a repr of the Future object.
-            if hasattr(dep, 'task_record') and dep.task_record['dfk'] == self:
-                tid = "task " + repr(dep.task_record['id'])
-            else:
-                tid = repr(dep)
+            tid = self.render_future_description(dep)
             dep_failures.extend([(e, tid)])
 
         # Replace item in args
@@ -1076,10 +1064,7 @@ class DataFlowKernel:
 
         depend_descs = []
         for d in depends:
-            if isinstance(d, AppFuture) or isinstance(d, DataFuture):
-                depend_descs.append("task {}".format(d.tid))
-            else:
-                depend_descs.append(repr(d))
+            depend_descs.append(self.render_future_description(d))
 
         if depend_descs != []:
             waiting_message = "waiting on {}".format(", ".join(depend_descs))
@@ -1215,10 +1200,8 @@ class DataFlowKernel:
                 self._checkpoint_timer.close()
 
         # Send final stats
-        logger.info("Sending end message for usage tracking")
         self.usage_tracker.send_end_message()
         self.usage_tracker.close()
-        logger.info("Closed usage tracking")
 
         logger.info("Closing job status poller")
         self.job_status_poller.close()
@@ -1280,7 +1263,7 @@ class DataFlowKernel:
         Returns:
             Checkpoint dir if checkpoints were written successfully.
             By default the checkpoints are written to the RUNDIR of the current
-            run under RUNDIR/checkpoints/{tasks.pkl, dfk.pkl}
+            run under RUNDIR/checkpoints/tasks.pkl
         """
         with self.checkpoint_lock:
             if tasks:
@@ -1290,18 +1273,11 @@ class DataFlowKernel:
                 self.checkpointable_tasks = []
 
             checkpoint_dir = '{0}/checkpoint'.format(self.run_dir)
-            checkpoint_dfk = checkpoint_dir + '/dfk.pkl'
             checkpoint_tasks = checkpoint_dir + '/tasks.pkl'
 
             if not os.path.exists(checkpoint_dir):
                 os.makedirs(checkpoint_dir, exist_ok=True)
 
-            with open(checkpoint_dfk, 'wb') as f:
-                state = {'rundir': self.run_dir,
-                         'task_count': self.task_count
-                         }
-                pickle.dump(state, f)
-
             count = 0
 
             with open(checkpoint_tasks, 'ab') as f:
@@ -1334,74 +1310,6 @@ class DataFlowKernel:
 
             return checkpoint_dir
 
-    def _load_checkpoints(self, checkpointDirs: Sequence[str]) -> Dict[str, Future[Any]]:
-        """Load a checkpoint file into a lookup table.
-
-        The data being loaded from the pickle file mostly contains input
-        attributes of the task: func, args, kwargs, env...
-        To simplify the check of whether the exact task has been completed
-        in the checkpoint, we hash these input params and use it as the key
-        for the memoized lookup table.
-
-        Args:
-            - checkpointDirs (list) : List of filepaths to checkpoints
-              Eg. ['runinfo/001', 'runinfo/002']
-
-        Returns:
-            - memoized_lookup_table (dict)
-        """
-        memo_lookup_table = {}
-
-        for checkpoint_dir in checkpointDirs:
-            logger.info("Loading checkpoints from {}".format(checkpoint_dir))
-            checkpoint_file = os.path.join(checkpoint_dir, 'tasks.pkl')
-            try:
-                with open(checkpoint_file, 'rb') as f:
-                    while True:
-                        try:
-                            data = pickle.load(f)
-                            # Copy and hash only the input attributes
-                            memo_fu: Future = Future()
-                            assert data['exception'] is None
-                            memo_fu.set_result(data['result'])
-                            memo_lookup_table[data['hash']] = memo_fu
-
-                        except EOFError:
-                            # Done with the checkpoint file
-                            break
-            except FileNotFoundError:
-                reason = "Checkpoint file was not found: {}".format(
-                    checkpoint_file)
-                logger.error(reason)
-                raise BadCheckpoint(reason)
-            except Exception:
-                reason = "Failed to load checkpoint: {}".format(
-                    checkpoint_file)
-                logger.error(reason)
-                raise BadCheckpoint(reason)
-
-            logger.info("Completed loading checkpoint: {0} with {1} tasks".format(checkpoint_file,
-                                                                                  len(memo_lookup_table.keys())))
-        return memo_lookup_table
-
-    @typeguard.typechecked
-    def load_checkpoints(self, checkpointDirs: Optional[Sequence[str]]) -> Dict[str, Future]:
-        """Load checkpoints from the checkpoint files into a dictionary.
-
-        The results are used to pre-populate the memoizer's lookup_table
-
-        Kwargs:
-             - checkpointDirs (list) : List of run folder to use as checkpoints
-               Eg. ['runinfo/001', 'runinfo/002']
-
-        Returns:
-             - dict containing, hashed -> future mappings
-        """
-        if checkpointDirs:
-            return self._load_checkpoints(checkpointDirs)
-        else:
-            return {}
-
     @staticmethod
     def _log_std_streams(task_record: TaskRecord) -> None:
         tid = task_record['id']
@@ -1438,6 +1346,18 @@ class DataFlowKernel:
                 '' if label is None else '_{}'.format(label),
                 kw))
 
+    def render_future_description(self, dep: Future) -> str:
+        """Renders a description of the future in the context of the
+        current DFK.
+        """
+        if isinstance(dep, AppFuture) and dep.task_record['dfk'] == self:
+            tid = "task " + repr(dep.task_record['id'])
+        elif isinstance(dep, DataFuture):
+            tid = "DataFuture from task " + repr(dep.tid)
+        else:
+            tid = repr(dep)
+        return tid
+
 
 class DataFlowKernelLoader:
     """Manage which DataFlowKernel is active.

parsl/dataflow/errors.py

@@ -1,4 +1,4 @@
-from typing import Optional, Sequence, Tuple
+from typing import List, Sequence, Tuple
 
 from parsl.errors import ParslError
 
@@ -29,35 +29,77 @@ class BadCheckpoint(DataFlowException):
         return self.reason
 
 
-class DependencyError(DataFlowException):
-    """Error raised if an app cannot run because there was an error
-       in a dependency.
+class PropagatedException(DataFlowException):
+    """Error raised if an app fails because there was an error
+       in a related task. This is intended to be subclassed for
+       dependency and join_app errors.
 
     Args:
-         - dependent_exceptions_tids: List of exceptions and identifiers for
-           dependencies which failed. The identifier might be a task ID or
-           the repr of a non-DFK Future.
+         - dependent_exceptions_tids: List of exceptions and brief descriptions
+           for dependencies which failed. The description might be a task ID or
+           the repr of a non-AppFuture.
          - task_id: Task ID of the task that failed because of the dependency error
     """
 
-    def __init__(self, dependent_exceptions_tids: Sequence[Tuple[Exception, str]], task_id: int) -> None:
+    def __init__(self,
+                 dependent_exceptions_tids: Sequence[Tuple[BaseException, str]],
+                 task_id: int,
+                 *,
+                 failure_description: str) -> None:
         self.dependent_exceptions_tids = dependent_exceptions_tids
         self.task_id = task_id
+        self._failure_description = failure_description
+
+        (cause, cause_sequence) = self._find_any_root_cause()
+        self.__cause__ = cause
+        self._cause_sequence = cause_sequence
 
     def __str__(self) -> str:
-        deps = ", ".join(tid for _exc, tid in self.dependent_exceptions_tids)
-        return f"Dependency failure for task {self.task_id} with failed dependencies from {deps}"
+        sequence_text = " <- ".join(self._cause_sequence)
+        return f"{self._failure_description} for task {self.task_id}. " \
+               f"The representative cause is via {sequence_text}"
+
+    def _find_any_root_cause(self) -> Tuple[BaseException, List[str]]:
+        """Looks recursively through self.dependent_exceptions_tids to find
+        an exception that caused this propagated error, that is not itself
+        a propagated error.
+        """
+        e: BaseException = self
+        dep_ids = []
+        while isinstance(e, PropagatedException) and len(e.dependent_exceptions_tids) >= 1:
+            id_txt = e.dependent_exceptions_tids[0][1]
+            assert isinstance(id_txt, str)
+            # if there are several causes for this exception, label that
+            # there are more so that we know that the representative fail
+            # sequence is not the full story.
+            if len(e.dependent_exceptions_tids) > 1:
+                id_txt += " (+ others)"
+            dep_ids.append(id_txt)
+            e = e.dependent_exceptions_tids[0][0]
+        return e, dep_ids
+
+
+class DependencyError(PropagatedException):
+    """Error raised if an app cannot run because there was an error
+       in a dependency. There can be several exceptions (one from each
+       dependency) and DependencyError collects them all together.
 
+    Args:
+         - dependent_exceptions_tids: List of exceptions and brief descriptions
+           for dependencies which failed. The description might be a task ID or
+           the repr of a non-AppFuture.
+         - task_id: Task ID of the task that failed because of the dependency error
+    """
+    def __init__(self, dependent_exceptions_tids: Sequence[Tuple[BaseException, str]], task_id: int) -> None:
+        super().__init__(dependent_exceptions_tids, task_id,
+                         failure_description="Dependency failure")
 
-class JoinError(DataFlowException):
+
+class JoinError(PropagatedException):
     """Error raised if apps joining into a join_app raise exceptions.
        There can be several exceptions (one from each joining app),
        and JoinError collects them all together.
     """
-    def __init__(self, dependent_exceptions_tids: Sequence[Tuple[BaseException, Optional[str]]], task_id: int) -> None:
-        self.dependent_exceptions_tids = dependent_exceptions_tids
-        self.task_id = task_id
-
-    def __str__(self) -> str:
-        dep_tids = [tid for (exception, tid) in self.dependent_exceptions_tids]
-        return "Join failure for task {} with failed join dependencies from tasks {}".format(self.task_id, dep_tids)
+    def __init__(self, dependent_exceptions_tids: Sequence[Tuple[BaseException, str]], task_id: int) -> None:
+        super().__init__(dependent_exceptions_tids, task_id,
+                         failure_description="Join failure")

parsl/dataflow/memoization.py

@@ -2,10 +2,14 @@ from __future__ import annotations
 
 import hashlib
 import logging
+import os
 import pickle
 from functools import lru_cache, singledispatch
-from typing import TYPE_CHECKING, Any, Dict, List, Optional
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence
 
+import typeguard
+
+from parsl.dataflow.errors import BadCheckpoint
 from parsl.dataflow.taskrecord import TaskRecord
 
 if TYPE_CHECKING:
@@ -146,7 +150,7 @@ class Memoizer:
 
     """
 
-    def __init__(self, dfk: DataFlowKernel, memoize: bool = True, checkpoint: Dict[str, Future[Any]] = {}):
+    def __init__(self, dfk: DataFlowKernel, *, memoize: bool = True, checkpoint_files: Sequence[str]):
         """Initialize the memoizer.
 
         Args:
@@ -159,6 +163,8 @@ class Memoizer:
         self.dfk = dfk
         self.memoize = memoize
 
+        checkpoint = self.load_checkpoints(checkpoint_files)
+
         if self.memoize:
             logger.info("App caching initialized")
             self.memo_lookup_table = checkpoint
@@ -274,3 +280,71 @@ class Memoizer:
         else:
             logger.debug(f"Storing app cache entry {task['hashsum']} with result from task {task_id}")
         self.memo_lookup_table[task['hashsum']] = r
+
+    def _load_checkpoints(self, checkpointDirs: Sequence[str]) -> Dict[str, Future[Any]]:
+        """Load a checkpoint file into a lookup table.
+
+        The data being loaded from the pickle file mostly contains input
+        attributes of the task: func, args, kwargs, env...
+        To simplify the check of whether the exact task has been completed
+        in the checkpoint, we hash these input params and use it as the key
+        for the memoized lookup table.
+
+        Args:
+            - checkpointDirs (list) : List of filepaths to checkpoints
+              Eg. ['runinfo/001', 'runinfo/002']
+
+        Returns:
+            - memoized_lookup_table (dict)
+        """
+        memo_lookup_table = {}
+
+        for checkpoint_dir in checkpointDirs:
+            logger.info("Loading checkpoints from {}".format(checkpoint_dir))
+            checkpoint_file = os.path.join(checkpoint_dir, 'tasks.pkl')
+            try:
+                with open(checkpoint_file, 'rb') as f:
+                    while True:
+                        try:
+                            data = pickle.load(f)
+                            # Copy and hash only the input attributes
+                            memo_fu: Future = Future()
+                            assert data['exception'] is None
+                            memo_fu.set_result(data['result'])
+                            memo_lookup_table[data['hash']] = memo_fu
+
+                        except EOFError:
+                            # Done with the checkpoint file
+                            break
+            except FileNotFoundError:
+                reason = "Checkpoint file was not found: {}".format(
+                    checkpoint_file)
+                logger.error(reason)
+                raise BadCheckpoint(reason)
+            except Exception:
+                reason = "Failed to load checkpoint: {}".format(
+                    checkpoint_file)
+                logger.error(reason)
+                raise BadCheckpoint(reason)
+
+            logger.info("Completed loading checkpoint: {0} with {1} tasks".format(checkpoint_file,
+                                                                                  len(memo_lookup_table.keys())))
+        return memo_lookup_table
+
+    @typeguard.typechecked
+    def load_checkpoints(self, checkpointDirs: Optional[Sequence[str]]) -> Dict[str, Future]:
+        """Load checkpoints from the checkpoint files into a dictionary.
+
+        The results are used to pre-populate the memoizer's lookup_table
+
+        Kwargs:
+             - checkpointDirs (list) : List of run folder to use as checkpoints
+               Eg. ['runinfo/001', 'runinfo/002']
+
+        Returns:
+             - dict containing, hashed -> future mappings
+        """
+        if checkpointDirs:
+            return self._load_checkpoints(checkpointDirs)
+        else:
+            return {}

parsl/dataflow/taskrecord.py

@@ -43,12 +43,11 @@ class TaskRecord(TypedDict, total=False):
     executed on.
     """
 
-    retries_left: int
     fail_count: int
     fail_cost: float
     fail_history: List[str]
 
-    checkpoint: bool  # this change is also in #1516
+    checkpoint: bool
     """Should this task be checkpointed?
     """
 
@@ -68,7 +67,6 @@ class TaskRecord(TypedDict, total=False):
 
     # these three could be more strongly typed perhaps but I'm not thinking about that now
     func: Callable
-    fn_hash: str
     args: Sequence[Any]
     # in some places we uses a Tuple[Any, ...] and in some places a List[Any].
     # This is an attempt to correctly type both of those.

parsl/executors/__init__.py

@@ -1,4 +1,5 @@
 from parsl.executors.flux.executor import FluxExecutor
+from parsl.executors.globus_compute import GlobusComputeExecutor
 from parsl.executors.high_throughput.executor import HighThroughputExecutor
 from parsl.executors.high_throughput.mpi_executor import MPIExecutor
 from parsl.executors.threads import ThreadPoolExecutor
@@ -8,4 +9,5 @@ __all__ = ['ThreadPoolExecutor',
            'HighThroughputExecutor',
            'MPIExecutor',
            'WorkQueueExecutor',
-           'FluxExecutor']
+           'FluxExecutor',
+           'GlobusComputeExecutor']

parsl/executors/globus_compute.py

@@ -0,0 +1,125 @@
+from __future__ import annotations
+
+import copy
+from concurrent.futures import Future
+from typing import Any, Callable, Dict
+
+import typeguard
+
+from parsl.errors import OptionalModuleMissing
+from parsl.executors.base import ParslExecutor
+from parsl.utils import RepresentationMixin
+
+try:
+    from globus_compute_sdk import Executor
+    _globus_compute_enabled = True
+except ImportError:
+    _globus_compute_enabled = False
+
+
+class GlobusComputeExecutor(ParslExecutor, RepresentationMixin):
+    """ GlobusComputeExecutor enables remote execution on Globus Compute endpoints
+
+    GlobusComputeExecutor is a thin wrapper over globus_compute_sdk.Executor
+    Refer to `globus-compute user documentation <https://globus-compute.readthedocs.io/en/latest/executor.html>`_
+    and `reference documentation <https://globus-compute.readthedocs.io/en/latest/reference/executor.html>`_
+    for more details.
+
+    .. note::
+       As a remote execution system, Globus Compute relies on serialization to ship
+       tasks and results between the Parsl client side and the remote Globus Compute
+       Endpoint side. Serialization is unreliable across python versions, and
+       wrappers used by Parsl assume identical Parsl versions across on both sides.
+       We recommend using matching Python, Parsl and Globus Compute version on both
+       the client side and the endpoint side for stable behavior.
+
+    """
+
+    @typeguard.typechecked
+    def __init__(
+        self,
+        executor: Executor,
+        label: str = 'GlobusComputeExecutor',
+    ):
+        """
+        Parameters
+        ----------
+
+        executor: globus_compute_sdk.Executor
+            Pass a globus_compute_sdk Executor that will be used to execute
+            tasks on a globus_compute endpoint. Refer to `globus-compute docs
+            <https://globus-compute.readthedocs.io/en/latest/reference/executor.html#globus-compute-executor>`_
+
+        label:
+            a label to name the executor
+        """
+        if not _globus_compute_enabled:
+            raise OptionalModuleMissing(
+                ['globus-compute-sdk'],
+                "GlobusComputeExecutor requires globus-compute-sdk installed"
+            )
+
+        super().__init__()
+        self.executor: Executor = executor
+        self.resource_specification = self.executor.resource_specification
+        self.user_endpoint_config = self.executor.user_endpoint_config
+        self.label = label
+
+    def start(self) -> None:
+        """ Start the Globus Compute Executor """
+        pass
+
+    def submit(self, func: Callable, resource_specification: Dict[str, Any], *args: Any, **kwargs: Any) -> Future:
+        """ Submit func to globus-compute
+
+
+        Parameters
+        ----------
+
+        func: Callable
+            Python function to execute remotely
+
+        resource_specification: Dict[str, Any]
+            Resource specification can be used specify MPI resources required by MPI applications on
+            Endpoints configured to use globus compute's MPIEngine. GCE also accepts *user_endpoint_config*
+            to configure endpoints when the endpoint is a `Multi-User Endpoint
+            <https://globus-compute.readthedocs.io/en/latest/endpoints/endpoints.html#templating-endpoint-configuration>`_
+
+        args:
+            Args to pass to the function
+
+        kwargs:
+            kwargs to pass to the function
+
+        Returns
+        -------
+
+        Future
+        """
+        res_spec = copy.deepcopy(resource_specification or self.resource_specification)
+        # Pop user_endpoint_config since it is illegal in resource_spec for globus_compute
+        if res_spec:
+            user_endpoint_config = res_spec.pop('user_endpoint_config', self.user_endpoint_config)
+        else:
+            user_endpoint_config = self.user_endpoint_config
+
+        try:
+            self.executor.resource_specification = res_spec
+            self.executor.user_endpoint_config = user_endpoint_config
+            return self.executor.submit(func, *args, **kwargs)
+        finally:
+            # Reset executor state to defaults set at configuration time
+            self.executor.resource_specification = self.resource_specification
+            self.executor.user_endpoint_config = self.user_endpoint_config
+
+    def shutdown(self):
+        """Clean-up the resources associated with the Executor.
+
+        GCE.shutdown will cancel all futures that have not yet registered with
+        Globus Compute and will not wait for the launched futures to complete.
+        This method explicitly shutsdown the result_watcher thread to avoid
+        it waiting for outstanding futures at thread exit.
+        """
+        self.executor.shutdown(wait=False, cancel_futures=True)
+        result_watcher = self.executor._get_result_watcher()
+        result_watcher.shutdown(wait=False, cancel_futures=True)

parsl/executors/high_throughput/errors.py

@@ -27,7 +27,7 @@ class VersionMismatch(Exception):
     def __str__(self) -> str:
         return (
             f"Manager version info {self.manager_version} does not match interchange"
-            f" version info {self.interchange_version}, causing a critical failure"
+            f" version info {self.interchange_version}"
         )