parsl 2024.5.13__tar.gz → 2024.5.27__tar.gz

{parsl-2024.5.13/parsl.egg-info → parsl-2024.5.27}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2024.5.13
+Version: 2024.5.27
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2024.05.13.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2024.05.27.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0

{parsl-2024.5.13 → parsl-2024.5.27}/README.rst

@@ -22,17 +22,19 @@ Parsl lets you chain functions together and will launch each function as inputs
         return x + 1
 
     @python_app
-    def g(x):
-        return x * 2
+    def g(x, y):
+        return x + y
 
     # Start Parsl on a single computer
     with parsl.load():
-        # These functions now return Futures, and can be chained
+        # These functions now return Futures
         future = f(1)
         assert future.result() == 2
 
-        future = g(f(1))
-        assert future.result() == 4
+        # Functions run concurrently, can be chained
+        f_a, f_b = f(2), f(3)
+        future = g(f_a, f_b)
+        assert future.result() == 7
 
 
 Start with the `configuration quickstart <https://parsl.readthedocs.io/en/stable/quickstart.html#getting-started>`_ to learn how to tell Parsl how to use your computing resource,
@@ -120,7 +122,7 @@ Parsl is supported in Python 3.8+. Requirements can be found `here <requirements
 Code of Conduct
 ===============
 
-Parsl seeks to foster an open and welcoming environment - Please see the `Parsl Code of Conduct <https://github.com/Parsl/parsl/blob/master/CODE_OF_CONDUCT.md>`_ for more details.
+Parsl seeks to foster an open and welcoming environment - Please see the `Parsl Code of Conduct <https://github.com/Parsl/parsl?tab=coc-ov-file#parsl-code-of-conduct>`_ for more details.
 
 Contributing
 ============

{parsl-2024.5.13 → parsl-2024.5.27}/parsl/channels/base.py

@@ -89,15 +89,8 @@ class Channel(metaclass=ABCMeta):
         pass
 
     @abstractmethod
-    def close(self) -> bool:
-        ''' Closes the channel. Clean out any auth credentials.
-
-        Args:
-            None
-
-        Returns:
-            Bool
-
+    def close(self) -> None:
+        ''' Closes the channel.
         '''
         pass
 

{parsl-2024.5.13 → parsl-2024.5.27}/parsl/channels/local/local.py

@@ -107,13 +107,10 @@ class LocalChannel(Channel, RepresentationMixin):
     def pull_file(self, remote_source, local_dir):
         return self.push_file(remote_source, local_dir)
 
-    def close(self):
-        ''' There's nothing to close here, and this really doesn't do anything
-
-        Returns:
-             - False, because it really did not "close" this channel.
+    def close(self) -> None:
+        ''' There's nothing to close here, and so this doesn't do anything
         '''
-        return False
+        pass
 
     def isdir(self, path):
         """Return true if the path refers to an existing directory.

{parsl-2024.5.13 → parsl-2024.5.27}/parsl/channels/oauth_ssh/oauth_ssh.py

@@ -106,5 +106,5 @@ class OAuthSSHChannel(SSHChannel):
 
         return exit_status, stdout, stderr
 
-    def close(self):
-        return self.transport.close()
+    def close(self) -> None:
+        self.transport.close()

{parsl-2024.5.13 → parsl-2024.5.27}/parsl/channels/ssh/ssh.py

@@ -217,9 +217,9 @@ class SSHChannel(Channel, RepresentationMixin):
 
         return local_dest
 
-    def close(self):
+    def close(self) -> None:
         if self._is_connected():
-            return self.ssh_client.close()
+            self.ssh_client.close()
 
     def isdir(self, path):
         """Return true if the path refers to an existing directory.

{parsl-2024.5.13 → parsl-2024.5.27}/parsl/config.py

@@ -5,6 +5,7 @@ from typing import Callable, Iterable, Optional, Sequence, Union
 from typing_extensions import Literal
 
 from parsl.utils import RepresentationMixin
+from parsl.dataflow.dependency_resolvers import DependencyResolver
 from parsl.executors.base import ParslExecutor
 from parsl.executors.threads import ThreadPoolExecutor
 from parsl.errors import ConfigurationError
@@ -35,6 +36,8 @@ class Config(RepresentationMixin, UsageInformation):
     checkpoint_period : str, optional
         Time interval (in "HH:MM:SS") at which to checkpoint completed tasks. Only has an effect if
         ``checkpoint_mode='periodic'``.
+    dependency_resolver: plugin point for custom dependency resolvers. Default: only resolve Futures,
+        using the `SHALLOW_DEPENDENCY_RESOLVER`.
     garbage_collect : bool. optional.
         Delete task records from DFK when tasks have completed. Default: True
     internal_tasks_max_threads : int, optional
@@ -88,6 +91,7 @@ class Config(RepresentationMixin, UsageInformation):
                                         Literal['dfk_exit'],
                                         Literal['manual']] = None,
                  checkpoint_period: Optional[str] = None,
+                 dependency_resolver: Optional[DependencyResolver] = None,
                  garbage_collect: bool = True,
                  internal_tasks_max_threads: int = 10,
                  retries: int = 0,
@@ -123,6 +127,7 @@ class Config(RepresentationMixin, UsageInformation):
         if checkpoint_mode == 'periodic' and checkpoint_period is None:
             checkpoint_period = "00:30:00"
         self.checkpoint_period = checkpoint_period
+        self.dependency_resolver = dependency_resolver
         self.garbage_collect = garbage_collect
         self.internal_tasks_max_threads = internal_tasks_max_threads
         self.retries = retries
@@ -152,4 +157,5 @@ class Config(RepresentationMixin, UsageInformation):
                 ', '.join(['label={}'.format(repr(d)) for d in duplicates])))
 
     def get_usage_information(self):
-        return {"executors_len": len(self.executors)}
+        return {"executors_len": len(self.executors),
+                "dependency_resolver": self.dependency_resolver is not None}

parsl-2024.5.27/parsl/dataflow/dependency_resolvers.py

@@ -0,0 +1,115 @@
+from concurrent.futures import Future
+from dataclasses import dataclass
+from functools import singledispatch
+from typing import Callable, Sequence
+
+
+@dataclass
+class DependencyResolver:
+    """A DependencyResolver describes how app dependencies can be resolved.
+    It is specified as two functions: `traverse_to_gather` which turns an
+    app parameter into a sequence of futures which must be waited for before
+    the task can be executed (for example, in the case of
+    `DEEP_DEPENDENCY_RESOLVER` this traverses structures such as lists to
+    find every contained ``Future``), and `traverse_to_unwrap` which turns an
+    app parameter into its value to be passed to the app on execution
+    (for example in the case of `DEEP_DEPENDENCY_RESOLVER` this replaces a
+    list containing futures with a new list containing the values of those
+    resolved futures).
+
+    By default, Parsl will use `SHALLOW_DEPENDENCY_RESOLVER` which only
+    resolves Futures passed directly as arguments.
+    """
+    traverse_to_gather: Callable[[object], Sequence[Future]]
+    traverse_to_unwrap: Callable[[object], object]
+
+
+@singledispatch
+def shallow_traverse_to_gather(o):
+    # objects in general do not expose futures that we can see
+    return []
+
+
+@singledispatch
+def shallow_traverse_to_unwrap(o):
+    # objects in general unwrap to themselves
+    return o
+
+
+@shallow_traverse_to_gather.register
+def _(fut: Future):
+    return [fut]
+
+
+@shallow_traverse_to_unwrap.register
+@singledispatch
+def _(fut: Future):
+    assert fut.done()
+    return fut.result()
+
+
+@singledispatch
+def deep_traverse_to_gather(o):
+    # objects in general do not expose futures that we can see
+    return []
+
+
+@singledispatch
+def deep_traverse_to_unwrap(o):
+    # objects in general unwrap to themselves
+    return o
+
+
+@deep_traverse_to_gather.register
+def _(fut: Future):
+    return [fut]
+
+
+@deep_traverse_to_unwrap.register
+@singledispatch
+def _(fut: Future):
+    assert fut.done()
+    return fut.result()
+
+
+@deep_traverse_to_gather.register(tuple)
+@deep_traverse_to_gather.register(list)
+@deep_traverse_to_gather.register(set)
+def _(iterable):
+    return [e for v in iterable for e in deep_traverse_to_gather(v)]
+
+
+@deep_traverse_to_unwrap.register(tuple)
+@deep_traverse_to_unwrap.register(list)
+@deep_traverse_to_unwrap.register(set)
+@singledispatch
+def _(iterable):
+
+    type_ = type(iterable)
+    return type_(map(deep_traverse_to_unwrap, iterable))
+
+
+@deep_traverse_to_gather.register(dict)
+def _(dictionary):
+    futures = []
+    for key, value in dictionary.items():
+        futures.extend(deep_traverse_to_gather(key))
+        futures.extend(deep_traverse_to_gather(value))
+    return futures
+
+
+@deep_traverse_to_unwrap.register(dict)
+def _(dictionary):
+    unwrapped_dict = {}
+    for key, value in dictionary.items():
+        key = deep_traverse_to_unwrap(key)
+        value = deep_traverse_to_unwrap(value)
+        unwrapped_dict[key] = value
+    return unwrapped_dict
+
+
+DEEP_DEPENDENCY_RESOLVER = DependencyResolver(traverse_to_gather=deep_traverse_to_gather,
+                                              traverse_to_unwrap=deep_traverse_to_unwrap)
+
+SHALLOW_DEPENDENCY_RESOLVER = DependencyResolver(traverse_to_gather=shallow_traverse_to_gather,
+                                                 traverse_to_unwrap=shallow_traverse_to_unwrap)

{parsl-2024.5.13 → parsl-2024.5.27}/parsl/dataflow/dflow.py

@@ -26,6 +26,7 @@ from parsl.channels import Channel
 from parsl.config import Config
 from parsl.data_provider.data_manager import DataManager
 from parsl.data_provider.files import File
+from parsl.dataflow.dependency_resolvers import SHALLOW_DEPENDENCY_RESOLVER
 from parsl.dataflow.errors import BadCheckpoint, DependencyError, JoinError
 from parsl.dataflow.futures import AppFuture
 from parsl.dataflow.memoization import Memoizer
@@ -203,6 +204,9 @@ class DataFlowKernel:
         self.tasks: Dict[int, TaskRecord] = {}
         self.submitter_lock = threading.Lock()
 
+        self.dependency_resolver = self.config.dependency_resolver if self.config.dependency_resolver is not None \
+            else SHALLOW_DEPENDENCY_RESOLVER
+
         atexit.register(self.atexit_cleanup)
 
     def __enter__(self):
@@ -852,8 +856,11 @@ class DataFlowKernel:
         depends: List[Future] = []
 
         def check_dep(d: Any) -> None:
-            if isinstance(d, Future):
-                depends.extend([d])
+            try:
+                depends.extend(self.dependency_resolver.traverse_to_gather(d))
+            except Exception:
+                logger.exception("Exception in dependency_resolver.traverse_to_gather")
+                raise
 
         # Check the positional args
         for dep in args:
@@ -870,7 +877,8 @@ class DataFlowKernel:
 
         return depends
 
-    def _unwrap_futures(self, args, kwargs):
+    def _unwrap_futures(self, args: Sequence[Any], kwargs: Dict[str, Any]) \
+            -> Tuple[Sequence[Any], Dict[str, Any], Sequence[Tuple[Exception, str]]]:
         """This function should be called when all dependencies have completed.
 
         It will rewrite the arguments for that task, replacing each Future
@@ -891,53 +899,40 @@ class DataFlowKernel:
         """
         dep_failures = []
 
+        def append_failure(e: Exception, dep: Future) -> None:
+            # If this Future is associated with a task inside this DFK,
+            # then refer to the task ID.
+            # Otherwise make a repr of the Future object.
+            if hasattr(dep, 'task_record') and dep.task_record['dfk'] == self:
+                tid = "task " + repr(dep.task_record['id'])
+            else:
+                tid = repr(dep)
+            dep_failures.extend([(e, tid)])
+
         # Replace item in args
         new_args = []
         for dep in args:
-            if isinstance(dep, Future):
-                try:
-                    new_args.extend([dep.result()])
-                except Exception as e:
-                    # If this Future is associated with a task inside this DFK,
-                    # then refer to the task ID.
-                    # Otherwise make a repr of the Future object.
-                    if hasattr(dep, 'task_record') and dep.task_record['dfk'] == self:
-                        tid = "task " + repr(dep.task_record['id'])
-                    else:
-                        tid = repr(dep)
-                    dep_failures.extend([(e, tid)])
-            else:
-                new_args.extend([dep])
+            try:
+                new_args.extend([self.dependency_resolver.traverse_to_unwrap(dep)])
+            except Exception as e:
+                append_failure(e, dep)
 
         # Check for explicit kwargs ex, fu_1=<fut>
         for key in kwargs:
             dep = kwargs[key]
-            if isinstance(dep, Future):
-                try:
-                    kwargs[key] = dep.result()
-                except Exception as e:
-                    if hasattr(dep, 'task_record'):
-                        tid = dep.task_record['id']
-                    else:
-                        tid = None
-                    dep_failures.extend([(e, tid)])
+            try:
+                kwargs[key] = self.dependency_resolver.traverse_to_unwrap(dep)
+            except Exception as e:
+                append_failure(e, dep)
 
         # Check for futures in inputs=[<fut>...]
         if 'inputs' in kwargs:
             new_inputs = []
             for dep in kwargs['inputs']:
-                if isinstance(dep, Future):
-                    try:
-                        new_inputs.extend([dep.result()])
-                    except Exception as e:
-                        if hasattr(dep, 'task_record'):
-                            tid = dep.task_record['id']
-                        else:
-                            tid = None
-                        dep_failures.extend([(e, tid)])
-
-                else:
-                    new_inputs.extend([dep])
+                try:
+                    new_inputs.extend([self.dependency_resolver.traverse_to_unwrap(dep)])
+                except Exception as e:
+                    append_failure(e, dep)
             kwargs['inputs'] = new_inputs
 
         return new_args, kwargs, dep_failures
@@ -1042,6 +1037,8 @@ class DataFlowKernel:
 
         func = self._add_output_deps(executor, app_args, app_kwargs, app_fu, func)
 
+        logger.debug("Added output dependencies")
+
         # Replace the function invocation in the TaskRecord with whatever file-staging
         # substitutions have been made.
         task_record.update({
@@ -1053,8 +1050,10 @@ class DataFlowKernel:
 
         self.tasks[task_id] = task_record
 
+        logger.debug("Gathering dependencies")
         # Get the list of dependencies for the task
         depends = self._gather_all_deps(app_args, app_kwargs)
+        logger.debug("Gathered dependencies")
         task_record['depends'] = depends
 
         depend_descs = []
@@ -1156,7 +1155,7 @@ class DataFlowKernel:
             executor.run_id = self.run_id
             executor.run_dir = self.run_dir
             executor.hub_address = self.hub_address
-            executor.hub_port = self.hub_zmq_port
+            executor.hub_zmq_port = self.hub_zmq_port
             if self.monitoring:
                 executor.monitoring_radio = self.monitoring.radio
             if hasattr(executor, 'provider'):
@@ -1271,6 +1270,13 @@ class DataFlowKernel:
         atexit.unregister(self.atexit_cleanup)
         logger.info("Unregistered atexit hook")
 
+        if DataFlowKernelLoader._dfk is self:
+            logger.info("Unregistering default DFK")
+            parsl.clear()
+            logger.info("Unregistered default DFK")
+        else:
+            logger.debug("Cleaning up non-default DFK - not unregistering")
+
         logger.info("DFK cleanup complete")
 
     def checkpoint(self, tasks: Optional[Sequence[TaskRecord]] = None) -> str:

{parsl-2024.5.13 → parsl-2024.5.27}/parsl/executors/__init__.py

@@ -1,9 +1,11 @@
 from parsl.executors.threads import ThreadPoolExecutor
 from parsl.executors.workqueue.executor import WorkQueueExecutor
 from parsl.executors.high_throughput.executor import HighThroughputExecutor
+from parsl.executors.high_throughput.mpi_executor import MPIExecutor
 from parsl.executors.flux.executor import FluxExecutor
 
 __all__ = ['ThreadPoolExecutor',
            'HighThroughputExecutor',
+           'MPIExecutor',
            'WorkQueueExecutor',
            'FluxExecutor']

{parsl-2024.5.13 → parsl-2024.5.27}/parsl/executors/base.py

@@ -50,13 +50,13 @@ class ParslExecutor(metaclass=ABCMeta):
         self,
         *,
         hub_address: Optional[str] = None,
-        hub_port: Optional[int] = None,
+        hub_zmq_port: Optional[int] = None,
         monitoring_radio: Optional[MonitoringRadio] = None,
         run_dir: str = ".",
         run_id: Optional[str] = None,
     ):
         self.hub_address = hub_address
-        self.hub_port = hub_port
+        self.hub_zmq_port = hub_zmq_port
         self.monitoring_radio = monitoring_radio
         self.run_dir = os.path.abspath(run_dir)
         self.run_id = run_id
@@ -136,14 +136,14 @@ class ParslExecutor(metaclass=ABCMeta):
         self._hub_address = value
 
     @property
-    def hub_port(self) -> Optional[int]:
+    def hub_zmq_port(self) -> Optional[int]:
         """Port to the Hub for monitoring.
         """
-        return self._hub_port
+        return self._hub_zmq_port
 
-    @hub_port.setter
-    def hub_port(self, value: Optional[int]) -> None:
-        self._hub_port = value
+    @hub_zmq_port.setter
+    def hub_zmq_port(self, value: Optional[int]) -> None:
+        self._hub_zmq_port = value
 
     @property
     def monitoring_radio(self) -> Optional[MonitoringRadio]:

{parsl-2024.5.13 → parsl-2024.5.27}/parsl/executors/high_throughput/errors.py

@@ -10,3 +10,13 @@ class WorkerLost(Exception):
 
     def __str__(self):
         return self.__repr__()
+
+
+class CommandClientTimeoutError(Exception):
+    """Raised when the command client times out waiting for a response.
+    """
+
+
+class CommandClientBadError(Exception):
+    """Raised when the command client is bad from an earlier timeout.
+    """