parsl 2023.10.16__tar.gz → 2023.11.13__tar.gz

{parsl-2023.10.16/parsl.egg-info → parsl-2023.11.13}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2023.10.16
+Version: 2023.11.13
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2023.10.16.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2023.11.13.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0
@@ -17,6 +17,7 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Requires-Python: >=3.8.0
 Provides-Extra: monitoring
+Provides-Extra: visualization
 Provides-Extra: aws
 Provides-Extra: kubernetes
 Provides-Extra: oauth_ssh

{parsl-2023.10.16 → parsl-2023.11.13}/parsl/app/app.py

@@ -6,11 +6,16 @@ import logging
 import typeguard
 from abc import ABCMeta, abstractmethod
 from inspect import signature
-from typing import List, Optional, Union
+from typing import List, Optional, Sequence, Union
 from typing_extensions import Literal
 
 from parsl.dataflow.dflow import DataFlowKernel
 
+from typing import Any, Callable, Dict
+
+from parsl.dataflow.futures import AppFuture
+
+
 logger = logging.getLogger(__name__)
 
 
@@ -22,7 +27,12 @@ class AppBase(metaclass=ABCMeta):
 
     """
 
-    def __init__(self, func, data_flow_kernel=None, executors='all', cache=False, ignore_for_cache=None):
+    @typeguard.typechecked
+    def __init__(self, func: Callable,
+                 data_flow_kernel: Optional[DataFlowKernel] = None,
+                 executors: Union[List[str], Literal['all']] = 'all',
+                 cache: bool = False,
+                 ignore_for_cache: Optional[Sequence[str]] = None) -> None:
         """Construct the App object.
 
         Args:
@@ -34,7 +44,7 @@ class AppBase(metaclass=ABCMeta):
                after calling :meth:`parsl.dataflow.dflow.DataFlowKernelLoader.load`.
              - executors (str|list) : Labels of the executors that this app can execute over. Default is 'all'.
              - cache (Bool) : Enable caching of this app ?
-             - ignore_for_cache (list|None): Names of arguments which will be ignored by the caching mechanism.
+             - ignore_for_cache (sequence|None): Names of arguments which will be ignored by the caching mechanism.
 
         Returns:
              - App object.
@@ -46,12 +56,10 @@ class AppBase(metaclass=ABCMeta):
         self.executors = executors
         self.cache = cache
         self.ignore_for_cache = ignore_for_cache
-        if not (isinstance(executors, list) or isinstance(executors, str)):
-            logger.error("App {} specifies invalid executor option, expects string or list".format(
-                func.__name__))
 
         params = signature(func).parameters
 
+        self.kwargs: Dict[str, Any]
         self.kwargs = {}
         if 'stdout' in params:
             self.kwargs['stdout'] = params['stdout'].default
@@ -65,16 +73,16 @@ class AppBase(metaclass=ABCMeta):
         self.inputs = params['inputs'].default if 'inputs' in params else []
 
     @abstractmethod
-    def __call__(self, *args, **kwargs):
+    def __call__(self, *args: Any, **kwargs: Any) -> AppFuture:
         pass
 
 
 @typeguard.typechecked
-def python_app(function=None,
+def python_app(function: Optional[Callable] = None,
                data_flow_kernel: Optional[DataFlowKernel] = None,
                cache: bool = False,
                executors: Union[List[str], Literal['all']] = 'all',
-               ignore_for_cache: Optional[List[str]] = None):
+               ignore_for_cache: Optional[Sequence[str]] = None) -> Callable:
     """Decorator function for making python apps.
 
     Parameters
@@ -91,13 +99,13 @@ def python_app(function=None,
         Labels of the executors that this app can execute over. Default is 'all'.
     cache : bool
         Enable caching of the app call. Default is False.
-    ignore_for_cache : (list|None)
+    ignore_for_cache : (sequence|None)
         Names of arguments which will be ignored by the caching mechanism.
     """
     from parsl.app.python import PythonApp
 
-    def decorator(func):
-        def wrapper(f):
+    def decorator(func: Callable) -> Callable:
+        def wrapper(f: Callable) -> PythonApp:
             return PythonApp(f,
                              data_flow_kernel=data_flow_kernel,
                              cache=cache,
@@ -111,10 +119,10 @@ def python_app(function=None,
 
 
 @typeguard.typechecked
-def join_app(function=None,
+def join_app(function: Optional[Callable] = None,
              data_flow_kernel: Optional[DataFlowKernel] = None,
              cache: bool = False,
-             ignore_for_cache: Optional[List[str]] = None):
+             ignore_for_cache: Optional[Sequence[str]] = None) -> Callable:
     """Decorator function for making join apps
 
     Parameters
@@ -129,13 +137,13 @@ def join_app(function=None,
         be omitted only after calling :meth:`parsl.dataflow.dflow.DataFlowKernelLoader.load`. Default is None.
     cache : bool
         Enable caching of the app call. Default is False.
-    ignore_for_cache : (list|None)
+    ignore_for_cache : (sequence|None)
         Names of arguments which will be ignored by the caching mechanism.
     """
     from parsl.app.python import PythonApp
 
-    def decorator(func):
-        def wrapper(f):
+    def decorator(func: Callable) -> Callable:
+        def wrapper(f: Callable) -> PythonApp:
             return PythonApp(f,
                              data_flow_kernel=data_flow_kernel,
                              cache=cache,
@@ -149,11 +157,11 @@ def join_app(function=None,
 
 
 @typeguard.typechecked
-def bash_app(function=None,
+def bash_app(function: Optional[Callable] = None,
              data_flow_kernel: Optional[DataFlowKernel] = None,
              cache: bool = False,
              executors: Union[List[str], Literal['all']] = 'all',
-             ignore_for_cache: Optional[List[str]] = None):
+             ignore_for_cache: Optional[Sequence[str]] = None) -> Callable:
     """Decorator function for making bash apps.
 
     Parameters
@@ -177,8 +185,8 @@ def bash_app(function=None,
     """
     from parsl.app.bash import BashApp
 
-    def decorator(func):
-        def wrapper(f):
+    def decorator(func: Callable) -> Callable:
+        def wrapper(f: Callable) -> BashApp:
             return BashApp(f,
                            data_flow_kernel=data_flow_kernel,
                            cache=cache,

{parsl-2023.10.16 → parsl-2023.11.13}/parsl/channels/base.py

@@ -4,33 +4,21 @@ from typing import Dict, Tuple
 
 
 class Channel(metaclass=ABCMeta):
-    """For certain resources such as campus clusters or supercomputers at
+    """Channels are abstractions that enable ExecutionProviders to talk to
+    resource managers of remote compute facilities.
+
+    For certain resources such as campus clusters or supercomputers at
     research laboratories, resource requirements may require authentication.
     For instance some resources may allow access to their job schedulers from
-    only their login-nodes which require you to authenticate on through SSH,
-    GSI-SSH and sometimes even require two factor authentication. Channels are
-    simple abstractions that enable the ExecutionProvider component to talk to
-    the resource managers of compute facilities. The simplest Channel,
-    *LocalChannel*, simply executes commands locally on a shell, while the
-    *SshChannel* authenticates you to remote systems.
-
-    Channels are usually called via the execute_wait function.
-    For channels that execute remotely, a push_file function allows you to copy over files.
-
-    .. code:: python
-
-                                +------------------
-                                |
-          cmd, wtime    ------->|  execute_wait
-          (ec, stdout, stderr)<-|---+
-                                |
-          src, dst_dir  ------->|  push_file
-             dst_path  <--------|----+
-                                |
-          dst_script_dir <------|  script_dir
-                                |
-                                +-------------------
+    only their login-nodes which require you to authenticate through SSH, or
+    require two factor authentication.
+
+    The simplest Channel, *LocalChannel*, executes commands locally in a
+    shell, while the *SSHChannel* authenticates you to remote systems.
 
+    Channels provide the ability to execute commands remotely, using the
+    execute_wait method, and manipulate the remote file system using methods
+    such as push_file, pull_file and makedirs.
 
     Channels should ensure that each launched command runs in a new process
     group, so that providers (such as AdHocProvider and LocalProvider) which

{parsl-2023.10.16 → parsl-2023.11.13}/parsl/config.py

@@ -1,7 +1,7 @@
 import logging
 import typeguard
 
-from typing import Callable, List, Optional, Sequence, Union
+from typing import Callable, Iterable, Optional, Sequence, Union
 from typing_extensions import Literal
 
 from parsl.utils import RepresentationMixin
@@ -20,9 +20,9 @@ class Config(RepresentationMixin):
 
     Parameters
     ----------
-    executors : list of ParslExecutor, optional
-        List of `ParslExecutor` instances to use for executing tasks.
-        Default is [:class:`~parsl.executors.threads.ThreadPoolExecutor()`].
+    executors : sequence of ParslExecutor, optional
+        List (or other iterable) of `ParslExecutor` instances to use for executing tasks.
+        Default is (:class:`~parsl.executors.threads.ThreadPoolExecutor()`,).
     app_cache : bool, optional
         Enable app caching. Default is True.
     checkpoint_files : sequence of str, optional
@@ -73,7 +73,7 @@ class Config(RepresentationMixin):
 
     @typeguard.typechecked
     def __init__(self,
-                 executors: Optional[List[ParslExecutor]] = None,
+                 executors: Optional[Iterable[ParslExecutor]] = None,
                  app_cache: bool = True,
                  checkpoint_files: Optional[Sequence[str]] = None,
                  checkpoint_mode: Union[None,
@@ -92,9 +92,14 @@ class Config(RepresentationMixin):
                  monitoring: Optional[MonitoringHub] = None,
                  usage_tracking: bool = False,
                  initialize_logging: bool = True) -> None:
-        if executors is None:
-            executors = [ThreadPoolExecutor()]
-        self.executors = executors
+
+        executors = tuple(executors or [])
+        if not executors:
+            executors = (ThreadPoolExecutor(),)
+
+        self._executors: Sequence[ParslExecutor] = executors
+        self._validate_executors()
+
         self.app_cache = app_cache
         self.checkpoint_files = checkpoint_files
         self.checkpoint_mode = checkpoint_mode
@@ -125,11 +130,13 @@ class Config(RepresentationMixin):
     def executors(self) -> Sequence[ParslExecutor]:
         return self._executors
 
-    @executors.setter
-    def executors(self, executors: Sequence[ParslExecutor]):
-        labels = [e.label for e in executors]
+    def _validate_executors(self) -> None:
+
+        if len(self.executors) == 0:
+            raise ConfigurationError('At least one executor must be specified')
+
+        labels = [e.label for e in self.executors]
         duplicates = [e for n, e in enumerate(labels) if e in labels[:n]]
         if len(duplicates) > 0:
             raise ConfigurationError('Executors must have unique labels ({})'.format(
                 ', '.join(['label={}'.format(repr(d)) for d in duplicates])))
-        self._executors = executors

{parsl-2023.10.16 → parsl-2023.11.13}/parsl/configs/ad_hoc.py

@@ -9,8 +9,8 @@ user_opts = {'adhoc':
              {'username': 'YOUR_USERNAME',
               'script_dir': 'YOUR_SCRIPT_DIR',
               'remote_hostnames': ['REMOTE_HOST_URL_1', 'REMOTE_HOST_URL_2']
+              }
              }
-}
 
 
 config = Config(
@@ -26,7 +26,7 @@ config = Config(
                 channels=[SSHChannel(hostname=m,
                                      username=user_opts['adhoc']['username'],
                                      script_dir=user_opts['adhoc']['script_dir'],
-                ) for m in user_opts['adhoc']['remote_hostnames']]
+                                     ) for m in user_opts['adhoc']['remote_hostnames']]
             )
         )
     ],

{parsl-2023.10.16 → parsl-2023.11.13}/parsl/dataflow/dflow.py

@@ -451,8 +451,8 @@ class DataFlowKernel:
             if isinstance(joinable, Future):
                 je = joinable.exception()
                 if je is not None:
-                    if hasattr(joinable, 'task_def'):
-                        tid = joinable.task_def['id']
+                    if hasattr(joinable, 'task_record'):
+                        tid = joinable.task_record['id']
                     else:
                         tid = None
                     exceptions_tids = [(je, tid)]
@@ -460,8 +460,8 @@ class DataFlowKernel:
                 for future in joinable:
                     je = future.exception()
                     if je is not None:
-                        if hasattr(joinable, 'task_def'):
-                            tid = joinable.task_def['id']
+                        if hasattr(joinable, 'task_record'):
+                            tid = joinable.task_record['id']
                         else:
                             tid = None
                         exceptions_tids.append((je, tid))
@@ -854,8 +854,8 @@ class DataFlowKernel:
                 try:
                     new_args.extend([dep.result()])
                 except Exception as e:
-                    if hasattr(dep, 'task_def'):
-                        tid = dep.task_def['id']
+                    if hasattr(dep, 'task_record'):
+                        tid = dep.task_record['id']
                     else:
                         tid = None
                     dep_failures.extend([(e, tid)])
@@ -869,8 +869,8 @@ class DataFlowKernel:
                 try:
                     kwargs[key] = dep.result()
                 except Exception as e:
-                    if hasattr(dep, 'task_def'):
-                        tid = dep.task_def['id']
+                    if hasattr(dep, 'task_record'):
+                        tid = dep.task_record['id']
                     else:
                         tid = None
                     dep_failures.extend([(e, tid)])
@@ -883,8 +883,8 @@ class DataFlowKernel:
                     try:
                         new_inputs.extend([dep.result()])
                     except Exception as e:
-                        if hasattr(dep, 'task_def'):
-                            tid = dep.task_def['id']
+                        if hasattr(dep, 'task_record'):
+                            tid = dep.task_record['id']
                         else:
                             tid = None
                         dep_failures.extend([(e, tid)])
@@ -918,7 +918,7 @@ class DataFlowKernel:
             - executors (list or string) : List of executors this call could go to.
                     Default='all'
             - cache (Bool) : To enable memoization or not
-            - ignore_for_cache (list) : List of kwargs to be ignored for memoization/checkpointing
+            - ignore_for_cache (sequence) : List of kwargs to be ignored for memoization/checkpointing
             - app_kwargs (dict) : Rest of the kwargs to the fn passed as dict.
 
         Returns:
@@ -967,38 +967,40 @@ class DataFlowKernel:
 
         resource_specification = app_kwargs.get('parsl_resource_specification', {})
 
-        task_def: TaskRecord
-        task_def = {'depends': [],
-                    'executor': executor,
-                    'func_name': func.__name__,
-                    'memoize': cache,
-                    'hashsum': None,
-                    'exec_fu': None,
-                    'fail_count': 0,
-                    'fail_cost': 0,
-                    'fail_history': [],
-                    'from_memo': None,
-                    'ignore_for_cache': ignore_for_cache,
-                    'join': join,
-                    'joins': None,
-                    'try_id': 0,
-                    'id': task_id,
-                    'time_invoked': datetime.datetime.now(),
-                    'time_returned': None,
-                    'try_time_launched': None,
-                    'try_time_returned': None,
-                    'resource_specification': resource_specification}
-
-        self.update_task_state(task_def, States.unsched)
-
-        app_fu = AppFuture(task_def)
+        task_record: TaskRecord
+        task_record = {'depends': [],
+                       'dfk': self,
+                       'executor': executor,
+                       'func_name': func.__name__,
+                       'memoize': cache,
+                       'hashsum': None,
+                       'exec_fu': None,
+                       'fail_count': 0,
+                       'fail_cost': 0,
+                       'fail_history': [],
+                       'from_memo': None,
+                       'ignore_for_cache': ignore_for_cache,
+                       'join': join,
+                       'joins': None,
+                       'try_id': 0,
+                       'id': task_id,
+                       'task_launch_lock': threading.Lock(),
+                       'time_invoked': datetime.datetime.now(),
+                       'time_returned': None,
+                       'try_time_launched': None,
+                       'try_time_returned': None,
+                       'resource_specification': resource_specification}
+
+        self.update_task_state(task_record, States.unsched)
+
+        app_fu = AppFuture(task_record)
 
         # Transform remote input files to data futures
         app_args, app_kwargs, func = self._add_input_deps(executor, app_args, app_kwargs, func)
 
         func = self._add_output_deps(executor, app_args, app_kwargs, app_fu, func)
 
-        task_def.update({
+        task_record.update({
                     'args': app_args,
                     'func': func,
                     'kwargs': app_kwargs,
@@ -1006,11 +1008,11 @@ class DataFlowKernel:
 
         assert task_id not in self.tasks
 
-        self.tasks[task_id] = task_def
+        self.tasks[task_id] = task_record
 
         # Get the list of dependencies for the task
         depends = self._gather_all_deps(app_args, app_kwargs)
-        task_def['depends'] = depends
+        task_record['depends'] = depends
 
         depend_descs = []
         for d in depends:
@@ -1025,16 +1027,14 @@ class DataFlowKernel:
             waiting_message = "not waiting on any dependency"
 
         logger.info("Task {} submitted for App {}, {}".format(task_id,
-                                                              task_def['func_name'],
+                                                              task_record['func_name'],
                                                               waiting_message))
 
-        task_def['task_launch_lock'] = threading.Lock()
+        app_fu.add_done_callback(partial(self.handle_app_update, task_record))
+        self.update_task_state(task_record, States.pending)
+        logger.debug("Task {} set to pending state with AppFuture: {}".format(task_id, task_record['app_fu']))
 
-        app_fu.add_done_callback(partial(self.handle_app_update, task_def))
-        self.update_task_state(task_def, States.pending)
-        logger.debug("Task {} set to pending state with AppFuture: {}".format(task_id, task_def['app_fu']))
-
-        self._send_task_log_info(task_def)
+        self._send_task_log_info(task_record)
 
         # at this point add callbacks to all dependencies to do a launch_if_ready
         # call whenever a dependency completes.
@@ -1051,14 +1051,14 @@ class DataFlowKernel:
         for d in depends:
 
             def callback_adapter(dep_fut: Future) -> None:
-                self.launch_if_ready(task_def)
+                self.launch_if_ready(task_record)
 
             try:
                 d.add_done_callback(callback_adapter)
             except Exception as e:
                 logger.error("add_done_callback got an exception {} which will be ignored".format(e))
 
-        self.launch_if_ready(task_def)
+        self.launch_if_ready(task_record)
 
         return app_fu
 
@@ -1090,7 +1090,14 @@ class DataFlowKernel:
         """
         run_dir = self.run_dir
         if channel.script_dir is None:
-            channel.script_dir = os.path.join(run_dir, 'submit_scripts')
+
+            # This case will be detected as unreachable by mypy, because of
+            # the type of script_dir, which is str, not Optional[str].
+            # The type system doesn't represent the initialized/uninitialized
+            # state of a channel so cannot represent that a channel needs
+            # its script directory set or not.
+
+            channel.script_dir = os.path.join(run_dir, 'submit_scripts')  # type: ignore[unreachable]
 
             # Only create dirs if we aren't on a shared-fs
             if not channel.isdir(run_dir):

{parsl-2023.10.16 → parsl-2023.11.13}/parsl/dataflow/futures.py

@@ -5,11 +5,14 @@ We have two basic types of futures:
     2. AppFutures which represent the futures on App/Leaf tasks.
 
 """
+from __future__ import annotations
 
 from concurrent.futures import Future
 import logging
 import threading
-from typing import Optional, Sequence
+from typing import Any, Optional, Sequence
+
+import parsl.app.app as app
 
 from parsl.app.futures import DataFuture
 from parsl.dataflow.taskrecord import TaskRecord
@@ -59,32 +62,32 @@ class AppFuture(Future):
 
     """
 
-    def __init__(self, task_def: TaskRecord) -> None:
+    def __init__(self, task_record: TaskRecord) -> None:
         """Initialize the AppFuture.
 
         Args:
 
         KWargs:
-             - task_def : The DFK task definition dictionary for the task represented
-                   by this future.
+             - task_record : The TaskRecord for the task represented by
+               this future.
         """
         super().__init__()
         self._update_lock = threading.Lock()
         self._outputs: Sequence[DataFuture]
         self._outputs = []
-        self.task_def = task_def
+        self.task_record = task_record
 
     @property
     def stdout(self) -> Optional[str]:
-        return self.task_def['kwargs'].get('stdout')
+        return self.task_record['kwargs'].get('stdout')
 
     @property
     def stderr(self) -> Optional[str]:
-        return self.task_def['kwargs'].get('stderr')
+        return self.task_record['kwargs'].get('stderr')
 
     @property
     def tid(self) -> int:
-        return self.task_def['id']
+        return self.task_record['id']
 
     def cancel(self) -> bool:
         raise NotImplementedError("Cancel not implemented")
@@ -113,8 +116,35 @@ class AppFuture(Future):
 
            Returns: str
         """
-        return self.task_def['status'].name
+        return self.task_record['status'].name
 
     @property
     def outputs(self) -> Sequence[DataFuture]:
         return self._outputs
+
+    def __getitem__(self, key: Any) -> AppFuture:
+        # This is decorated on each invocation because the getitem task
+        # should be bound to the same DFK as the task associated with this
+        # Future.
+        deferred_getitem_app = app.python_app(deferred_getitem, executors=['_parsl_internal'], data_flow_kernel=self.task_record['dfk'])
+
+        return deferred_getitem_app(self, key)
+
+    def __getattr__(self, name: str) -> AppFuture:
+        # this will avoid lifting behaviour on private methods and attributes,
+        # including __double_underscore__ methods which implement other
+        # Python syntax (such as iterators in for loops)
+        if name.startswith("_"):
+            raise AttributeError()
+
+        deferred_getattr_app = app.python_app(deferred_getattr, executors=['_parsl_internal'], data_flow_kernel=self.task_record['dfk'])
+
+        return deferred_getattr_app(self, name)
+
+
+def deferred_getitem(o: Any, k: Any) -> Any:
+    return o[k]
+
+
+def deferred_getattr(o: Any, name: str) -> Any:
+    return getattr(o, name)

{parsl-2023.10.16 → parsl-2023.11.13}/parsl/dataflow/taskrecord.py

@@ -5,18 +5,25 @@ import datetime
 from typing_extensions import TypedDict
 from concurrent.futures import Future
 
+
 # only for type checking:
 from typing import Any, Callable, Dict, Optional, List, Sequence, TYPE_CHECKING, Union
 
 if TYPE_CHECKING:
     from parsl.dataflow.futures import AppFuture
 
+import parsl.dataflow.dflow as dflow
+
 from parsl.dataflow.states import States
 
 
 class TaskRecord(TypedDict, total=False):
     """This stores most information about a Parsl task"""
 
+    dfk: dflow.DataFlowKernel
+    """The DataFlowKernel which is managing this task.
+    """
+
     func_name: str
 
     status: States

{parsl-2023.10.16 → parsl-2023.11.13}/parsl/executors/base.py

@@ -74,7 +74,7 @@ class ParslExecutor(metaclass=ABCMeta):
         pass
 
     @abstractmethod
-    def shutdown(self) -> bool:
+    def shutdown(self) -> None:
         """Shutdown the executor.
 
         This includes all attached resources such as workers and controllers.

{parsl-2023.10.16 → parsl-2023.11.13}/parsl/executors/high_throughput/executor.py

@@ -479,10 +479,10 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin):
                                                     "heartbeat_threshold": self.heartbeat_threshold,
                                                     "poll_period": self.poll_period,
                                                     "logging_level": logging.DEBUG if self.worker_debug else logging.INFO
-                                            },
+                                                    },
                                             daemon=True,
                                             name="HTEX-Interchange"
-        )
+                                            )
         self.interchange_proc.start()
         try:
             (self.worker_task_port, self.worker_result_port) = comm_q.get(block=True, timeout=120)