parsl 2023.10.23__py3-none-any.whl → 2023.11.20__py3-none-any.whl

parsl/__init__.py

@@ -62,6 +62,7 @@ __all__ = [
     'ThreadPoolExecutor',
     'HighThroughputExecutor',
     'WorkQueueExecutor',
+    'RadicalPilotExecutor',
 
     # monitoring
     'MonitoringHub',

parsl/app/app.py

@@ -6,11 +6,16 @@ import logging
 import typeguard
 from abc import ABCMeta, abstractmethod
 from inspect import signature
-from typing import List, Optional, Union
+from typing import List, Optional, Sequence, Union
 from typing_extensions import Literal
 
 from parsl.dataflow.dflow import DataFlowKernel
 
+from typing import Any, Callable, Dict
+
+from parsl.dataflow.futures import AppFuture
+
+
 logger = logging.getLogger(__name__)
 
 
@@ -22,7 +27,12 @@ class AppBase(metaclass=ABCMeta):
 
     """
 
-    def __init__(self, func, data_flow_kernel=None, executors='all', cache=False, ignore_for_cache=None):
+    @typeguard.typechecked
+    def __init__(self, func: Callable,
+                 data_flow_kernel: Optional[DataFlowKernel] = None,
+                 executors: Union[List[str], Literal['all']] = 'all',
+                 cache: bool = False,
+                 ignore_for_cache: Optional[Sequence[str]] = None) -> None:
         """Construct the App object.
 
         Args:
@@ -34,7 +44,7 @@ class AppBase(metaclass=ABCMeta):
                after calling :meth:`parsl.dataflow.dflow.DataFlowKernelLoader.load`.
              - executors (str|list) : Labels of the executors that this app can execute over. Default is 'all'.
              - cache (Bool) : Enable caching of this app ?
-             - ignore_for_cache (list|None): Names of arguments which will be ignored by the caching mechanism.
+             - ignore_for_cache (sequence|None): Names of arguments which will be ignored by the caching mechanism.
 
         Returns:
              - App object.
@@ -46,12 +56,10 @@ class AppBase(metaclass=ABCMeta):
         self.executors = executors
         self.cache = cache
         self.ignore_for_cache = ignore_for_cache
-        if not (isinstance(executors, list) or isinstance(executors, str)):
-            logger.error("App {} specifies invalid executor option, expects string or list".format(
-                func.__name__))
 
         params = signature(func).parameters
 
+        self.kwargs: Dict[str, Any]
         self.kwargs = {}
         if 'stdout' in params:
             self.kwargs['stdout'] = params['stdout'].default
@@ -65,16 +73,16 @@ class AppBase(metaclass=ABCMeta):
         self.inputs = params['inputs'].default if 'inputs' in params else []
 
     @abstractmethod
-    def __call__(self, *args, **kwargs):
+    def __call__(self, *args: Any, **kwargs: Any) -> AppFuture:
         pass
 
 
 @typeguard.typechecked
-def python_app(function=None,
+def python_app(function: Optional[Callable] = None,
                data_flow_kernel: Optional[DataFlowKernel] = None,
                cache: bool = False,
                executors: Union[List[str], Literal['all']] = 'all',
-               ignore_for_cache: Optional[List[str]] = None):
+               ignore_for_cache: Optional[Sequence[str]] = None) -> Callable:
     """Decorator function for making python apps.
 
     Parameters
@@ -91,13 +99,13 @@ def python_app(function=None,
         Labels of the executors that this app can execute over. Default is 'all'.
     cache : bool
         Enable caching of the app call. Default is False.
-    ignore_for_cache : (list|None)
+    ignore_for_cache : (sequence|None)
         Names of arguments which will be ignored by the caching mechanism.
     """
     from parsl.app.python import PythonApp
 
-    def decorator(func):
-        def wrapper(f):
+    def decorator(func: Callable) -> Callable:
+        def wrapper(f: Callable) -> PythonApp:
             return PythonApp(f,
                              data_flow_kernel=data_flow_kernel,
                              cache=cache,
@@ -111,10 +119,10 @@ def python_app(function=None,
 
 
 @typeguard.typechecked
-def join_app(function=None,
+def join_app(function: Optional[Callable] = None,
              data_flow_kernel: Optional[DataFlowKernel] = None,
              cache: bool = False,
-             ignore_for_cache: Optional[List[str]] = None):
+             ignore_for_cache: Optional[Sequence[str]] = None) -> Callable:
     """Decorator function for making join apps
 
     Parameters
@@ -129,13 +137,13 @@ def join_app(function=None,
         be omitted only after calling :meth:`parsl.dataflow.dflow.DataFlowKernelLoader.load`. Default is None.
     cache : bool
         Enable caching of the app call. Default is False.
-    ignore_for_cache : (list|None)
+    ignore_for_cache : (sequence|None)
         Names of arguments which will be ignored by the caching mechanism.
     """
     from parsl.app.python import PythonApp
 
-    def decorator(func):
-        def wrapper(f):
+    def decorator(func: Callable) -> Callable:
+        def wrapper(f: Callable) -> PythonApp:
             return PythonApp(f,
                              data_flow_kernel=data_flow_kernel,
                              cache=cache,
@@ -149,11 +157,11 @@ def join_app(function=None,
 
 
 @typeguard.typechecked
-def bash_app(function=None,
+def bash_app(function: Optional[Callable] = None,
              data_flow_kernel: Optional[DataFlowKernel] = None,
              cache: bool = False,
              executors: Union[List[str], Literal['all']] = 'all',
-             ignore_for_cache: Optional[List[str]] = None):
+             ignore_for_cache: Optional[Sequence[str]] = None) -> Callable:
     """Decorator function for making bash apps.
 
     Parameters
@@ -177,8 +185,8 @@ def bash_app(function=None,
     """
     from parsl.app.bash import BashApp
 
-    def decorator(func):
-        def wrapper(f):
+    def decorator(func: Callable) -> Callable:
+        def wrapper(f: Callable) -> BashApp:
             return BashApp(f,
                            data_flow_kernel=data_flow_kernel,
                            cache=cache,

parsl/channels/base.py

@@ -4,33 +4,21 @@ from typing import Dict, Tuple
 
 
 class Channel(metaclass=ABCMeta):
-    """For certain resources such as campus clusters or supercomputers at
+    """Channels are abstractions that enable ExecutionProviders to talk to
+    resource managers of remote compute facilities.
+
+    For certain resources such as campus clusters or supercomputers at
     research laboratories, resource requirements may require authentication.
     For instance some resources may allow access to their job schedulers from
-    only their login-nodes which require you to authenticate on through SSH,
-    GSI-SSH and sometimes even require two factor authentication. Channels are
-    simple abstractions that enable the ExecutionProvider component to talk to
-    the resource managers of compute facilities. The simplest Channel,
-    *LocalChannel*, simply executes commands locally on a shell, while the
-    *SshChannel* authenticates you to remote systems.
-
-    Channels are usually called via the execute_wait function.
-    For channels that execute remotely, a push_file function allows you to copy over files.
-
-    .. code:: python
-
-                                +------------------
-                                |
-          cmd, wtime    ------->|  execute_wait
-          (ec, stdout, stderr)<-|---+
-                                |
-          src, dst_dir  ------->|  push_file
-             dst_path  <--------|----+
-                                |
-          dst_script_dir <------|  script_dir
-                                |
-                                +-------------------
+    only their login-nodes which require you to authenticate through SSH, or
+    require two factor authentication.
+
+    The simplest Channel, *LocalChannel*, executes commands locally in a
+    shell, while the *SSHChannel* authenticates you to remote systems.
 
+    Channels provide the ability to execute commands remotely, using the
+    execute_wait method, and manipulate the remote file system using methods
+    such as push_file, pull_file and makedirs.
 
     Channels should ensure that each launched command runs in a new process
     group, so that providers (such as AdHocProvider and LocalProvider) which

parsl/config.py

@@ -1,7 +1,7 @@
 import logging
 import typeguard
 
-from typing import Callable, List, Optional, Sequence, Union
+from typing import Callable, Iterable, Optional, Sequence, Union
 from typing_extensions import Literal
 
 from parsl.utils import RepresentationMixin
@@ -20,9 +20,9 @@ class Config(RepresentationMixin):
 
     Parameters
     ----------
-    executors : list of ParslExecutor, optional
-        List of `ParslExecutor` instances to use for executing tasks.
-        Default is [:class:`~parsl.executors.threads.ThreadPoolExecutor()`].
+    executors : sequence of ParslExecutor, optional
+        List (or other iterable) of `ParslExecutor` instances to use for executing tasks.
+        Default is (:class:`~parsl.executors.threads.ThreadPoolExecutor()`,).
     app_cache : bool, optional
         Enable app caching. Default is True.
     checkpoint_files : sequence of str, optional
@@ -73,7 +73,7 @@ class Config(RepresentationMixin):
 
     @typeguard.typechecked
     def __init__(self,
-                 executors: Optional[List[ParslExecutor]] = None,
+                 executors: Optional[Iterable[ParslExecutor]] = None,
                  app_cache: bool = True,
                  checkpoint_files: Optional[Sequence[str]] = None,
                  checkpoint_mode: Union[None,
@@ -92,9 +92,14 @@ class Config(RepresentationMixin):
                  monitoring: Optional[MonitoringHub] = None,
                  usage_tracking: bool = False,
                  initialize_logging: bool = True) -> None:
-        if executors is None:
-            executors = [ThreadPoolExecutor()]
-        self.executors = executors
+
+        executors = tuple(executors or [])
+        if not executors:
+            executors = (ThreadPoolExecutor(),)
+
+        self._executors: Sequence[ParslExecutor] = executors
+        self._validate_executors()
+
         self.app_cache = app_cache
         self.checkpoint_files = checkpoint_files
         self.checkpoint_mode = checkpoint_mode
@@ -125,11 +130,13 @@ class Config(RepresentationMixin):
     def executors(self) -> Sequence[ParslExecutor]:
         return self._executors
 
-    @executors.setter
-    def executors(self, executors: Sequence[ParslExecutor]):
-        labels = [e.label for e in executors]
+    def _validate_executors(self) -> None:
+
+        if len(self.executors) == 0:
+            raise ConfigurationError('At least one executor must be specified')
+
+        labels = [e.label for e in self.executors]
         duplicates = [e for n, e in enumerate(labels) if e in labels[:n]]
         if len(duplicates) > 0:
             raise ConfigurationError('Executors must have unique labels ({})'.format(
                 ', '.join(['label={}'.format(repr(d)) for d in duplicates])))
-        self._executors = executors

parsl/configs/ad_hoc.py

@@ -9,8 +9,8 @@ user_opts = {'adhoc':
              {'username': 'YOUR_USERNAME',
               'script_dir': 'YOUR_SCRIPT_DIR',
               'remote_hostnames': ['REMOTE_HOST_URL_1', 'REMOTE_HOST_URL_2']
+              }
              }
-}
 
 
 config = Config(
@@ -26,7 +26,7 @@ config = Config(
                 channels=[SSHChannel(hostname=m,
                                      username=user_opts['adhoc']['username'],
                                      script_dir=user_opts['adhoc']['script_dir'],
-                ) for m in user_opts['adhoc']['remote_hostnames']]
+                                     ) for m in user_opts['adhoc']['remote_hostnames']]
             )
         )
     ],

parsl/dataflow/dflow.py

@@ -918,7 +918,7 @@ class DataFlowKernel:
             - executors (list or string) : List of executors this call could go to.
                     Default='all'
             - cache (Bool) : To enable memoization or not
-            - ignore_for_cache (list) : List of kwargs to be ignored for memoization/checkpointing
+            - ignore_for_cache (sequence) : List of kwargs to be ignored for memoization/checkpointing
             - app_kwargs (dict) : Rest of the kwargs to the fn passed as dict.
 
         Returns:
@@ -984,6 +984,7 @@ class DataFlowKernel:
                        'joins': None,
                        'try_id': 0,
                        'id': task_id,
+                       'task_launch_lock': threading.Lock(),
                        'time_invoked': datetime.datetime.now(),
                        'time_returned': None,
                        'try_time_launched': None,
@@ -1029,8 +1030,6 @@ class DataFlowKernel:
                                                               task_record['func_name'],
                                                               waiting_message))
 
-        task_record['task_launch_lock'] = threading.Lock()
-
         app_fu.add_done_callback(partial(self.handle_app_update, task_record))
         self.update_task_state(task_record, States.pending)
         logger.debug("Task {} set to pending state with AppFuture: {}".format(task_id, task_record['app_fu']))
@@ -1091,7 +1090,14 @@ class DataFlowKernel:
         """
         run_dir = self.run_dir
         if channel.script_dir is None:
-            channel.script_dir = os.path.join(run_dir, 'submit_scripts')
+
+            # This case will be detected as unreachable by mypy, because of
+            # the type of script_dir, which is str, not Optional[str].
+            # The type system doesn't represent the initialized/uninitialized
+            # state of a channel so cannot represent that a channel needs
+            # its script directory set or not.
+
+            channel.script_dir = os.path.join(run_dir, 'submit_scripts')  # type: ignore[unreachable]
 
             # Only create dirs if we aren't on a shared-fs
             if not channel.isdir(run_dir):

parsl/executors/base.py

@@ -5,8 +5,6 @@ from typing_extensions import Literal, Self
 
 from parsl.jobs.states import JobStatus
 
-import parsl  # noqa F401
-
 
 class ParslExecutor(metaclass=ABCMeta):
     """Executors are abstractions that represent available compute resources
@@ -74,7 +72,7 @@ class ParslExecutor(metaclass=ABCMeta):
         pass
 
     @abstractmethod
-    def shutdown(self) -> bool:
+    def shutdown(self) -> None:
         """Shutdown the executor.
 
         This includes all attached resources such as workers and controllers.

parsl/executors/high_throughput/executor.py

@@ -8,7 +8,7 @@ import datetime
 import pickle
 import warnings
 from multiprocessing import Queue
-from typing import Dict, Sequence  # noqa F401 (used in type annotation)
+from typing import Dict, Sequence
 from typing import List, Optional, Tuple, Union, Callable
 import math
 
@@ -479,10 +479,10 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin):
                                                     "heartbeat_threshold": self.heartbeat_threshold,
                                                     "poll_period": self.poll_period,
                                                     "logging_level": logging.DEBUG if self.worker_debug else logging.INFO
-                                            },
+                                                    },
                                             daemon=True,
                                             name="HTEX-Interchange"
-        )
+                                            )
         self.interchange_proc.start()
         try:
             (self.worker_task_port, self.worker_result_port) = comm_q.get(block=True, timeout=120)