parsl 2024.5.27__py3-none-any.whl → 2024.6.10__py3-none-any.whl

parsl/__init__.py

@@ -15,24 +15,23 @@ AUTO_LOGNAME
 
 """
 import logging
+import multiprocessing as _multiprocessing
 import os
 import platform
 
-from parsl.version import VERSION
 from parsl.app.app import bash_app, join_app, python_app
 from parsl.config import Config
-from parsl.executors import ThreadPoolExecutor
-from parsl.executors import HighThroughputExecutor
-from parsl.executors import WorkQueueExecutor
-from parsl.log_utils import set_stream_logger
-from parsl.log_utils import set_file_logger
-from parsl.monitoring import MonitoringHub
-
 from parsl.data_provider.files import File
-
 from parsl.dataflow.dflow import DataFlowKernel, DataFlowKernelLoader
+from parsl.executors import (
+    HighThroughputExecutor,
+    ThreadPoolExecutor,
+    WorkQueueExecutor,
+)
+from parsl.log_utils import set_file_logger, set_stream_logger
+from parsl.monitoring import MonitoringHub
+from parsl.version import VERSION
 
-import multiprocessing as _multiprocessing
 if platform.system() == 'Darwin':
     _multiprocessing.set_start_method('fork', force=True)
 

parsl/addresses.py

@@ -8,17 +8,19 @@ so some experimentation will probably be needed to choose the correct one.
 
 import logging
 import platform
-import requests
 import socket
+
+import requests
+
 try:
     import fcntl
 except ImportError:
     fcntl = None  # type: ignore[assignment]
 import struct
-import typeguard
-import psutil
+from typing import Callable, List, Set
 
-from typing import Set, List, Callable
+import psutil
+import typeguard
 
 logger = logging.getLogger(__name__)
 

parsl/app/app.py

@@ -3,19 +3,16 @@
 The App class encapsulates a generic leaf task that can be executed asynchronously.
 """
 import logging
-import typeguard
 from abc import ABCMeta, abstractmethod
 from inspect import signature
-from typing import List, Optional, Sequence, Union
+from typing import Any, Callable, Dict, List, Optional, Sequence, Union
+
+import typeguard
 from typing_extensions import Literal
 
 from parsl.dataflow.dflow import DataFlowKernel
-
-from typing import Any, Callable, Dict
-
 from parsl.dataflow.futures import AppFuture
 
-
 logger = logging.getLogger(__name__)
 
 

parsl/app/bash.py

@@ -1,10 +1,9 @@
-from functools import update_wrapper
-from functools import partial
-from inspect import signature, Parameter
 import logging
+from functools import partial, update_wrapper
+from inspect import Parameter, signature
 
-from parsl.app.errors import wrap_error
 from parsl.app.app import AppBase
+from parsl.app.errors import wrap_error
 from parsl.data_provider.files import File
 from parsl.dataflow.dflow import DataFlowKernelLoader
 
@@ -17,6 +16,7 @@ def remote_side_bash_executor(func, *args, **kwargs):
     """
     import os
     import subprocess
+
     import parsl.app.errors as pe
     from parsl.utils import get_std_fname_mode
 

parsl/app/errors.py

@@ -1,10 +1,11 @@
 """Exceptions raised by Apps."""
+import logging
 from functools import wraps
-from typing import Callable, List, Optional, TypeVar, Union
-from typing_extensions import ParamSpec
 from types import TracebackType
-import logging
+from typing import Callable, List, Optional, TypeVar, Union
+
 from tblib import Traceback
+from typing_extensions import ParamSpec
 
 from parsl.data_provider.files import File
 from parsl.errors import ParslError
@@ -134,6 +135,7 @@ def wrap_error(func: Callable[P, R]) -> Callable[P, Union[R, RemoteExceptionWrap
     @wraps(func)
     def wrapper(*args: P.args, **kwargs: P.kwargs) -> Union[R, RemoteExceptionWrapper]:
         import sys
+
         from parsl.app.errors import RemoteExceptionWrapper
         try:
             return func(*args, **kwargs)

parsl/app/futures.py

@@ -1,12 +1,12 @@
 """This module implements DataFutures.
 """
 import logging
-import typeguard
 from concurrent.futures import Future
+from typing import Optional
 
-from parsl.data_provider.files import File
+import typeguard
 
-from typing import Optional
+from parsl.data_provider.files import File
 
 logger = logging.getLogger(__name__)
 

parsl/app/python.py

@@ -2,6 +2,7 @@ import logging
 import threading
 
 import tblib.pickling_support
+
 tblib.pickling_support.install()
 
 from functools import wraps
@@ -11,7 +12,6 @@ from parsl.app.errors import wrap_error
 from parsl.dataflow.dflow import DataFlowKernelLoader
 from parsl.utils import AutoCancelTimer
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -19,6 +19,7 @@ def timeout(f, seconds: float):
     @wraps(f)
     def wrapper(*args, **kwargs):
         import ctypes
+
         import parsl.app.errors
 
         def inject_exception(thread):

parsl/benchmark/perf.py

@@ -1,7 +1,8 @@
 import argparse
+import concurrent.futures
 import importlib
 import time
-import concurrent.futures
+
 import parsl
 
 min_iterations = 2

parsl/channels/__init__.py

@@ -1,7 +1,7 @@
 from parsl.channels.base import Channel
-from parsl.channels.ssh.ssh import SSHChannel
 from parsl.channels.local.local import LocalChannel
-from parsl.channels.ssh_il.ssh_il import SSHInteractiveLoginChannel
 from parsl.channels.oauth_ssh.oauth_ssh import OAuthSSHChannel
+from parsl.channels.ssh.ssh import SSHChannel
+from parsl.channels.ssh_il.ssh_il import SSHInteractiveLoginChannel
 
 __all__ = ['Channel', 'SSHChannel', 'LocalChannel', 'SSHInteractiveLoginChannel', 'OAuthSSHChannel']

parsl/channels/base.py

@@ -1,5 +1,4 @@
 from abc import ABCMeta, abstractmethod, abstractproperty
-
 from typing import Dict, Tuple
 
 

parsl/channels/errors.py

@@ -1,8 +1,9 @@
 ''' Exceptions raise by Apps.
 '''
-from parsl.errors import ParslError
 from typing import Optional
 
+from parsl.errors import ParslError
+
 
 class ChannelError(ParslError):
     """ Base class for all exceptions

parsl/channels/oauth_ssh/oauth_ssh.py

@@ -1,13 +1,14 @@
 import logging
-import paramiko
 import socket
 
-from parsl.errors import OptionalModuleMissing
+import paramiko
+
 from parsl.channels.ssh.ssh import SSHChannel
+from parsl.errors import OptionalModuleMissing
 
 try:
-    from oauth_ssh.ssh_service import SSHService
     from oauth_ssh.oauth_ssh_token import find_access_token
+    from oauth_ssh.ssh_service import SSHService
     _oauth_ssh_enabled = True
 except (ImportError, NameError):
     _oauth_ssh_enabled = False

parsl/channels/ssh/ssh.py

@@ -3,8 +3,16 @@ import logging
 import os
 
 import paramiko
+
 from parsl.channels.base import Channel
-from parsl.channels.errors import BadHostKeyException, AuthException, SSHException, BadScriptPath, BadPermsScriptPath, FileCopyException
+from parsl.channels.errors import (
+    AuthException,
+    BadHostKeyException,
+    BadPermsScriptPath,
+    BadScriptPath,
+    FileCopyException,
+    SSHException,
+)
 from parsl.utils import RepresentationMixin
 
 logger = logging.getLogger(__name__)

parsl/channels/ssh_il/ssh_il.py

@@ -2,6 +2,7 @@ import getpass
 import logging
 
 import paramiko
+
 from parsl.channels.ssh.ssh import SSHChannel
 
 logger = logging.getLogger(__name__)

parsl/concurrent/__init__.py

@@ -1,8 +1,8 @@
 """Interfaces modeled after Python's `concurrent library <https://docs.python.org/3/library/concurrent.html>`_"""
-from typing import Callable, Dict, Optional, Iterator, Iterable
+import time
 from concurrent.futures import Executor
+from typing import Callable, Dict, Iterable, Iterator, Optional
 from warnings import warn
-import time
 
 from parsl import Config, DataFlowKernel
 from parsl.app.python import PythonApp

parsl/config.py

@@ -1,17 +1,19 @@
 import logging
-import typeguard
-
 from typing import Callable, Iterable, Optional, Sequence, Union
+
+import typeguard
 from typing_extensions import Literal
 
-from parsl.utils import RepresentationMixin
 from parsl.dataflow.dependency_resolvers import DependencyResolver
+from parsl.dataflow.taskrecord import TaskRecord
+from parsl.errors import ConfigurationError
 from parsl.executors.base import ParslExecutor
 from parsl.executors.threads import ThreadPoolExecutor
-from parsl.errors import ConfigurationError
-from parsl.dataflow.taskrecord import TaskRecord
 from parsl.monitoring import MonitoringHub
 from parsl.usage_tracking.api import UsageInformation
+from parsl.usage_tracking.levels import DISABLED as USAGE_TRACKING_DISABLED
+from parsl.usage_tracking.levels import LEVEL_3 as USAGE_TRACKING_LEVEL_3
+from parsl.utils import RepresentationMixin
 
 logger = logging.getLogger(__name__)
 
@@ -38,6 +40,15 @@ class Config(RepresentationMixin, UsageInformation):
         ``checkpoint_mode='periodic'``.
     dependency_resolver: plugin point for custom dependency resolvers. Default: only resolve Futures,
         using the `SHALLOW_DEPENDENCY_RESOLVER`.
+    exit_mode: str, optional
+        When Parsl is used as a context manager (using ``with parsl.load`` syntax) then this parameter
+        controls what will happen to running tasks and exceptions at exit. The options are:
+
+        * ``cleanup``: cleanup the DFK on exit without waiting for any tasks
+        * ``skip``: skip all shutdown behaviour when exiting the context manager
+        * ``wait``: wait for all tasks to complete when exiting normally, but exit immediately when exiting due to an exception.
+
+        Default is ``cleanup``.
     garbage_collect : bool. optional.
         Delete task records from DFK when tasks have completed. Default: True
     internal_tasks_max_threads : int, optional
@@ -66,9 +77,12 @@ class Config(RepresentationMixin, UsageInformation):
         How often the scaling strategy should be executed. Default is 5 seconds.
     max_idletime : float, optional
         The maximum idle time allowed for an executor before strategy could shut down unused blocks. Default is 120.0 seconds.
-    usage_tracking : bool, optional
-        Set this field to True to opt-in to Parsl's usage tracking system. Parsl only collects minimal, non personally-identifiable,
-        information used for reporting to our funding agencies. Default is False.
+    usage_tracking : int, optional
+        Set this field to 1, 2, or 3 to opt-in to Parsl's usage tracking system.
+        The value represents the level of usage tracking detail to be collected.
+        Setting this field to 0 will disable usage tracking. Default (this field is not set): usage tracking is not enabled.
+        Parsl only collects minimal, non personally-identifiable,
+        information used for reporting to our funding agencies.
     initialize_logging : bool, optional
         Make DFK optionally not initialize any logging. Log messages
         will still be passed into the python logging system under the
@@ -92,6 +106,7 @@ class Config(RepresentationMixin, UsageInformation):
                                         Literal['manual']] = None,
                  checkpoint_period: Optional[str] = None,
                  dependency_resolver: Optional[DependencyResolver] = None,
+                 exit_mode: Literal['cleanup', 'skip', 'wait'] = 'cleanup',
                  garbage_collect: bool = True,
                  internal_tasks_max_threads: int = 10,
                  retries: int = 0,
@@ -102,7 +117,7 @@ class Config(RepresentationMixin, UsageInformation):
                  strategy_period: Union[float, int] = 5,
                  max_idletime: float = 120.0,
                  monitoring: Optional[MonitoringHub] = None,
-                 usage_tracking: bool = False,
+                 usage_tracking: int = 0,
                  initialize_logging: bool = True) -> None:
 
         executors = tuple(executors or [])
@@ -128,6 +143,7 @@ class Config(RepresentationMixin, UsageInformation):
             checkpoint_period = "00:30:00"
         self.checkpoint_period = checkpoint_period
         self.dependency_resolver = dependency_resolver
+        self.exit_mode = exit_mode
         self.garbage_collect = garbage_collect
         self.internal_tasks_max_threads = internal_tasks_max_threads
         self.retries = retries
@@ -136,6 +152,7 @@ class Config(RepresentationMixin, UsageInformation):
         self.strategy = strategy
         self.strategy_period = strategy_period
         self.max_idletime = max_idletime
+        self.validate_usage_tracking(usage_tracking)
         self.usage_tracking = usage_tracking
         self.initialize_logging = initialize_logging
         self.monitoring = monitoring
@@ -156,6 +173,12 @@ class Config(RepresentationMixin, UsageInformation):
             raise ConfigurationError('Executors must have unique labels ({})'.format(
                 ', '.join(['label={}'.format(repr(d)) for d in duplicates])))
 
+    def validate_usage_tracking(self, level: int) -> None:
+        if not USAGE_TRACKING_DISABLED <= level <= USAGE_TRACKING_LEVEL_3:
+            raise ConfigurationError(
+                f"Usage Tracking values must be 0, 1, 2, or 3 and not {level}"
+            )
+
     def get_usage_information(self):
         return {"executors_len": len(self.executors),
                 "dependency_resolver": self.dependency_resolver is not None}

parsl/configs/ASPIRE1.py

@@ -1,9 +1,9 @@
-from parsl.providers import PBSProProvider
-from parsl.launchers import MpiRunLauncher
+from parsl.addresses import address_by_interface
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
-from parsl.addresses import address_by_interface
+from parsl.launchers import MpiRunLauncher
 from parsl.monitoring.monitoring import MonitoringHub
+from parsl.providers import PBSProProvider
 
 config = Config(
         executors=[

parsl/configs/Azure.py

@@ -1,14 +1,13 @@
 """Config for Azure"""
-from parsl.config import Config
+import getpass
 
-from parsl.providers import AzureProvider
-from parsl.executors import HighThroughputExecutor
-from parsl.data_provider.http import HTTPInTaskStaging
+from parsl.addresses import address_by_query
+from parsl.config import Config
 from parsl.data_provider.ftp import FTPInTaskStaging
+from parsl.data_provider.http import HTTPInTaskStaging
 from parsl.data_provider.rsync import RSyncStaging
-from parsl.addresses import address_by_query
-
-import getpass
+from parsl.executors import HighThroughputExecutor
+from parsl.providers import AzureProvider
 
 vm_reference = {
     # All fields below are required

parsl/configs/ad_hoc.py

@@ -1,8 +1,9 @@
-from parsl.providers import AdHocProvider
+from typing import Any, Dict
+
 from parsl.channels import SSHChannel
-from parsl.executors import HighThroughputExecutor
 from parsl.config import Config
-from typing import Any, Dict
+from parsl.executors import HighThroughputExecutor
+from parsl.providers import AdHocProvider
 
 user_opts: Dict[str, Dict[str, Any]]
 user_opts = {'adhoc':

parsl/configs/bridges.py

@@ -1,8 +1,8 @@
+from parsl.addresses import address_by_interface
 from parsl.config import Config
-from parsl.providers import SlurmProvider
-from parsl.launchers import SrunLauncher
 from parsl.executors import HighThroughputExecutor
-from parsl.addresses import address_by_interface
+from parsl.launchers import SrunLauncher
+from parsl.providers import SlurmProvider
 
 """ This config assumes that it is used to launch parsl tasks from the login nodes
 of Bridges at PSC. Each job submitted to the scheduler will request 2 nodes for 10 minutes.

parsl/configs/cc_in2p3.py

@@ -1,7 +1,7 @@
-from parsl.config import Config
 from parsl.channels import LocalChannel
-from parsl.providers import GridEngineProvider
+from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
+from parsl.providers import GridEngineProvider
 
 config = Config(
     executors=[

parsl/configs/ec2.py

@@ -1,6 +1,6 @@
 from parsl.config import Config
-from parsl.providers import AWSProvider
 from parsl.executors import HighThroughputExecutor
+from parsl.providers import AWSProvider
 
 config = Config(
     executors=[

parsl/configs/expanse.py

@@ -1,8 +1,7 @@
 from parsl.config import Config
+from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SrunLauncher
 from parsl.providers import SlurmProvider
-from parsl.executors import HighThroughputExecutor
-
 
 config = Config(
     executors=[

parsl/configs/frontera.py

@@ -1,9 +1,8 @@
-from parsl.config import Config
 from parsl.channels import LocalChannel
-from parsl.providers import SlurmProvider
+from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SrunLauncher
-
+from parsl.providers import SlurmProvider
 
 """ This config assumes that it is used to launch parsl tasks from the login nodes
 of Frontera at TACC. Each job submitted to the scheduler will request 2 nodes for 10 minutes.

parsl/configs/htex_local.py

@@ -1,8 +1,7 @@
-from parsl.providers import LocalProvider
 from parsl.channels import LocalChannel
-
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
+from parsl.providers import LocalProvider
 
 config = Config(
     executors=[

parsl/configs/illinoiscluster.py

@@ -1,7 +1,7 @@
 from parsl.config import Config
-from parsl.providers import SlurmProvider
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SrunLauncher
+from parsl.providers import SlurmProvider
 
 """ This config assumes that it is used to launch parsl tasks from the login nodes
 of the Campus Cluster at UIUC. Each job submitted to the scheduler will request 2 nodes for 10 minutes.

parsl/configs/kubernetes.py

@@ -1,8 +1,7 @@
+from parsl.addresses import address_by_route
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.providers import KubernetesProvider
-from parsl.addresses import address_by_route
-
 
 config = Config(
     executors=[

parsl/configs/midway.py

@@ -1,8 +1,8 @@
+from parsl.addresses import address_by_interface
 from parsl.config import Config
-from parsl.providers import SlurmProvider
-from parsl.launchers import SrunLauncher
 from parsl.executors import HighThroughputExecutor
-from parsl.addresses import address_by_interface
+from parsl.launchers import SrunLauncher
+from parsl.providers import SlurmProvider
 
 config = Config(
     executors=[

parsl/configs/osg.py

@@ -1,6 +1,6 @@
 from parsl.config import Config
-from parsl.providers import CondorProvider
 from parsl.executors import HighThroughputExecutor
+from parsl.providers import CondorProvider
 
 config = Config(
     executors=[

parsl/configs/polaris.py

@@ -1,8 +1,8 @@
 from parsl.addresses import address_by_interface
+from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import MpiExecLauncher
 from parsl.providers import PBSProProvider
-from parsl.config import Config
 
 # There are three user parameters to change for the PBSProProvider:
 #  YOUR_ACCOUNT: Account to charge usage

parsl/configs/stampede2.py

@@ -1,10 +1,9 @@
+from parsl.addresses import address_by_interface
 from parsl.config import Config
-from parsl.providers import SlurmProvider
-from parsl.launchers import SrunLauncher
-from parsl.executors import HighThroughputExecutor
 from parsl.data_provider.globus import GlobusStaging
-from parsl.addresses import address_by_interface
-
+from parsl.executors import HighThroughputExecutor
+from parsl.launchers import SrunLauncher
+from parsl.providers import SlurmProvider
 
 config = Config(
     executors=[

parsl/configs/summit.py

@@ -1,11 +1,9 @@
+from parsl.addresses import address_by_interface
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
-
 from parsl.launchers import JsrunLauncher
 from parsl.providers import LSFProvider
 
-from parsl.addresses import address_by_interface
-
 config = Config(
     executors=[
         HighThroughputExecutor(

parsl/configs/toss3_llnl.py

@@ -1,8 +1,7 @@
 from parsl.config import Config
 from parsl.executors import FluxExecutor
-from parsl.providers import SlurmProvider
 from parsl.launchers import SrunLauncher
-
+from parsl.providers import SlurmProvider
 
 config = Config(
     executors=[

parsl/configs/vineex_local.py

@@ -1,8 +1,8 @@
-from parsl.config import Config
-from parsl.executors.taskvine import TaskVineExecutor
-from parsl.executors.taskvine import TaskVineManagerConfig
 import uuid
 
+from parsl.config import Config
+from parsl.executors.taskvine import TaskVineExecutor, TaskVineManagerConfig
+
 config = Config(
     executors=[
         TaskVineExecutor(

parsl/configs/wqex_local.py

@@ -1,8 +1,8 @@
+import uuid
+
 from parsl.config import Config
 from parsl.executors import WorkQueueExecutor
 
-import uuid
-
 config = Config(
     executors=[
         WorkQueueExecutor(

parsl/data_provider/data_manager.py

@@ -1,14 +1,14 @@
 import logging
 from concurrent.futures import Future
-from typing import Any, Callable, List, Optional, TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, Callable, List, Optional
 
 from parsl.app.futures import DataFuture
-from parsl.data_provider.files import File
 from parsl.data_provider.file_noop import NoOpFileStaging
+from parsl.data_provider.files import File
 from parsl.data_provider.ftp import FTPSeparateTaskStaging
 from parsl.data_provider.http import HTTPSeparateTaskStaging
-from parsl.data_provider.zip import ZipFileStaging
 from parsl.data_provider.staging import Staging
+from parsl.data_provider.zip import ZipFileStaging
 
 if TYPE_CHECKING:
     from parsl.dataflow.dflow import DataFlowKernel

parsl/data_provider/file_noop.py

@@ -1,8 +1,7 @@
 import logging
 
-from parsl.utils import RepresentationMixin
 from parsl.data_provider.staging import Staging
-
+from parsl.utils import RepresentationMixin
 
 logger = logging.getLogger(__name__)
 

parsl/data_provider/files.py

@@ -5,13 +5,13 @@ to transfer the file as well as to give the appropriate filepath depending
 on where (client-side, remote-side, intermediary-side) the File.filepath is
 being called from.
 """
-import os
-
-import typeguard
 import logging
+import os
 from typing import Optional, Union
 from urllib.parse import urlparse
 
+import typeguard
+
 logger = logging.getLogger(__name__)