parsl 2024.3.11__py3-none-any.whl → 2025.1.13__py3-none-any.whl

parsl/__init__.py

@@ -15,24 +15,23 @@ AUTO_LOGNAME
 
 """
 import logging
+import multiprocessing as _multiprocessing
 import os
 import platform
 
-from parsl.version import VERSION
 from parsl.app.app import bash_app, join_app, python_app
 from parsl.config import Config
-from parsl.executors import ThreadPoolExecutor
-from parsl.executors import HighThroughputExecutor
-from parsl.executors import WorkQueueExecutor
-from parsl.log_utils import set_stream_logger
-from parsl.log_utils import set_file_logger
-from parsl.monitoring import MonitoringHub
-
 from parsl.data_provider.files import File
-
 from parsl.dataflow.dflow import DataFlowKernel, DataFlowKernelLoader
+from parsl.executors import (
+    HighThroughputExecutor,
+    ThreadPoolExecutor,
+    WorkQueueExecutor,
+)
+from parsl.log_utils import set_file_logger, set_stream_logger
+from parsl.monitoring import MonitoringHub
+from parsl.version import VERSION
 
-import multiprocessing as _multiprocessing
 if platform.system() == 'Darwin':
     _multiprocessing.set_start_method('fork', force=True)
 

parsl/addresses.py

@@ -6,19 +6,22 @@ The helper to use depends on the network environment around the submitter,
 so some experimentation will probably be needed to choose the correct one.
 """
 
+import ipaddress
 import logging
 import platform
-import requests
 import socket
+
+import requests
+
 try:
     import fcntl
 except ImportError:
     fcntl = None  # type: ignore[assignment]
 import struct
-import typeguard
-import psutil
+from typing import Callable, List, Set, Union
 
-from typing import Set, List, Callable
+import psutil
+import typeguard
 
 logger = logging.getLogger(__name__)
 
@@ -81,7 +84,9 @@ def address_by_hostname() -> str:
 def address_by_interface(ifname: str) -> str:
     """Returns the IP address of the given interface name, e.g. 'eth0'
 
-    This is from a Stack Overflow answer: https://stackoverflow.com/questions/24196932/how-can-i-get-the-ip-address-of-eth0-in-python#24196955
+    This is taken from a Stack Overflow answer:
+    https://stackoverflow.com/questions/24196932/how-can-i-get-the-ip-address-of-eth0-in-python#24196955
+
 
     Parameters
     ----------
@@ -111,7 +116,7 @@ def get_all_addresses() -> Set[str]:
         try:
             s_addresses.add(address_by_interface(interface))
         except Exception:
-            logger.info("Ignoring failure to fetch address from interface {}".format(interface))
+            logger.debug("Ignoring failure to fetch address from interface {}".format(interface))
 
     resolution_functions: List[Callable[[], str]]
     resolution_functions = [address_by_hostname, address_by_route, address_by_query]
@@ -119,7 +124,7 @@ def get_all_addresses() -> Set[str]:
         try:
             s_addresses.add(f())
         except Exception:
-            logger.info("Ignoring an address finder exception")
+            logger.debug("Ignoring an address finder exception")
 
     return s_addresses
 
@@ -152,3 +157,20 @@ def get_any_address() -> str:
     if addr == '':
         raise Exception('Cannot find address of the local machine.')
     return addr
+
+
+def tcp_url(address: str, port: Union[str, int, None] = None) -> str:
+    """Construct a tcp url safe for IPv4 and IPv6"""
+    if address == "*":
+        return "tcp://*"
+
+    ip_addr = ipaddress.ip_address(address)
+
+    port_suffix = f":{port}" if port else ""
+
+    if ip_addr.version == 6 and port_suffix:
+        url = f"tcp://[{address}]{port_suffix}"
+    else:
+        url = f"tcp://{address}{port_suffix}"
+
+    return url

parsl/app/app.py

@@ -3,19 +3,16 @@
 The App class encapsulates a generic leaf task that can be executed asynchronously.
 """
 import logging
-import typeguard
 from abc import ABCMeta, abstractmethod
 from inspect import signature
-from typing import List, Optional, Sequence, Union
+from typing import Any, Callable, Dict, List, Optional, Sequence, Union
+
+import typeguard
 from typing_extensions import Literal
 
 from parsl.dataflow.dflow import DataFlowKernel
-
-from typing import Any, Callable, Dict
-
 from parsl.dataflow.futures import AppFuture
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -69,8 +66,10 @@ class AppBase(metaclass=ABCMeta):
             self.kwargs['walltime'] = params['walltime'].default
         if 'parsl_resource_specification' in params:
             self.kwargs['parsl_resource_specification'] = params['parsl_resource_specification'].default
-        self.outputs = params['outputs'].default if 'outputs' in params else []
-        self.inputs = params['inputs'].default if 'inputs' in params else []
+        if 'outputs' in params:
+            self.kwargs['outputs'] = params['outputs'].default
+        if 'inputs' in params:
+            self.kwargs['inputs'] = params['inputs'].default
 
     @abstractmethod
     def __call__(self, *args: Any, **kwargs: Any) -> AppFuture:

parsl/app/bash.py

@@ -1,10 +1,10 @@
-from functools import update_wrapper
-from functools import partial
-from inspect import signature, Parameter
 import logging
+from functools import partial
+from inspect import Parameter, signature
 
-from parsl.app.errors import wrap_error
 from parsl.app.app import AppBase
+from parsl.app.errors import wrap_error
+from parsl.data_provider.files import File
 from parsl.dataflow.dflow import DataFlowKernelLoader
 
 logger = logging.getLogger(__name__)
@@ -16,6 +16,7 @@ def remote_side_bash_executor(func, *args, **kwargs):
     """
     import os
     import subprocess
+
     import parsl.app.errors as pe
     from parsl.utils import get_std_fname_mode
 
@@ -54,13 +55,20 @@ def remote_side_bash_executor(func, *args, **kwargs):
         if stdfspec is None:
             return None
 
-        fname, mode = get_std_fname_mode(fdname, stdfspec)
+        if isinstance(stdfspec, File):
+            # a File is an os.PathLike and so we can use it directly for
+            # the subsequent file operations
+            fname = stdfspec
+            mode = "w"
+        else:
+            fname, mode = get_std_fname_mode(fdname, stdfspec)
+
         try:
             if os.path.dirname(fname):
                 os.makedirs(os.path.dirname(fname), exist_ok=True)
             fd = open(fname, mode)
         except Exception as e:
-            raise pe.BadStdStreamFile(fname, e)
+            raise pe.BadStdStreamFile(str(fname)) from e
         return fd
 
     std_out = open_std_fd('stdout')
@@ -115,11 +123,10 @@ class BashApp(AppBase):
             if sig.parameters[s].default is not Parameter.empty:
                 self.kwargs[s] = sig.parameters[s].default
 
-        # update_wrapper allows remote_side_bash_executor to masquerade as self.func
         # partial is used to attach the first arg the "func" to the remote_side_bash_executor
         # this is done to avoid passing a function type in the args which parsl.serializer
         # doesn't support
-        remote_fn = partial(update_wrapper(remote_side_bash_executor, self.func), self.func)
+        remote_fn = partial(remote_side_bash_executor, self.func)
         remote_fn.__name__ = self.func.__name__
         self.wrapped_remote_function = wrap_error(remote_fn)
 

parsl/app/errors.py

@@ -1,10 +1,11 @@
 """Exceptions raised by Apps."""
+import logging
 from functools import wraps
-from typing import Callable, List, Optional, TypeVar, Union
-from typing_extensions import ParamSpec
 from types import TracebackType
-import logging
+from typing import Callable, List, Optional, TypeVar, Union
+
 from tblib import Traceback
+from typing_extensions import ParamSpec
 
 from parsl.data_provider.files import File
 from parsl.errors import ParslError
@@ -69,8 +70,8 @@ class MissingOutputs(ParslError):
         self.reason = reason
         self.outputs = outputs
 
-    def __repr__(self) -> str:
-        return "Missing Outputs: {0}, Reason:{1}".format(self.outputs, self.reason)
+    def __str__(self) -> str:
+        return "Missing Outputs: {0}, Reason: {1}".format(self.outputs, self.reason)
 
 
 class BadStdStreamFile(ParslError):
@@ -78,19 +79,14 @@ class BadStdStreamFile(ParslError):
 
     Contains:
        reason(string)
-       exception object
     """
 
-    def __init__(self, reason: str, exception: Exception) -> None:
-        super().__init__(reason, exception)
+    def __init__(self, reason: str) -> None:
+        super().__init__(reason)
         self._reason = reason
-        self._exception = exception
-
-    def __repr__(self) -> str:
-        return "Bad Stream File: {} Exception: {}".format(self._reason, self._exception)
 
     def __str__(self) -> str:
-        return self.__repr__()
+        return "Bad Stream File: {}".format(self._reason)
 
 
 class RemoteExceptionWrapper:
@@ -136,6 +132,7 @@ def wrap_error(func: Callable[P, R]) -> Callable[P, Union[R, RemoteExceptionWrap
     @wraps(func)
     def wrapper(*args: P.args, **kwargs: P.kwargs) -> Union[R, RemoteExceptionWrapper]:
         import sys
+
         from parsl.app.errors import RemoteExceptionWrapper
         try:
             return func(*args, **kwargs)

parsl/app/futures.py

@@ -1,12 +1,11 @@
 """This module implements DataFutures.
 """
 import logging
-import typeguard
 from concurrent.futures import Future
 
-from parsl.data_provider.files import File
+import typeguard
 
-from typing import Optional
+from parsl.data_provider.files import File
 
 logger = logging.getLogger(__name__)
 
@@ -39,24 +38,23 @@ class DataFuture(Future):
             self.set_result(self.file_obj)
 
     @typeguard.typechecked
-    def __init__(self, fut: Future, file_obj: File, tid: Optional[int] = None) -> None:
+    def __init__(self, fut: Future, file_obj: File, tid: int) -> None:
         """Construct the DataFuture object.
 
         If the file_obj is a string convert to a File.
 
         Args:
-            - fut (AppFuture) : AppFuture that this DataFuture will track
-            - file_obj (string/File obj) : Something representing file(s)
+            - fut (Future) : Future that this DataFuture will track.
+                             Completion of ``fut`` indicates that the data is
+                             ready.
+            - file_obj (File) : File that this DataFuture represents the availability of
 
         Kwargs:
             - tid (task_id) : Task id that this DataFuture tracks
         """
         super().__init__()
         self._tid = tid
-        if isinstance(file_obj, File):
-            self.file_obj = file_obj
-        else:
-            raise ValueError("DataFuture must be initialized with a File, not {}".format(type(file_obj)))
+        self.file_obj = file_obj
         self.parent = fut
 
         self.parent.add_done_callback(self.parent_callback)

parsl/app/python.py

@@ -2,6 +2,7 @@ import logging
 import threading
 
 import tblib.pickling_support
+
 tblib.pickling_support.install()
 
 from functools import wraps
@@ -11,7 +12,6 @@ from parsl.app.errors import wrap_error
 from parsl.dataflow.dflow import DataFlowKernelLoader
 from parsl.utils import AutoCancelTimer
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -19,6 +19,7 @@ def timeout(f, seconds: float):
     @wraps(f)
     def wrapper(*args, **kwargs):
         import ctypes
+
         import parsl.app.errors
 
         def inject_exception(thread):

parsl/benchmark/perf.py

@@ -1,7 +1,8 @@
 import argparse
+import concurrent.futures
 import importlib
 import time
-import concurrent.futures
+
 import parsl
 
 min_iterations = 2

parsl/concurrent/__init__.py

@@ -1,8 +1,8 @@
 """Interfaces modeled after Python's `concurrent library <https://docs.python.org/3/library/concurrent.html>`_"""
-from typing import Callable, Dict, Optional, Iterator, Iterable
+import time
 from concurrent.futures import Executor
+from typing import Callable, Dict, Iterable, Iterator, Optional
 from warnings import warn
-import time
 
 from parsl import Config, DataFlowKernel
 from parsl.app.python import PythonApp

parsl/config.py

@@ -1,20 +1,24 @@
 import logging
-import typeguard
-
 from typing import Callable, Iterable, Optional, Sequence, Union
+
+import typeguard
 from typing_extensions import Literal
 
-from parsl.utils import RepresentationMixin
+from parsl.dataflow.dependency_resolvers import DependencyResolver
+from parsl.dataflow.taskrecord import TaskRecord
+from parsl.errors import ConfigurationError
 from parsl.executors.base import ParslExecutor
 from parsl.executors.threads import ThreadPoolExecutor
-from parsl.errors import ConfigurationError
-from parsl.dataflow.taskrecord import TaskRecord
 from parsl.monitoring import MonitoringHub
+from parsl.usage_tracking.api import UsageInformation
+from parsl.usage_tracking.levels import DISABLED as USAGE_TRACKING_DISABLED
+from parsl.usage_tracking.levels import LEVEL_3 as USAGE_TRACKING_LEVEL_3
+from parsl.utils import RepresentationMixin
 
 logger = logging.getLogger(__name__)
 
 
-class Config(RepresentationMixin):
+class Config(RepresentationMixin, UsageInformation):
     """
     Specification of Parsl configuration options.
 
@@ -34,6 +38,17 @@ class Config(RepresentationMixin):
     checkpoint_period : str, optional
         Time interval (in "HH:MM:SS") at which to checkpoint completed tasks. Only has an effect if
         ``checkpoint_mode='periodic'``.
+    dependency_resolver: plugin point for custom dependency resolvers. Default: only resolve Futures,
+        using the `SHALLOW_DEPENDENCY_RESOLVER`.
+    exit_mode: str, optional
+        When Parsl is used as a context manager (using ``with parsl.load`` syntax) then this parameter
+        controls what will happen to running tasks and exceptions at exit. The options are:
+
+        * ``cleanup``: cleanup the DFK on exit without waiting for any tasks
+        * ``skip``: skip all shutdown behaviour when exiting the context manager
+        * ``wait``: wait for all tasks to complete when exiting normally, but exit immediately when exiting due to an exception.
+
+        Default is ``cleanup``.
     garbage_collect : bool. optional.
         Delete task records from DFK when tasks have completed. Default: True
     internal_tasks_max_threads : int, optional
@@ -50,16 +65,27 @@ class Config(RepresentationMixin):
         of 1.
     run_dir : str, optional
         Path to run directory. Default is 'runinfo'.
+    std_autopath : function, optional
+        Sets the function used to generate stdout/stderr specifications when parsl.AUTO_LOGPATH is used. If no function
+        is specified, generates paths that look like: ``rundir/NNN/task_logs/X/task_{id}_{name}{label}.{out/err}``
     strategy : str, optional
         Strategy to use for scaling blocks according to workflow needs. Can be 'simple', 'htex_auto_scale', 'none'
         or `None`.
         If 'none' or `None`, dynamic scaling will be disabled. Default is 'simple'. The literal value `None` is
         deprecated.
+    strategy_period : float or int, optional
+        How often the scaling strategy should be executed. Default is 5 seconds.
     max_idletime : float, optional
         The maximum idle time allowed for an executor before strategy could shut down unused blocks. Default is 120.0 seconds.
-    usage_tracking : bool, optional
-        Set this field to True to opt-in to Parsl's usage tracking system. Parsl only collects minimal, non personally-identifiable,
-        information used for reporting to our funding agencies. Default is False.
+    usage_tracking : int, optional
+        Set this field to 1, 2, or 3 to opt-in to Parsl's usage tracking system.
+        The value represents the level of usage tracking detail to be collected.
+        Setting this field to 0 will disable usage tracking. Default (this field is not set): usage tracking is not enabled.
+        Parsl only collects minimal, non personally-identifiable,
+        information used for reporting to our funding agencies.
+    project_name: str, optional
+        Option to deanonymize usage tracking data.
+        If set, this value will be used as the project name in the usage tracking data and placed on the leaderboard.
     initialize_logging : bool, optional
         Make DFK optionally not initialize any logging. Log messages
         will still be passed into the python logging system under the
@@ -82,15 +108,20 @@ class Config(RepresentationMixin):
                                         Literal['dfk_exit'],
                                         Literal['manual']] = None,
                  checkpoint_period: Optional[str] = None,
+                 dependency_resolver: Optional[DependencyResolver] = None,
+                 exit_mode: Literal['cleanup', 'skip', 'wait'] = 'cleanup',
                  garbage_collect: bool = True,
                  internal_tasks_max_threads: int = 10,
                  retries: int = 0,
                  retry_handler: Optional[Callable[[Exception, TaskRecord], float]] = None,
                  run_dir: str = 'runinfo',
+                 std_autopath: Optional[Callable] = None,
                  strategy: Optional[str] = 'simple',
+                 strategy_period: Union[float, int] = 5,
                  max_idletime: float = 120.0,
                  monitoring: Optional[MonitoringHub] = None,
-                 usage_tracking: bool = False,
+                 usage_tracking: int = 0,
+                 project_name: Optional[str] = None,
                  initialize_logging: bool = True) -> None:
 
         executors = tuple(executors or [])
@@ -115,16 +146,22 @@ class Config(RepresentationMixin):
         if checkpoint_mode == 'periodic' and checkpoint_period is None:
             checkpoint_period = "00:30:00"
         self.checkpoint_period = checkpoint_period
+        self.dependency_resolver = dependency_resolver
+        self.exit_mode = exit_mode
         self.garbage_collect = garbage_collect
         self.internal_tasks_max_threads = internal_tasks_max_threads
         self.retries = retries
         self.retry_handler = retry_handler
         self.run_dir = run_dir
         self.strategy = strategy
+        self.strategy_period = strategy_period
         self.max_idletime = max_idletime
+        self.validate_usage_tracking(usage_tracking)
         self.usage_tracking = usage_tracking
+        self.project_name = project_name
         self.initialize_logging = initialize_logging
         self.monitoring = monitoring
+        self.std_autopath: Optional[Callable] = std_autopath
 
     @property
     def executors(self) -> Sequence[ParslExecutor]:
@@ -140,3 +177,13 @@ class Config(RepresentationMixin):
         if len(duplicates) > 0:
             raise ConfigurationError('Executors must have unique labels ({})'.format(
                 ', '.join(['label={}'.format(repr(d)) for d in duplicates])))
+
+    def validate_usage_tracking(self, level: int) -> None:
+        if not USAGE_TRACKING_DISABLED <= level <= USAGE_TRACKING_LEVEL_3:
+            raise ConfigurationError(
+                f"Usage Tracking values must be 0, 1, 2, or 3 and not {level}"
+            )
+
+    def get_usage_information(self):
+        return {"executors_len": len(self.executors),
+                "dependency_resolver": self.dependency_resolver is not None}

parsl/configs/ASPIRE1.py

@@ -1,9 +1,10 @@
-from parsl.providers import PBSProProvider
-from parsl.launchers import MpiRunLauncher
+from parsl.addresses import address_by_interface
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
-from parsl.addresses import address_by_interface
+from parsl.launchers import MpiRunLauncher
 from parsl.monitoring.monitoring import MonitoringHub
+from parsl.providers import PBSProProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 config = Config(
         executors=[
@@ -33,11 +34,11 @@ config = Config(
         ],
         monitoring=MonitoringHub(
             hub_address=address_by_interface('ib0'),
-            hub_port=55055,
             resource_monitoring_interval=10,
         ),
         strategy='simple',
         retries=3,
         app_cache=True,
-        checkpoint_mode='task_exit'
+        checkpoint_mode='task_exit',
+        usage_tracking=LEVEL_1,
 )

parsl/configs/Azure.py

@@ -1,14 +1,14 @@
 """Config for Azure"""
-from parsl.config import Config
+import getpass
 
-from parsl.providers import AzureProvider
-from parsl.executors import HighThroughputExecutor
-from parsl.data_provider.http import HTTPInTaskStaging
+from parsl.addresses import address_by_query
+from parsl.config import Config
 from parsl.data_provider.ftp import FTPInTaskStaging
+from parsl.data_provider.http import HTTPInTaskStaging
 from parsl.data_provider.rsync import RSyncStaging
-from parsl.addresses import address_by_query
-
-import getpass
+from parsl.executors import HighThroughputExecutor
+from parsl.providers import AzureProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 vm_reference = {
     # All fields below are required
@@ -34,5 +34,6 @@ config = Config(
                             FTPInTaskStaging(),
                             RSyncStaging(getpass.getuser() + "@" + address_by_query())],
         )
-    ]
+    ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/bridges.py

@@ -1,8 +1,9 @@
+from parsl.addresses import address_by_interface
 from parsl.config import Config
-from parsl.providers import SlurmProvider
-from parsl.launchers import SrunLauncher
 from parsl.executors import HighThroughputExecutor
-from parsl.addresses import address_by_interface
+from parsl.launchers import SrunLauncher
+from parsl.providers import SlurmProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 """ This config assumes that it is used to launch parsl tasks from the login nodes
 of Bridges at PSC. Each job submitted to the scheduler will request 2 nodes for 10 minutes.
@@ -34,5 +35,6 @@ config = Config(
                 cmd_timeout=120,
             ),
         )
-    ]
+    ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/cc_in2p3.py

@@ -1,7 +1,7 @@
 from parsl.config import Config
-from parsl.channels import LocalChannel
-from parsl.providers import GridEngineProvider
 from parsl.executors import HighThroughputExecutor
+from parsl.providers import GridEngineProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 config = Config(
     executors=[
@@ -9,7 +9,6 @@ config = Config(
             label='cc_in2p3_htex',
             max_workers_per_node=2,
             provider=GridEngineProvider(
-                channel=LocalChannel(),
                 nodes_per_block=1,
                 init_blocks=2,
                 max_blocks=2,
@@ -19,4 +18,5 @@ config = Config(
             ),
         )
     ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/ec2.py

@@ -1,6 +1,7 @@
 from parsl.config import Config
-from parsl.providers import AWSProvider
 from parsl.executors import HighThroughputExecutor
+from parsl.providers import AWSProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 config = Config(
     executors=[
@@ -25,4 +26,5 @@ config = Config(
             ),
         )
     ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/expanse.py

@@ -1,8 +1,8 @@
 from parsl.config import Config
+from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SrunLauncher
 from parsl.providers import SlurmProvider
-from parsl.executors import HighThroughputExecutor
-
+from parsl.usage_tracking.levels import LEVEL_1
 
 config = Config(
     executors=[
@@ -25,5 +25,6 @@ config = Config(
                 nodes_per_block=2,
             ),
         )
-    ]
+    ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/frontera.py

@@ -1,9 +1,8 @@
 from parsl.config import Config
-from parsl.channels import LocalChannel
-from parsl.providers import SlurmProvider
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SrunLauncher
-
+from parsl.providers import SlurmProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 """ This config assumes that it is used to launch parsl tasks from the login nodes
 of Frontera at TACC. Each job submitted to the scheduler will request 2 nodes for 10 minutes.
@@ -15,7 +14,6 @@ config = Config(
             max_workers_per_node=1,          # Set number of workers per node
             provider=SlurmProvider(
                 cmd_timeout=60,     # Add extra time for slow scheduler responses
-                channel=LocalChannel(),
                 nodes_per_block=2,
                 init_blocks=1,
                 min_blocks=1,
@@ -33,4 +31,5 @@ config = Config(
             ),
         )
     ],
+    usage_tracking=LEVEL_1,
 )