parsl 2024.5.20__tar.gz → 2024.6.3__tar.gz

{parsl-2024.5.20/parsl.egg-info → parsl-2024.6.3}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2024.5.20
+Version: 2024.6.3
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2024.05.20.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2024.06.03.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0

{parsl-2024.5.20 → parsl-2024.6.3}/README.rst

@@ -122,7 +122,7 @@ Parsl is supported in Python 3.8+. Requirements can be found `here <requirements
 Code of Conduct
 ===============
 
-Parsl seeks to foster an open and welcoming environment - Please see the `Parsl Code of Conduct <https://github.com/Parsl/parsl/blob/master/CODE_OF_CONDUCT.md>`_ for more details.
+Parsl seeks to foster an open and welcoming environment - Please see the `Parsl Code of Conduct <https://github.com/Parsl/parsl?tab=coc-ov-file#parsl-code-of-conduct>`_ for more details.
 
 Contributing
 ============

{parsl-2024.5.20 → parsl-2024.6.3}/parsl/__init__.py

@@ -15,24 +15,23 @@ AUTO_LOGNAME
 
 """
 import logging
+import multiprocessing as _multiprocessing
 import os
 import platform
 
-from parsl.version import VERSION
 from parsl.app.app import bash_app, join_app, python_app
 from parsl.config import Config
-from parsl.executors import ThreadPoolExecutor
-from parsl.executors import HighThroughputExecutor
-from parsl.executors import WorkQueueExecutor
-from parsl.log_utils import set_stream_logger
-from parsl.log_utils import set_file_logger
-from parsl.monitoring import MonitoringHub
-
 from parsl.data_provider.files import File
-
 from parsl.dataflow.dflow import DataFlowKernel, DataFlowKernelLoader
+from parsl.executors import (
+    HighThroughputExecutor,
+    ThreadPoolExecutor,
+    WorkQueueExecutor,
+)
+from parsl.log_utils import set_file_logger, set_stream_logger
+from parsl.monitoring import MonitoringHub
+from parsl.version import VERSION
 
-import multiprocessing as _multiprocessing
 if platform.system() == 'Darwin':
     _multiprocessing.set_start_method('fork', force=True)
 

{parsl-2024.5.20 → parsl-2024.6.3}/parsl/addresses.py

@@ -8,17 +8,19 @@ so some experimentation will probably be needed to choose the correct one.
 
 import logging
 import platform
-import requests
 import socket
+
+import requests
+
 try:
     import fcntl
 except ImportError:
     fcntl = None  # type: ignore[assignment]
 import struct
-import typeguard
-import psutil
+from typing import Callable, List, Set
 
-from typing import Set, List, Callable
+import psutil
+import typeguard
 
 logger = logging.getLogger(__name__)
 

{parsl-2024.5.20 → parsl-2024.6.3}/parsl/app/app.py

@@ -3,19 +3,16 @@
 The App class encapsulates a generic leaf task that can be executed asynchronously.
 """
 import logging
-import typeguard
 from abc import ABCMeta, abstractmethod
 from inspect import signature
-from typing import List, Optional, Sequence, Union
+from typing import Any, Callable, Dict, List, Optional, Sequence, Union
+
+import typeguard
 from typing_extensions import Literal
 
 from parsl.dataflow.dflow import DataFlowKernel
-
-from typing import Any, Callable, Dict
-
 from parsl.dataflow.futures import AppFuture
 
-
 logger = logging.getLogger(__name__)
 
 

{parsl-2024.5.20 → parsl-2024.6.3}/parsl/app/bash.py

@@ -1,10 +1,9 @@
-from functools import update_wrapper
-from functools import partial
-from inspect import signature, Parameter
 import logging
+from functools import partial, update_wrapper
+from inspect import Parameter, signature
 
-from parsl.app.errors import wrap_error
 from parsl.app.app import AppBase
+from parsl.app.errors import wrap_error
 from parsl.data_provider.files import File
 from parsl.dataflow.dflow import DataFlowKernelLoader
 
@@ -17,6 +16,7 @@ def remote_side_bash_executor(func, *args, **kwargs):
     """
     import os
     import subprocess
+
     import parsl.app.errors as pe
     from parsl.utils import get_std_fname_mode
 

{parsl-2024.5.20 → parsl-2024.6.3}/parsl/app/errors.py

@@ -1,10 +1,11 @@
 """Exceptions raised by Apps."""
+import logging
 from functools import wraps
-from typing import Callable, List, Optional, TypeVar, Union
-from typing_extensions import ParamSpec
 from types import TracebackType
-import logging
+from typing import Callable, List, Optional, TypeVar, Union
+
 from tblib import Traceback
+from typing_extensions import ParamSpec
 
 from parsl.data_provider.files import File
 from parsl.errors import ParslError
@@ -134,6 +135,7 @@ def wrap_error(func: Callable[P, R]) -> Callable[P, Union[R, RemoteExceptionWrap
     @wraps(func)
     def wrapper(*args: P.args, **kwargs: P.kwargs) -> Union[R, RemoteExceptionWrapper]:
         import sys
+
         from parsl.app.errors import RemoteExceptionWrapper
         try:
             return func(*args, **kwargs)

{parsl-2024.5.20 → parsl-2024.6.3}/parsl/app/python.py

@@ -2,6 +2,7 @@ import logging
 import threading
 
 import tblib.pickling_support
+
 tblib.pickling_support.install()
 
 from functools import wraps
@@ -11,7 +12,6 @@ from parsl.app.errors import wrap_error
 from parsl.dataflow.dflow import DataFlowKernelLoader
 from parsl.utils import AutoCancelTimer
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -19,6 +19,7 @@ def timeout(f, seconds: float):
     @wraps(f)
     def wrapper(*args, **kwargs):
         import ctypes
+
         import parsl.app.errors
 
         def inject_exception(thread):

{parsl-2024.5.20 → parsl-2024.6.3}/parsl/benchmark/perf.py

@@ -1,7 +1,8 @@
 import argparse
+import concurrent.futures
 import importlib
 import time
-import concurrent.futures
+
 import parsl
 
 min_iterations = 2

{parsl-2024.5.20 → parsl-2024.6.3}/parsl/channels/base.py

@@ -1,5 +1,4 @@
 from abc import ABCMeta, abstractmethod, abstractproperty
-
 from typing import Dict, Tuple
 
 

{parsl-2024.5.20 → parsl-2024.6.3}/parsl/channels/errors.py

@@ -1,8 +1,9 @@
 ''' Exceptions raise by Apps.
 '''
-from parsl.errors import ParslError
 from typing import Optional
 
+from parsl.errors import ParslError
+
 
 class ChannelError(ParslError):
     """ Base class for all exceptions

{parsl-2024.5.20 → parsl-2024.6.3}/parsl/channels/oauth_ssh/oauth_ssh.py

@@ -1,13 +1,14 @@
 import logging
-import paramiko
 import socket
 
-from parsl.errors import OptionalModuleMissing
+import paramiko
+
 from parsl.channels.ssh.ssh import SSHChannel
+from parsl.errors import OptionalModuleMissing
 
 try:
-    from oauth_ssh.ssh_service import SSHService
     from oauth_ssh.oauth_ssh_token import find_access_token
+    from oauth_ssh.ssh_service import SSHService
     _oauth_ssh_enabled = True
 except (ImportError, NameError):
     _oauth_ssh_enabled = False

{parsl-2024.5.20 → parsl-2024.6.3}/parsl/channels/ssh/ssh.py

@@ -3,8 +3,16 @@ import logging
 import os
 
 import paramiko
+
 from parsl.channels.base import Channel
-from parsl.channels.errors import BadHostKeyException, AuthException, SSHException, BadScriptPath, BadPermsScriptPath, FileCopyException
+from parsl.channels.errors import (
+    AuthException,
+    BadHostKeyException,
+    BadPermsScriptPath,
+    BadScriptPath,
+    FileCopyException,
+    SSHException,
+)
 from parsl.utils import RepresentationMixin
 
 logger = logging.getLogger(__name__)

{parsl-2024.5.20 → parsl-2024.6.3}/parsl/channels/ssh_il/ssh_il.py

@@ -2,6 +2,7 @@ import getpass
 import logging
 
 import paramiko
+
 from parsl.channels.ssh.ssh import SSHChannel
 
 logger = logging.getLogger(__name__)

{parsl-2024.5.20 → parsl-2024.6.3}/parsl/concurrent/__init__.py

@@ -1,8 +1,8 @@
 """Interfaces modeled after Python's `concurrent library <https://docs.python.org/3/library/concurrent.html>`_"""
-from typing import Callable, Dict, Optional, Iterator, Iterable
+import time
 from concurrent.futures import Executor
+from typing import Callable, Dict, Iterable, Iterator, Optional
 from warnings import warn
-import time
 
 from parsl import Config, DataFlowKernel
 from parsl.app.python import PythonApp

{parsl-2024.5.20 → parsl-2024.6.3}/parsl/config.py

@@ -1,16 +1,17 @@
 import logging
-import typeguard
-
 from typing import Callable, Iterable, Optional, Sequence, Union
+
+import typeguard
 from typing_extensions import Literal
 
-from parsl.utils import RepresentationMixin
+from parsl.dataflow.dependency_resolvers import DependencyResolver
+from parsl.dataflow.taskrecord import TaskRecord
+from parsl.errors import ConfigurationError
 from parsl.executors.base import ParslExecutor
 from parsl.executors.threads import ThreadPoolExecutor
-from parsl.errors import ConfigurationError
-from parsl.dataflow.taskrecord import TaskRecord
 from parsl.monitoring import MonitoringHub
 from parsl.usage_tracking.api import UsageInformation
+from parsl.utils import RepresentationMixin
 
 logger = logging.getLogger(__name__)
 
@@ -35,6 +36,8 @@ class Config(RepresentationMixin, UsageInformation):
     checkpoint_period : str, optional
         Time interval (in "HH:MM:SS") at which to checkpoint completed tasks. Only has an effect if
         ``checkpoint_mode='periodic'``.
+    dependency_resolver: plugin point for custom dependency resolvers. Default: only resolve Futures,
+        using the `SHALLOW_DEPENDENCY_RESOLVER`.
     garbage_collect : bool. optional.
         Delete task records from DFK when tasks have completed. Default: True
     internal_tasks_max_threads : int, optional
@@ -88,6 +91,7 @@ class Config(RepresentationMixin, UsageInformation):
                                         Literal['dfk_exit'],
                                         Literal['manual']] = None,
                  checkpoint_period: Optional[str] = None,
+                 dependency_resolver: Optional[DependencyResolver] = None,
                  garbage_collect: bool = True,
                  internal_tasks_max_threads: int = 10,
                  retries: int = 0,
@@ -123,6 +127,7 @@ class Config(RepresentationMixin, UsageInformation):
         if checkpoint_mode == 'periodic' and checkpoint_period is None:
             checkpoint_period = "00:30:00"
         self.checkpoint_period = checkpoint_period
+        self.dependency_resolver = dependency_resolver
         self.garbage_collect = garbage_collect
         self.internal_tasks_max_threads = internal_tasks_max_threads
         self.retries = retries
@@ -152,4 +157,5 @@ class Config(RepresentationMixin, UsageInformation):
                 ', '.join(['label={}'.format(repr(d)) for d in duplicates])))
 
     def get_usage_information(self):
-        return {"executors_len": len(self.executors)}
+        return {"executors_len": len(self.executors),
+                "dependency_resolver": self.dependency_resolver is not None}

{parsl-2024.5.20 → parsl-2024.6.3}/parsl/configs/Azure.py

@@ -1,14 +1,13 @@
 """Config for Azure"""
-from parsl.config import Config
+import getpass
 
-from parsl.providers import AzureProvider
-from parsl.executors import HighThroughputExecutor
-from parsl.data_provider.http import HTTPInTaskStaging
+from parsl.addresses import address_by_query
+from parsl.config import Config
 from parsl.data_provider.ftp import FTPInTaskStaging
+from parsl.data_provider.http import HTTPInTaskStaging
 from parsl.data_provider.rsync import RSyncStaging
-from parsl.addresses import address_by_query
-
-import getpass
+from parsl.executors import HighThroughputExecutor
+from parsl.providers import AzureProvider
 
 vm_reference = {
     # All fields below are required

{parsl-2024.5.20 → parsl-2024.6.3}/parsl/configs/ad_hoc.py

@@ -1,8 +1,9 @@
-from parsl.providers import AdHocProvider
+from typing import Any, Dict
+
 from parsl.channels import SSHChannel
-from parsl.executors import HighThroughputExecutor
 from parsl.config import Config
-from typing import Any, Dict
+from parsl.executors import HighThroughputExecutor
+from parsl.providers import AdHocProvider
 
 user_opts: Dict[str, Dict[str, Any]]
 user_opts = {'adhoc':

{parsl-2024.5.20 → parsl-2024.6.3}/parsl/configs/expanse.py

@@ -1,8 +1,7 @@
 from parsl.config import Config
+from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SrunLauncher
 from parsl.providers import SlurmProvider
-from parsl.executors import HighThroughputExecutor
-
 
 config = Config(
     executors=[

{parsl-2024.5.20 → parsl-2024.6.3}/parsl/configs/frontera.py

@@ -1,9 +1,8 @@
-from parsl.config import Config
 from parsl.channels import LocalChannel
-from parsl.providers import SlurmProvider
+from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SrunLauncher
-
+from parsl.providers import SlurmProvider
 
 """ This config assumes that it is used to launch parsl tasks from the login nodes
 of Frontera at TACC. Each job submitted to the scheduler will request 2 nodes for 10 minutes.

{parsl-2024.5.20 → parsl-2024.6.3}/parsl/configs/htex_local.py

@@ -1,8 +1,7 @@
-from parsl.providers import LocalProvider
 from parsl.channels import LocalChannel
-
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
+from parsl.providers import LocalProvider
 
 config = Config(
     executors=[

{parsl-2024.5.20 → parsl-2024.6.3}/parsl/configs/kubernetes.py

@@ -1,8 +1,7 @@
+from parsl.addresses import address_by_route
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.providers import KubernetesProvider
-from parsl.addresses import address_by_route
-
 
 config = Config(
     executors=[

{parsl-2024.5.20 → parsl-2024.6.3}/parsl/configs/stampede2.py

@@ -1,10 +1,9 @@
+from parsl.addresses import address_by_interface
 from parsl.config import Config
-from parsl.providers import SlurmProvider
-from parsl.launchers import SrunLauncher
-from parsl.executors import HighThroughputExecutor
 from parsl.data_provider.globus import GlobusStaging
-from parsl.addresses import address_by_interface
-
+from parsl.executors import HighThroughputExecutor
+from parsl.launchers import SrunLauncher
+from parsl.providers import SlurmProvider
 
 config = Config(
     executors=[

{parsl-2024.5.20 → parsl-2024.6.3}/parsl/configs/summit.py

@@ -1,11 +1,9 @@
+from parsl.addresses import address_by_interface
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
-
 from parsl.launchers import JsrunLauncher
 from parsl.providers import LSFProvider
 
-from parsl.addresses import address_by_interface
-
 config = Config(
     executors=[
         HighThroughputExecutor(

{parsl-2024.5.20 → parsl-2024.6.3}/parsl/configs/toss3_llnl.py

@@ -1,8 +1,7 @@
 from parsl.config import Config
 from parsl.executors import FluxExecutor
-from parsl.providers import SlurmProvider
 from parsl.launchers import SrunLauncher
-
+from parsl.providers import SlurmProvider
 
 config = Config(
     executors=[

{parsl-2024.5.20 → parsl-2024.6.3}/parsl/configs/vineex_local.py

@@ -1,8 +1,8 @@
-from parsl.config import Config
-from parsl.executors.taskvine import TaskVineExecutor
-from parsl.executors.taskvine import TaskVineManagerConfig
 import uuid
 
+from parsl.config import Config
+from parsl.executors.taskvine import TaskVineExecutor, TaskVineManagerConfig
+
 config = Config(
     executors=[
         TaskVineExecutor(

{parsl-2024.5.20 → parsl-2024.6.3}/parsl/data_provider/data_manager.py

@@ -1,14 +1,14 @@
 import logging
 from concurrent.futures import Future
-from typing import Any, Callable, List, Optional, TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, Callable, List, Optional
 
 from parsl.app.futures import DataFuture
-from parsl.data_provider.files import File
 from parsl.data_provider.file_noop import NoOpFileStaging
+from parsl.data_provider.files import File
 from parsl.data_provider.ftp import FTPSeparateTaskStaging
 from parsl.data_provider.http import HTTPSeparateTaskStaging
-from parsl.data_provider.zip import ZipFileStaging
 from parsl.data_provider.staging import Staging
+from parsl.data_provider.zip import ZipFileStaging
 
 if TYPE_CHECKING:
     from parsl.dataflow.dflow import DataFlowKernel

{parsl-2024.5.20 → parsl-2024.6.3}/parsl/data_provider/file_noop.py

@@ -1,8 +1,7 @@
 import logging
 
-from parsl.utils import RepresentationMixin
 from parsl.data_provider.staging import Staging
-
+from parsl.utils import RepresentationMixin
 
 logger = logging.getLogger(__name__)
 

{parsl-2024.5.20 → parsl-2024.6.3}/parsl/data_provider/ftp.py

@@ -3,10 +3,8 @@ import logging
 import os
 
 import parsl
-
-from parsl.utils import RepresentationMixin
 from parsl.data_provider.staging import Staging
-
+from parsl.utils import RepresentationMixin
 
 logger = logging.getLogger(__name__)
 

{parsl-2024.5.20 → parsl-2024.6.3}/parsl/data_provider/globus.py

@@ -1,15 +1,16 @@
-import logging
 import json
-import globus_sdk
+import logging
 import os
-import parsl
-import typeguard
-
 from functools import partial
 from typing import Optional
+
+import globus_sdk
+import typeguard
+
+import parsl
 from parsl.app.app import python_app
-from parsl.utils import RepresentationMixin
 from parsl.data_provider.staging import Staging
+from parsl.utils import RepresentationMixin
 
 logger = logging.getLogger(__name__)
 

{parsl-2024.5.20 → parsl-2024.6.3}/parsl/data_provider/staging.py

@@ -1,9 +1,9 @@
 from concurrent.futures import Future
-from typing import Optional, Callable
+from typing import TYPE_CHECKING, Callable, Optional
+
 from parsl.app.futures import DataFuture
 from parsl.data_provider.files import File
 
-from typing import TYPE_CHECKING
 if TYPE_CHECKING:
     from parsl.data_provider.data_manager import DataManager
 

{parsl-2024.5.20 → parsl-2024.6.3}/parsl/data_provider/zip.py

@@ -1,16 +1,15 @@
-import filelock
 import logging
 import os
-import parsl
 import zipfile
-
 from typing import Tuple
 
-from parsl.data_provider.staging import Staging
+import filelock
+
+import parsl
 from parsl.data_provider.files import File
+from parsl.data_provider.staging import Staging
 from parsl.errors import ParslError
 
-
 logger = logging.getLogger(__name__)
 
 

parsl-2024.6.3/parsl/dataflow/dependency_resolvers.py

@@ -0,0 +1,115 @@
+from concurrent.futures import Future
+from dataclasses import dataclass
+from functools import singledispatch
+from typing import Callable, Sequence
+
+
+@dataclass
+class DependencyResolver:
+    """A DependencyResolver describes how app dependencies can be resolved.
+    It is specified as two functions: `traverse_to_gather` which turns an
+    app parameter into a sequence of futures which must be waited for before
+    the task can be executed (for example, in the case of
+    `DEEP_DEPENDENCY_RESOLVER` this traverses structures such as lists to
+    find every contained ``Future``), and `traverse_to_unwrap` which turns an
+    app parameter into its value to be passed to the app on execution
+    (for example in the case of `DEEP_DEPENDENCY_RESOLVER` this replaces a
+    list containing futures with a new list containing the values of those
+    resolved futures).
+
+    By default, Parsl will use `SHALLOW_DEPENDENCY_RESOLVER` which only
+    resolves Futures passed directly as arguments.
+    """
+    traverse_to_gather: Callable[[object], Sequence[Future]]
+    traverse_to_unwrap: Callable[[object], object]
+
+
+@singledispatch
+def shallow_traverse_to_gather(o):
+    # objects in general do not expose futures that we can see
+    return []
+
+
+@singledispatch
+def shallow_traverse_to_unwrap(o):
+    # objects in general unwrap to themselves
+    return o
+
+
+@shallow_traverse_to_gather.register
+def _(fut: Future):
+    return [fut]
+
+
+@shallow_traverse_to_unwrap.register
+@singledispatch
+def _(fut: Future):
+    assert fut.done()
+    return fut.result()
+
+
+@singledispatch
+def deep_traverse_to_gather(o):
+    # objects in general do not expose futures that we can see
+    return []
+
+
+@singledispatch
+def deep_traverse_to_unwrap(o):
+    # objects in general unwrap to themselves
+    return o
+
+
+@deep_traverse_to_gather.register
+def _(fut: Future):
+    return [fut]
+
+
+@deep_traverse_to_unwrap.register
+@singledispatch
+def _(fut: Future):
+    assert fut.done()
+    return fut.result()
+
+
+@deep_traverse_to_gather.register(tuple)
+@deep_traverse_to_gather.register(list)
+@deep_traverse_to_gather.register(set)
+def _(iterable):
+    return [e for v in iterable for e in deep_traverse_to_gather(v)]
+
+
+@deep_traverse_to_unwrap.register(tuple)
+@deep_traverse_to_unwrap.register(list)
+@deep_traverse_to_unwrap.register(set)
+@singledispatch
+def _(iterable):
+
+    type_ = type(iterable)
+    return type_(map(deep_traverse_to_unwrap, iterable))
+
+
+@deep_traverse_to_gather.register(dict)
+def _(dictionary):
+    futures = []
+    for key, value in dictionary.items():
+        futures.extend(deep_traverse_to_gather(key))
+        futures.extend(deep_traverse_to_gather(value))
+    return futures
+
+
+@deep_traverse_to_unwrap.register(dict)
+def _(dictionary):
+    unwrapped_dict = {}
+    for key, value in dictionary.items():
+        key = deep_traverse_to_unwrap(key)
+        value = deep_traverse_to_unwrap(value)
+        unwrapped_dict[key] = value
+    return unwrapped_dict
+
+
+DEEP_DEPENDENCY_RESOLVER = DependencyResolver(traverse_to_gather=deep_traverse_to_gather,
+                                              traverse_to_unwrap=deep_traverse_to_unwrap)
+
+SHALLOW_DEPENDENCY_RESOLVER = DependencyResolver(traverse_to_gather=shallow_traverse_to_gather,
+                                                 traverse_to_unwrap=shallow_traverse_to_unwrap)