parsl 2024.3.11__py3-none-any.whl → 2025.1.13__py3-none-any.whl

parsl/configs/htex_local.py

@@ -1,8 +1,7 @@
-from parsl.providers import LocalProvider
-from parsl.channels import LocalChannel
-
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
+from parsl.providers import LocalProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 config = Config(
     executors=[
@@ -10,10 +9,10 @@ config = Config(
             label="htex_local",
             cores_per_worker=1,
             provider=LocalProvider(
-                channel=LocalChannel(),
                 init_blocks=1,
                 max_blocks=1,
             ),
         )
     ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/illinoiscluster.py

@@ -1,7 +1,8 @@
 from parsl.config import Config
-from parsl.providers import SlurmProvider
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SrunLauncher
+from parsl.providers import SlurmProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 """ This config assumes that it is used to launch parsl tasks from the login nodes
 of the Campus Cluster at UIUC. Each job submitted to the scheduler will request 2 nodes for 10 minutes.
@@ -25,4 +26,5 @@ config = Config(
                ),
           )
      ],
+     usage_tracking=LEVEL_1,
 )

parsl/configs/improv.py

@@ -0,0 +1,34 @@
+from parsl.config import Config
+from parsl.executors import HighThroughputExecutor
+from parsl.launchers import MpiRunLauncher
+from parsl.providers import PBSProProvider
+
+config = Config(
+        executors=[
+            HighThroughputExecutor(
+                label="Improv_multinode",
+                max_workers_per_node=32,
+                provider=PBSProProvider(
+                    account="YOUR_ALLOCATION_ON_IMPROV",
+                    # PBS directives (header lines), for example:
+                    # scheduler_options='#PBS -l mem=4gb',
+                    scheduler_options='',
+
+                    queue="compute",
+
+                    # Command to be run before starting a worker:
+                    # **WARNING** Improv requires an openmpi module to be
+                    # loaded for the MpiRunLauncher. Add additional env
+                    # load commands to this multiline string.
+                    worker_init='''
+module load gcc/13.2.0;
+module load openmpi/5.0.3-gcc-13.2.0; ''',
+                    launcher=MpiRunLauncher(),
+
+                    # number of compute nodes allocated for each block
+                    nodes_per_block=2,
+                    walltime='00:10:00'
+                ),
+            ),
+        ],
+)

parsl/configs/kubernetes.py

@@ -1,8 +1,8 @@
+from parsl.addresses import address_by_route
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.providers import KubernetesProvider
-from parsl.addresses import address_by_route
-
+from parsl.usage_tracking.levels import LEVEL_1
 
 config = Config(
     executors=[
@@ -37,5 +37,6 @@ config = Config(
                 max_blocks=10,
             ),
         ),
-    ]
+    ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/local_threads.py

@@ -1,4 +1,8 @@
 from parsl.config import Config
 from parsl.executors.threads import ThreadPoolExecutor
+from parsl.usage_tracking.levels import LEVEL_1
 
-config = Config(executors=[ThreadPoolExecutor()])
+config = Config(
+    executors=[ThreadPoolExecutor()],
+    usage_tracking=LEVEL_1,
+)

parsl/configs/midway.py

@@ -1,8 +1,9 @@
+from parsl.addresses import address_by_interface
 from parsl.config import Config
-from parsl.providers import SlurmProvider
-from parsl.launchers import SrunLauncher
 from parsl.executors import HighThroughputExecutor
-from parsl.addresses import address_by_interface
+from parsl.launchers import SrunLauncher
+from parsl.providers import SlurmProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 config = Config(
     executors=[
@@ -28,4 +29,5 @@ config = Config(
             ),
         )
     ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/osg.py

@@ -1,6 +1,7 @@
 from parsl.config import Config
-from parsl.providers import CondorProvider
 from parsl.executors import HighThroughputExecutor
+from parsl.providers import CondorProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 config = Config(
     executors=[
@@ -26,5 +27,6 @@ python3 -m venv parsl_env; source parsl_env/bin/activate; python3 -m pip install
             worker_logdir_root='$OSG_WN_TMP',
             worker_ports=(31000, 31001)
         )
-    ]
+    ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/polaris.py

@@ -1,8 +1,9 @@
 from parsl.addresses import address_by_interface
+from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import MpiExecLauncher
 from parsl.providers import PBSProProvider
-from parsl.config import Config
+from parsl.usage_tracking.levels import LEVEL_1
 
 # There are three user parameters to change for the PBSProProvider:
 #  YOUR_ACCOUNT: Account to charge usage
@@ -34,5 +35,6 @@ config = Config(
                 cpus_per_node=64,
             ),
         ),
-    ]
+    ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/stampede2.py

@@ -1,10 +1,10 @@
+from parsl.addresses import address_by_interface
 from parsl.config import Config
-from parsl.providers import SlurmProvider
-from parsl.launchers import SrunLauncher
-from parsl.executors import HighThroughputExecutor
 from parsl.data_provider.globus import GlobusStaging
-from parsl.addresses import address_by_interface
-
+from parsl.executors import HighThroughputExecutor
+from parsl.launchers import SrunLauncher
+from parsl.providers import SlurmProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 config = Config(
     executors=[
@@ -35,4 +35,5 @@ config = Config(
         )
 
     ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/summit.py

@@ -1,10 +1,9 @@
+from parsl.addresses import address_by_interface
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
-
 from parsl.launchers import JsrunLauncher
 from parsl.providers import LSFProvider
-
-from parsl.addresses import address_by_interface
+from parsl.usage_tracking.levels import LEVEL_1
 
 config = Config(
     executors=[
@@ -28,4 +27,5 @@ config = Config(
         )
 
     ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/toss3_llnl.py

@@ -1,8 +1,8 @@
 from parsl.config import Config
 from parsl.executors import FluxExecutor
-from parsl.providers import SlurmProvider
 from parsl.launchers import SrunLauncher
-
+from parsl.providers import SlurmProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 config = Config(
     executors=[
@@ -25,5 +25,6 @@ config = Config(
                 cmd_timeout=120,
             ),
         )
-    ]
+    ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/vineex_local.py

@@ -1,8 +1,9 @@
-from parsl.config import Config
-from parsl.executors.taskvine import TaskVineExecutor
-from parsl.executors.taskvine import TaskVineManagerConfig
 import uuid
 
+from parsl.config import Config
+from parsl.executors.taskvine import TaskVineExecutor, TaskVineManagerConfig
+from parsl.usage_tracking.levels import LEVEL_1
+
 config = Config(
     executors=[
         TaskVineExecutor(
@@ -15,5 +16,6 @@ config = Config(
             # To disable status reporting, comment out the project_name.
             manager_config=TaskVineManagerConfig(project_name="parsl-vine-" + str(uuid.uuid4())),
         )
-    ]
+    ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/wqex_local.py

@@ -1,7 +1,8 @@
+import uuid
+
 from parsl.config import Config
 from parsl.executors import WorkQueueExecutor
-
-import uuid
+from parsl.usage_tracking.levels import LEVEL_1
 
 config = Config(
     executors=[
@@ -21,5 +22,6 @@ config = Config(
             # A shared filesystem is not needed when using Work Queue.
             shared_fs=False
         )
-    ]
+    ],
+    usage_tracking=LEVEL_1,
 )

parsl/curvezmq.py

@@ -160,6 +160,9 @@ class ServerContext(BaseContext):
             except zmq.ZMQError as e:
                 raise ValueError("Invalid CurveZMQ key format") from e
             sock.setsockopt(zmq.CURVE_SERVER, True)  # Must come before bind
+
+        # This flag enables IPV6 in addition to IPV4
+        sock.setsockopt(zmq.IPV6, True)
         return sock
 
     def term(self):
@@ -202,4 +205,5 @@ class ClientContext(BaseContext):
                 sock.setsockopt(zmq.CURVE_SERVERKEY, server_public_key)
             except zmq.ZMQError as e:
                 raise ValueError("Invalid CurveZMQ key format") from e
+        sock.setsockopt(zmq.IPV6, True)
         return sock

parsl/data_provider/data_manager.py

@@ -1,13 +1,14 @@
 import logging
 from concurrent.futures import Future
-from typing import Any, Callable, List, Optional, TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, Callable, List, Optional
 
 from parsl.app.futures import DataFuture
-from parsl.data_provider.files import File
 from parsl.data_provider.file_noop import NoOpFileStaging
+from parsl.data_provider.files import File
 from parsl.data_provider.ftp import FTPSeparateTaskStaging
 from parsl.data_provider.http import HTTPSeparateTaskStaging
 from parsl.data_provider.staging import Staging
+from parsl.data_provider.zip import ZipFileStaging
 
 if TYPE_CHECKING:
     from parsl.dataflow.dflow import DataFlowKernel
@@ -17,7 +18,7 @@ logger = logging.getLogger(__name__)
 # these will be shared between all executors that do not explicitly
 # override, so should not contain executor-specific state
 default_staging: List[Staging]
-default_staging = [NoOpFileStaging(), FTPSeparateTaskStaging(), HTTPSeparateTaskStaging()]
+default_staging = [NoOpFileStaging(), FTPSeparateTaskStaging(), HTTPSeparateTaskStaging(), ZipFileStaging()]
 
 
 class DataManager:

parsl/data_provider/file_noop.py

@@ -1,8 +1,7 @@
 import logging
 
-from parsl.utils import RepresentationMixin
 from parsl.data_provider.staging import Staging
-
+from parsl.utils import RepresentationMixin
 
 logger = logging.getLogger(__name__)
 

parsl/data_provider/files.py

@@ -5,13 +5,13 @@ to transfer the file as well as to give the appropriate filepath depending
 on where (client-side, remote-side, intermediary-side) the File.filepath is
 being called from.
 """
-import os
-
-import typeguard
 import logging
+import os
 from typing import Optional, Union
 from urllib.parse import urlparse
 
+import typeguard
+
 logger = logging.getLogger(__name__)
 
 

parsl/data_provider/ftp.py

@@ -3,10 +3,8 @@ import logging
 import os
 
 import parsl
-
-from parsl.utils import RepresentationMixin
 from parsl.data_provider.staging import Staging
-
+from parsl.utils import RepresentationMixin
 
 logger = logging.getLogger(__name__)
 

parsl/data_provider/globus.py

@@ -1,15 +1,16 @@
-import logging
 import json
-import globus_sdk
+import logging
 import os
-import parsl
-import typeguard
-
 from functools import partial
 from typing import Optional
+
+import globus_sdk
+import typeguard
+
+import parsl
 from parsl.app.app import python_app
-from parsl.utils import RepresentationMixin
 from parsl.data_provider.staging import Staging
+from parsl.utils import RepresentationMixin
 
 logger = logging.getLogger(__name__)
 

parsl/data_provider/http.py

@@ -1,11 +1,11 @@
 import logging
 import os
+
 import requests
 
 import parsl
-
-from parsl.utils import RepresentationMixin
 from parsl.data_provider.staging import Staging
+from parsl.utils import RepresentationMixin
 
 logger = logging.getLogger(__name__)
 

parsl/data_provider/rsync.py

@@ -1,8 +1,8 @@
 import logging
 import os
 
-from parsl.utils import RepresentationMixin
 from parsl.data_provider.staging import Staging
+from parsl.utils import RepresentationMixin
 
 logger = logging.getLogger(__name__)
 

parsl/data_provider/staging.py

@@ -1,9 +1,9 @@
 from concurrent.futures import Future
-from typing import Optional, Callable
+from typing import TYPE_CHECKING, Callable, Optional
+
 from parsl.app.futures import DataFuture
 from parsl.data_provider.files import File
 
-from typing import TYPE_CHECKING
 if TYPE_CHECKING:
     from parsl.data_provider.data_manager import DataManager
 

parsl/data_provider/zip.py

@@ -0,0 +1,135 @@
+import logging
+import os
+import zipfile
+from typing import Tuple
+
+import filelock
+
+import parsl
+from parsl.data_provider.files import File
+from parsl.data_provider.staging import Staging
+from parsl.errors import ParslError
+
+logger = logging.getLogger(__name__)
+
+
+class ZipAuthorityError(ParslError):
+    def __init__(self, file):
+        self.file = file
+
+    def __str__(self):
+        return f"ZipFileStaging cannot stage Files with an authority (netloc) section ({self.file.netloc}), for {self.file.url}"
+
+
+class ZipFileStaging(Staging):
+    """A stage-out provider for zip files.
+
+    This provider will stage out files by writing them into the specified zip
+    file.
+
+    The filename of both the zip file and the file contained in that zip are
+    specified using a zip: URL, like this:
+
+    zip:/tmp/foo/this.zip/inside/here.txt
+
+    This URL names a zip file ``/tmp/foo/this.zip`` containing a file
+    ``inside/here.txt``.
+
+    The provider will use the Python filelock package to lock the zip file so
+    that it does not conflict with other instances of itself. This lock will
+    not protect against other modifications to the zip file.
+    """
+
+    def can_stage_out(self, file: File) -> bool:
+        return self.is_zip_url(file)
+
+    def can_stage_in(self, file: File) -> bool:
+        return self.is_zip_url(file)
+
+    def is_zip_url(self, file: File) -> bool:
+        logger.debug("archive provider checking File {}".format(repr(file)))
+
+        # First check if this is the scheme we care about
+        if file.scheme != "zip":
+            return False
+
+        # This is some basic validation to check that the user isn't specifying
+        # an authority section and expecting it to mean something.
+        if file.netloc != "":
+            raise ZipAuthorityError(file)
+
+        # If we got this far, we can stage this file
+        return True
+
+    def stage_out(self, dm, executor, file, parent_fut):
+        assert file.scheme == 'zip'
+
+        zip_path, inside_path = zip_path_split(file.path)
+
+        working_dir = dm.dfk.executors[executor].working_dir
+
+        if working_dir:
+            file.local_path = os.path.join(working_dir, inside_path)
+
+            # TODO: I think its the right behaviour that a staging out provider should create the directory structure
+            # for the file to be placed in?
+            os.makedirs(os.path.dirname(file.local_path), exist_ok=True)
+        else:
+            raise RuntimeError("zip file staging requires a working_dir to be specified")
+
+        stage_out_app = _zip_stage_out_app(dm)
+        app_fut = stage_out_app(zip_path, inside_path, working_dir, inputs=[file], _parsl_staging_inhibit=True, parent_fut=parent_fut)
+        return app_fut
+
+    def stage_in(self, dm, executor, file, parent_fut):
+        assert file.scheme == 'zip'
+
+        zip_path, inside_path = zip_path_split(file.path)
+
+        working_dir = dm.dfk.executors[executor].working_dir
+
+        if working_dir:
+            file.local_path = os.path.join(working_dir, inside_path)
+
+        stage_in_app = _zip_stage_in_app(dm)
+        app_fut = stage_in_app(zip_path, inside_path, working_dir, outputs=[file], _parsl_staging_inhibit=True, parent_fut=parent_fut)
+        return app_fut._outputs[0]
+
+
+def _zip_stage_out(zip_file, inside_path, working_dir, parent_fut=None, inputs=[], _parsl_staging_inhibit=True):
+    file = inputs[0]
+
+    os.makedirs(os.path.dirname(zip_file), exist_ok=True)
+
+    with filelock.FileLock(zip_file + ".lock"):
+        with zipfile.ZipFile(zip_file, mode='a', compression=zipfile.ZIP_DEFLATED) as z:
+            z.write(file, arcname=inside_path)
+
+    os.remove(file)
+
+
+def _zip_stage_out_app(dm):
+    return parsl.python_app(executors=['_parsl_internal'], data_flow_kernel=dm.dfk)(_zip_stage_out)
+
+
+def _zip_stage_in(zip_file, inside_path, working_dir, *, parent_fut, outputs, _parsl_staging_inhibit=True):
+    with filelock.FileLock(zip_file + ".lock"):
+        with zipfile.ZipFile(zip_file, mode='r') as z:
+            content = z.read(inside_path)
+        with open(outputs[0], "wb") as of:
+            of.write(content)
+
+
+def _zip_stage_in_app(dm):
+    return parsl.python_app(executors=['_parsl_internal'], data_flow_kernel=dm.dfk)(_zip_stage_in)
+
+
+def zip_path_split(path: str) -> Tuple[str, str]:
+    """Split zip: path into a zipfile name and a contained-file name.
+    """
+    index = path.find(".zip/")
+
+    zip_path = path[:index + 4]
+    inside_path = path[index + 5:]
+
+    return (zip_path, inside_path)

parsl/dataflow/dependency_resolvers.py

@@ -0,0 +1,115 @@
+from concurrent.futures import Future
+from dataclasses import dataclass
+from functools import singledispatch
+from typing import Callable, Sequence
+
+
+@dataclass
+class DependencyResolver:
+    """A DependencyResolver describes how app dependencies can be resolved.
+    It is specified as two functions: `traverse_to_gather` which turns an
+    app parameter into a sequence of futures which must be waited for before
+    the task can be executed (for example, in the case of
+    `DEEP_DEPENDENCY_RESOLVER` this traverses structures such as lists to
+    find every contained ``Future``), and `traverse_to_unwrap` which turns an
+    app parameter into its value to be passed to the app on execution
+    (for example in the case of `DEEP_DEPENDENCY_RESOLVER` this replaces a
+    list containing futures with a new list containing the values of those
+    resolved futures).
+
+    By default, Parsl will use `SHALLOW_DEPENDENCY_RESOLVER` which only
+    resolves Futures passed directly as arguments.
+    """
+    traverse_to_gather: Callable[[object], Sequence[Future]]
+    traverse_to_unwrap: Callable[[object], object]
+
+
+@singledispatch
+def shallow_traverse_to_gather(o):
+    # objects in general do not expose futures that we can see
+    return []
+
+
+@singledispatch
+def shallow_traverse_to_unwrap(o):
+    # objects in general unwrap to themselves
+    return o
+
+
+@shallow_traverse_to_gather.register
+def _(fut: Future):
+    return [fut]
+
+
+@shallow_traverse_to_unwrap.register
+@singledispatch
+def _(fut: Future):
+    assert fut.done()
+    return fut.result()
+
+
+@singledispatch
+def deep_traverse_to_gather(o):
+    # objects in general do not expose futures that we can see
+    return []
+
+
+@singledispatch
+def deep_traverse_to_unwrap(o):
+    # objects in general unwrap to themselves
+    return o
+
+
+@deep_traverse_to_gather.register
+def _(fut: Future):
+    return [fut]
+
+
+@deep_traverse_to_unwrap.register
+@singledispatch
+def _(fut: Future):
+    assert fut.done()
+    return fut.result()
+
+
+@deep_traverse_to_gather.register(tuple)
+@deep_traverse_to_gather.register(list)
+@deep_traverse_to_gather.register(set)
+def _(iterable):
+    return [e for v in iterable for e in deep_traverse_to_gather(v)]
+
+
+@deep_traverse_to_unwrap.register(tuple)
+@deep_traverse_to_unwrap.register(list)
+@deep_traverse_to_unwrap.register(set)
+@singledispatch
+def _(iterable):
+
+    type_ = type(iterable)
+    return type_(map(deep_traverse_to_unwrap, iterable))
+
+
+@deep_traverse_to_gather.register(dict)
+def _(dictionary):
+    futures = []
+    for key, value in dictionary.items():
+        futures.extend(deep_traverse_to_gather(key))
+        futures.extend(deep_traverse_to_gather(value))
+    return futures
+
+
+@deep_traverse_to_unwrap.register(dict)
+def _(dictionary):
+    unwrapped_dict = {}
+    for key, value in dictionary.items():
+        key = deep_traverse_to_unwrap(key)
+        value = deep_traverse_to_unwrap(value)
+        unwrapped_dict[key] = value
+    return unwrapped_dict
+
+
+DEEP_DEPENDENCY_RESOLVER = DependencyResolver(traverse_to_gather=deep_traverse_to_gather,
+                                              traverse_to_unwrap=deep_traverse_to_unwrap)
+
+SHALLOW_DEPENDENCY_RESOLVER = DependencyResolver(traverse_to_gather=shallow_traverse_to_gather,
+                                                 traverse_to_unwrap=shallow_traverse_to_unwrap)