parsl 2024.9.2__tar.gz → 2024.9.9__tar.gz

{parsl-2024.9.2/parsl.egg-info → parsl-2024.9.9}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2024.9.2
+Version: 2024.9.9
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2024.09.02.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2024.09.09.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0

{parsl-2024.9.2 → parsl-2024.9.9}/parsl/executors/high_throughput/executor.py

@@ -199,9 +199,6 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
         will check the available memory at startup and limit the number of workers such that
         the there's sufficient memory for each worker. Default: None
 
-    max_workers : int
-        Deprecated. Please use max_workers_per_node instead.
-
     max_workers_per_node : int
         Caps the number of workers launched per node. Default: None
 
@@ -239,7 +236,6 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
                  worker_debug: bool = False,
                  cores_per_worker: float = 1.0,
                  mem_per_worker: Optional[float] = None,
-                 max_workers: Optional[Union[int, float]] = None,
                  max_workers_per_node: Optional[Union[int, float]] = None,
                  cpu_affinity: str = 'none',
                  available_accelerators: Union[int, Sequence[str]] = (),
@@ -272,9 +268,7 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
         else:
             self.all_addresses = ','.join(get_all_addresses())
 
-        if max_workers:
-            self._warn_deprecated("max_workers", "max_workers_per_node")
-        self.max_workers_per_node = max_workers_per_node or max_workers or float("inf")
+        self.max_workers_per_node = max_workers_per_node or float("inf")
 
         mem_slots = self.max_workers_per_node
         cpu_slots = self.max_workers_per_node
@@ -335,16 +329,6 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
             stacklevel=2
         )
 
-    @property
-    def max_workers(self):
-        self._warn_deprecated("max_workers", "max_workers_per_node")
-        return self.max_workers_per_node
-
-    @max_workers.setter
-    def max_workers(self, val: Union[int, float]):
-        self._warn_deprecated("max_workers", "max_workers_per_node")
-        self.max_workers_per_node = val
-
     @property
     def logdir(self):
         return "{}/{}".format(self.run_dir, self.label)

{parsl-2024.9.2 → parsl-2024.9.9}/parsl/executors/taskvine/executor.py

@@ -3,6 +3,7 @@ Cooperative Computing Lab (CCL) at Notre Dame to provide a fault-tolerant,
 high-throughput system for delegating Parsl tasks to thousands of remote machines
 """
 
+import getpass
 import hashlib
 import inspect
 import itertools
@@ -18,6 +19,7 @@ import tempfile
 import threading
 import uuid
 from concurrent.futures import Future
+from datetime import datetime
 from typing import List, Literal, Optional, Union
 
 # Import other libraries
@@ -215,9 +217,9 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
 
         # Create directories for data and results
         log_dir = os.path.join(run_dir, self.label)
-        self._function_data_dir = os.path.join(run_dir, self.label, "function_data")
         os.makedirs(log_dir)
-        os.makedirs(self._function_data_dir)
+        tmp_prefix = f'{self.label}-{getpass.getuser()}-{datetime.now().strftime("%Y%m%d%H%M%S%f")}-'
+        self._function_data_dir = tempfile.TemporaryDirectory(prefix=tmp_prefix)
 
         # put TaskVine logs outside of a Parsl run as TaskVine caches between runs while
         # Parsl does not.
@@ -227,7 +229,7 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
 
         # factory logs go with manager logs regardless
         self.factory_config.scratch_dir = self.manager_config.vine_log_dir
-        logger.debug(f"Function data directory: {self._function_data_dir}, log directory: {log_dir}")
+        logger.debug(f"Function data directory: {self._function_data_dir.name}, log directory: {log_dir}")
         logger.debug(
             f"TaskVine manager log directory: {self.manager_config.vine_log_dir}, "
             f"factory log directory: {self.factory_config.scratch_dir}")
@@ -293,7 +295,7 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
             'map': Pickled file with a dict between local parsl names, and remote taskvine names.
         """
         task_dir = "{:04d}".format(executor_task_id)
-        return os.path.join(self._function_data_dir, task_dir, *path_components)
+        return os.path.join(self._function_data_dir.name, task_dir, *path_components)
 
     def submit(self, func, resource_specification, *args, **kwargs):
         """Processes the Parsl app by its arguments and submits the function

parsl-2024.9.9/parsl/tests/configs/slurm_local.py

@@ -0,0 +1,26 @@
+from parsl.channels import LocalChannel
+from parsl.config import Config
+from parsl.executors import HighThroughputExecutor
+from parsl.launchers import SrunLauncher
+from parsl.providers import SlurmProvider
+
+
+def fresh_config():
+    return Config(
+        executors=[
+            HighThroughputExecutor(
+                label="docker_slurm",
+                encrypted=True,
+                provider=SlurmProvider(
+                    cmd_timeout=60,     # Add extra time for slow scheduler responses
+                    channel=LocalChannel(),
+                    nodes_per_block=1,
+                    init_blocks=1,
+                    min_blocks=1,
+                    max_blocks=1,
+                    walltime='00:10:00',
+                    launcher=SrunLauncher(),
+                ),
+            )
+        ],
+    )

{parsl-2024.9.2 → parsl-2024.9.9}/parsl/tests/test_htex/test_htex.py

@@ -126,18 +126,6 @@ def test_htex_shutdown(
         assert "HighThroughputExecutor has not started" in caplog.text
 
 
-@pytest.mark.local
-def test_max_workers_per_node():
-    with pytest.warns(DeprecationWarning) as record:
-        htex = HighThroughputExecutor(max_workers_per_node=1, max_workers=2)
-
-    warning_msg = "max_workers is deprecated"
-    assert any(warning_msg in str(warning.message) for warning in record)
-
-    # Ensure max_workers_per_node takes precedence
-    assert htex.max_workers_per_node == htex.max_workers == 1
-
-
 @pytest.mark.local
 @pytest.mark.parametrize("cmd", (None, "custom-launch-cmd"))
 def test_htex_worker_pool_launch_cmd(cmd: Optional[str]):

{parsl-2024.9.2 → parsl-2024.9.9}/parsl/tests/test_mpi_apps/test_mpiex.py

@@ -43,7 +43,7 @@ def test_init():
 
     new_kwargs = {'max_workers_per_block', 'mpi_launcher'}
     excluded_kwargs = {'available_accelerators', 'cores_per_worker', 'max_workers_per_node',
-                       'mem_per_worker', 'cpu_affinity', 'max_workers', 'manager_selector'}
+                       'mem_per_worker', 'cpu_affinity', 'manager_selector'}
 
     # Get the kwargs from both HTEx and MPIEx
     htex_kwargs = set(signature(HighThroughputExecutor.__init__).parameters)

{parsl-2024.9.2 → parsl-2024.9.9}/parsl/tests/test_scaling/test_scale_down_htex_auto_scale.py

@@ -23,7 +23,7 @@ def local_config():
                 poll_period=100,
                 label="htex_local",
                 address="127.0.0.1",
-                max_workers=1,
+                max_workers_per_node=1,
                 encrypted=True,
                 provider=LocalProvider(
                     channel=LocalChannel(),

{parsl-2024.9.2 → parsl-2024.9.9}/parsl/tests/test_scaling/test_scale_down_htex_unregistered.py

@@ -27,7 +27,7 @@ def local_config():
                 poll_period=100,
                 label="htex_local",
                 address="127.0.0.1",
-                max_workers=1,
+                max_workers_per_node=1,
                 encrypted=True,
                 launch_cmd="sleep inf",
                 provider=LocalProvider(

{parsl-2024.9.2 → parsl-2024.9.9}/parsl/version.py

@@ -3,4 +3,4 @@
 Year.Month.Day[alpha/beta/..]
 Alphas will be numbered like this -> 2024.12.10a0
 """
-VERSION = '2024.09.02'
+VERSION = '2024.09.09'

{parsl-2024.9.2 → parsl-2024.9.9/parsl.egg-info}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2024.9.2
+Version: 2024.9.9
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2024.09.02.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2024.09.09.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0

{parsl-2024.9.2 → parsl-2024.9.9}/parsl.egg-info/SOURCES.txt

@@ -251,6 +251,7 @@ parsl/tests/configs/midway.py
 parsl/tests/configs/nscc_singapore.py
 parsl/tests/configs/osg_htex.py
 parsl/tests/configs/petrelkube.py
+parsl/tests/configs/slurm_local.py
 parsl/tests/configs/summit.py
 parsl/tests/configs/taskvine_ex.py
 parsl/tests/configs/theta.py