parsl 2024.8.5__py3-none-any.whl → 2024.8.19__py3-none-any.whl

parsl/monitoring/router.py

@@ -1,15 +1,16 @@
 from __future__ import annotations
 
 import logging
+import multiprocessing.queues as mpq
 import os
 import pickle
-import queue
 import socket
 import threading
 import time
 from multiprocessing.synchronize import Event
-from typing import Optional, Tuple, Union
+from typing import Optional, Tuple
 
+import typeguard
 import zmq
 
 from parsl.log_utils import set_file_logger
@@ -31,13 +32,12 @@ class MonitoringRouter:
 
                  monitoring_hub_address: str = "127.0.0.1",
                  logdir: str = ".",
-                 run_id: str,
                  logging_level: int = logging.INFO,
                  atexit_timeout: int = 3,   # in seconds
-                 priority_msgs: "queue.Queue[AddressedMonitoringMessage]",
-                 node_msgs: "queue.Queue[AddressedMonitoringMessage]",
-                 block_msgs: "queue.Queue[AddressedMonitoringMessage]",
-                 resource_msgs: "queue.Queue[AddressedMonitoringMessage]",
+                 priority_msgs: mpq.Queue,
+                 node_msgs: mpq.Queue,
+                 block_msgs: mpq.Queue,
+                 resource_msgs: mpq.Queue,
                  exit_event: Event,
                  ):
         """ Initializes a monitoring configuration class.
@@ -71,7 +71,6 @@ class MonitoringRouter:
 
         self.hub_address = hub_address
         self.atexit_timeout = atexit_timeout
-        self.run_id = run_id
 
         self.loop_freq = 10.0  # milliseconds
 
@@ -172,7 +171,6 @@ class MonitoringRouter:
                         msg_0 = (msg, 0)
 
                         if msg[0] == MessageType.NODE_INFO:
-                            msg[1]['run_id'] = self.run_id
                             self.node_msgs.put(msg_0)
                         elif msg[0] == MessageType.RESOURCE_INFO:
                             self.resource_msgs.put(msg_0)
@@ -205,12 +203,14 @@ class MonitoringRouter:
 
 
 @wrap_with_logs
-def router_starter(comm_q: "queue.Queue[Union[Tuple[int, int], str]]",
-                   exception_q: "queue.Queue[Tuple[str, str]]",
-                   priority_msgs: "queue.Queue[AddressedMonitoringMessage]",
-                   node_msgs: "queue.Queue[AddressedMonitoringMessage]",
-                   block_msgs: "queue.Queue[AddressedMonitoringMessage]",
-                   resource_msgs: "queue.Queue[AddressedMonitoringMessage]",
+@typeguard.typechecked
+def router_starter(*,
+                   comm_q: mpq.Queue,
+                   exception_q: mpq.Queue,
+                   priority_msgs: mpq.Queue,
+                   node_msgs: mpq.Queue,
+                   block_msgs: mpq.Queue,
+                   resource_msgs: mpq.Queue,
                    exit_event: Event,
 
                    hub_address: str,
@@ -218,8 +218,7 @@ def router_starter(comm_q: "queue.Queue[Union[Tuple[int, int], str]]",
                    zmq_port_range: Tuple[int, int],
 
                    logdir: str,
-                   logging_level: int,
-                   run_id: str) -> None:
+                   logging_level: int) -> None:
     setproctitle("parsl: monitoring router")
     try:
         router = MonitoringRouter(hub_address=hub_address,
@@ -227,7 +226,6 @@ def router_starter(comm_q: "queue.Queue[Union[Tuple[int, int], str]]",
                                   zmq_port_range=zmq_port_range,
                                   logdir=logdir,
                                   logging_level=logging_level,
-                                  run_id=run_id,
                                   priority_msgs=priority_msgs,
                                   node_msgs=node_msgs,
                                   block_msgs=block_msgs,

parsl/providers/__init__.py

@@ -1,6 +1,3 @@
-# Workstation Provider
-from parsl.providers.ad_hoc.ad_hoc import AdHocProvider
-
 # Cloud Providers
 from parsl.providers.aws.aws import AWSProvider
 from parsl.providers.azure.azure import AzureProvider
@@ -24,7 +21,6 @@ __all__ = ['LocalProvider',
            'SlurmProvider',
            'TorqueProvider',
            'LSFProvider',
-           'AdHocProvider',
            'PBSProProvider',
            'AWSProvider',
            'GoogleCloudProvider',

parsl/providers/ad_hoc/ad_hoc.py

@@ -12,8 +12,12 @@ from parsl.utils import RepresentationMixin
 logger = logging.getLogger(__name__)
 
 
-class AdHocProvider(ExecutionProvider, RepresentationMixin):
-    """ Ad-hoc execution provider
+class DeprecatedAdHocProvider(ExecutionProvider, RepresentationMixin):
+    """ Deprecated ad-hoc execution provider
+
+    The (former) AdHocProvider is deprecated. See
+    `issue #3515 <https://github.com/Parsl/parsl/issues/3515>`_
+    for further discussion.
 
     This provider is used to provision execution resources over one or more ad hoc nodes
     that are each accessible over a Channel (say, ssh) but otherwise lack a cluster scheduler.

parsl/tests/configs/local_adhoc.py

@@ -1,7 +1,7 @@
 from parsl.channels import LocalChannel
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
-from parsl.providers import AdHocProvider
+from parsl.providers.ad_hoc.ad_hoc import DeprecatedAdHocProvider
 
 
 def fresh_config():
@@ -10,7 +10,7 @@ def fresh_config():
             HighThroughputExecutor(
                 label='AdHoc',
                 encrypted=True,
-                provider=AdHocProvider(
+                provider=DeprecatedAdHocProvider(
                     channels=[LocalChannel(), LocalChannel()]
                 )
             )

parsl/tests/test_htex/test_resource_spec_validation.py

@@ -0,0 +1,40 @@
+import queue
+from unittest import mock
+
+import pytest
+
+from parsl.executors import HighThroughputExecutor
+from parsl.executors.high_throughput.mpi_prefix_composer import (
+    InvalidResourceSpecification,
+)
+
+
+def double(x):
+    return x * 2
+
+
+@pytest.mark.local
+def test_submit_calls_validate():
+
+    htex = HighThroughputExecutor()
+    htex.outgoing_q = mock.Mock(spec=queue.Queue)
+    htex.validate_resource_spec = mock.Mock(spec=htex.validate_resource_spec)
+
+    res_spec = {}
+    htex.submit(double, res_spec, (5,), {})
+    htex.validate_resource_spec.assert_called()
+
+
+@pytest.mark.local
+def test_resource_spec_validation():
+    htex = HighThroughputExecutor()
+    ret_val = htex.validate_resource_spec({})
+    assert ret_val is None
+
+
+@pytest.mark.local
+def test_resource_spec_validation_bad_keys():
+    htex = HighThroughputExecutor()
+
+    with pytest.raises(InvalidResourceSpecification):
+        htex.validate_resource_spec({"num_nodes": 2})

parsl/tests/test_htex/test_zmq_binding.py

@@ -25,7 +25,8 @@ def make_interchange(*, interchange_address: Optional[str], cert_dir: Optional[s
                        logdir=".",
                        logging_level=logging.INFO,
                        manager_selector=RandomManagerSelector(),
-                       poll_period=10)
+                       poll_period=10,
+                       run_id="test_run_id")
 
 
 @pytest.fixture

parsl/tests/test_mpi_apps/test_bad_mpi_config.py

@@ -1,33 +1,48 @@
 import pytest
 
 from parsl import Config
-from parsl.executors import HighThroughputExecutor
+from parsl.executors import MPIExecutor
 from parsl.launchers import AprunLauncher, SimpleLauncher, SrunLauncher
 from parsl.providers import SlurmProvider
 
 
 @pytest.mark.local
-def test_bad_launcher_with_mpi_mode():
-    """AssertionError if a launcher other than SimpleLauncher is supplied"""
+def test_bad_launcher():
+    """TypeError if a launcher other than SimpleLauncher is supplied"""
 
     for launcher in [SrunLauncher(), AprunLauncher()]:
-        with pytest.raises(AssertionError):
+        with pytest.raises(TypeError):
             Config(executors=[
-                HighThroughputExecutor(
-                    enable_mpi_mode=True,
+                MPIExecutor(
                     provider=SlurmProvider(launcher=launcher),
                 )
             ])
 
 
 @pytest.mark.local
-def test_correct_launcher_with_mpi_mode():
+def test_bad_mpi_launcher():
+    """ValueError if an unsupported mpi_launcher is specified"""
+
+    with pytest.raises(ValueError):
+        Config(executors=[
+            MPIExecutor(
+                mpi_launcher="bad_launcher",
+                provider=SlurmProvider(launcher=SimpleLauncher()),
+            )
+        ])
+
+
+@pytest.mark.local
+@pytest.mark.parametrize(
+    "mpi_launcher",
+    ["srun", "aprun", "mpiexec"]
+)
+def test_correct_launcher_with_mpi_mode(mpi_launcher: str):
     """Confirm that SimpleLauncher works with mpi_mode"""
 
-    config = Config(executors=[
-        HighThroughputExecutor(
-            enable_mpi_mode=True,
-            provider=SlurmProvider(launcher=SimpleLauncher()),
-        )
-    ])
-    assert isinstance(config.executors[0].provider.launcher, SimpleLauncher)
+    executor = MPIExecutor(
+        mpi_launcher=mpi_launcher,
+        provider=SlurmProvider(launcher=SimpleLauncher()),
+    )
+
+    assert isinstance(executor.provider.launcher, SimpleLauncher)

parsl/tests/test_mpi_apps/test_mpi_mode_enabled.py

@@ -6,26 +6,34 @@ from typing import Dict
 import pytest
 
 import parsl
-from parsl import bash_app, python_app
+from parsl import Config, bash_app, python_app
+from parsl.executors import MPIExecutor
 from parsl.executors.high_throughput.mpi_prefix_composer import (
     MissingResourceSpecification,
 )
-from parsl.tests.configs.htex_local import fresh_config
+from parsl.launchers import SimpleLauncher
+from parsl.providers import LocalProvider
 
 EXECUTOR_LABEL = "MPI_TEST"
 
 
 def local_setup():
-    config = fresh_config()
-    config.executors[0].label = EXECUTOR_LABEL
-    config.executors[0].max_workers_per_node = 2
-    config.executors[0].enable_mpi_mode = True
-    config.executors[0].mpi_launcher = "mpiexec"
 
     cwd = os.path.abspath(os.path.dirname(__file__))
     pbs_nodefile = os.path.join(cwd, "mocks", "pbs_nodefile")
 
-    config.executors[0].provider.worker_init = f"export PBS_NODEFILE={pbs_nodefile}"
+    config = Config(
+        executors=[
+            MPIExecutor(
+                label=EXECUTOR_LABEL,
+                max_workers_per_block=2,
+                mpi_launcher="mpiexec",
+                provider=LocalProvider(
+                    worker_init=f"export PBS_NODEFILE={pbs_nodefile}",
+                    launcher=SimpleLauncher()
+                )
+            )
+        ])
 
     parsl.load(config)
 

parsl/tests/test_mpi_apps/test_mpiex.py

@@ -4,7 +4,6 @@ from pathlib import Path
 
 import pytest
 
-import parsl
 from parsl import Config, HighThroughputExecutor
 from parsl.executors.high_throughput.mpi_executor import MPIExecutor
 from parsl.launchers import SimpleLauncher
@@ -42,8 +41,8 @@ def test_docstring():
 def test_init():
     """Ensure all relevant kwargs are copied over from HTEx"""
 
-    new_kwargs = {'max_workers_per_block'}
-    excluded_kwargs = {'available_accelerators', 'enable_mpi_mode', 'cores_per_worker', 'max_workers_per_node',
+    new_kwargs = {'max_workers_per_block', 'mpi_launcher'}
+    excluded_kwargs = {'available_accelerators', 'cores_per_worker', 'max_workers_per_node',
                        'mem_per_worker', 'cpu_affinity', 'max_workers', 'manager_selector'}
 
     # Get the kwargs from both HTEx and MPIEx

parsl/tests/test_mpi_apps/test_resource_spec.py

@@ -1,18 +1,20 @@
 import contextlib
 import logging
 import os
+import queue
 import typing
 import unittest
 from typing import Dict
+from unittest import mock
 
 import pytest
 
-import parsl
 from parsl.app.app import python_app
+from parsl.executors.high_throughput.executor import HighThroughputExecutor
+from parsl.executors.high_throughput.mpi_executor import MPIExecutor
 from parsl.executors.high_throughput.mpi_prefix_composer import (
     InvalidResourceSpecification,
     MissingResourceSpecification,
-    validate_resource_spec,
 )
 from parsl.executors.high_throughput.mpi_resource_management import (
     get_nodes_in_batchjob,
@@ -20,6 +22,8 @@ from parsl.executors.high_throughput.mpi_resource_management import (
     get_slurm_hosts_list,
     identify_scheduler,
 )
+from parsl.launchers import SimpleLauncher
+from parsl.providers import LocalProvider
 from parsl.tests.configs.htex_local import fresh_config
 
 EXECUTOR_LABEL = "MPI_TEST"
@@ -48,23 +52,6 @@ def get_env_vars(parsl_resource_specification: Dict = {}) -> Dict:
     return parsl_vars
 
 
-@pytest.mark.local
-def test_resource_spec_env_vars():
-    resource_spec = {
-        "num_nodes": 4,
-        "ranks_per_node": 2,
-    }
-
-    assert double(5).result() == 10
-
-    future = get_env_vars(parsl_resource_specification=resource_spec)
-
-    result = future.result()
-    assert isinstance(result, Dict)
-    assert result["PARSL_NUM_NODES"] == str(resource_spec["num_nodes"])
-    assert result["PARSL_RANKS_PER_NODE"] == str(resource_spec["ranks_per_node"])
-
-
 @pytest.mark.local
 @unittest.mock.patch("subprocess.check_output", return_value=b"c203-031\nc203-032\n")
 def test_slurm_mocked_mpi_fetch(subprocess_check):
@@ -83,16 +70,6 @@ def add_to_path(path: os.PathLike) -> typing.Generator[None, None, None]:
         os.environ["PATH"] = old_path
 
 
-@pytest.mark.local
-@pytest.mark.skip
-def test_slurm_mpi_fetch():
-    logging.warning(f"Current pwd : {os.path.dirname(__file__)}")
-    with add_to_path(os.path.dirname(__file__)):
-        logging.warning(f"PATH: {os.environ['PATH']}")
-        nodeinfo = get_slurm_hosts_list()
-    logging.warning(f"Got : {nodeinfo}")
-
-
 @contextlib.contextmanager
 def mock_pbs_nodefile(nodefile: str = "pbs_nodefile") -> typing.Generator[None, None, None]:
     cwd = os.path.abspath(os.path.dirname(__file__))
@@ -122,22 +99,43 @@ def test_top_level():
 
 @pytest.mark.local
 @pytest.mark.parametrize(
-    "resource_spec, is_mpi_enabled, exception",
+    "resource_spec, exception",
     (
-        ({"num_nodes": 2, "ranks_per_node": 1}, False, None),
-        ({"launcher_options": "--debug_foo"}, False, None),
-        ({"num_nodes": 2, "BAD_OPT": 1}, False, InvalidResourceSpecification),
-        ({}, False, None),
-        ({"num_nodes": 2, "ranks_per_node": 1}, True, None),
-        ({"launcher_options": "--debug_foo"}, True, None),
-        ({"num_nodes": 2, "BAD_OPT": 1}, True, InvalidResourceSpecification),
-        ({}, True, MissingResourceSpecification),
+
+        ({"num_nodes": 2, "ranks_per_node": 1}, None),
+        ({"launcher_options": "--debug_foo"}, None),
+        ({"num_nodes": 2, "BAD_OPT": 1}, InvalidResourceSpecification),
+        ({}, MissingResourceSpecification),
     )
 )
-def test_resource_spec(resource_spec: Dict, is_mpi_enabled: bool, exception):
+def test_mpi_resource_spec(resource_spec: Dict, exception):
+    """Test validation of resource_specification in MPIExecutor"""
+
+    mpi_ex = MPIExecutor(provider=LocalProvider(launcher=SimpleLauncher()))
+    mpi_ex.outgoing_q = mock.Mock(spec=queue.Queue)
+
     if exception:
         with pytest.raises(exception):
-            validate_resource_spec(resource_spec, is_mpi_enabled)
+            mpi_ex.validate_resource_spec(resource_spec)
     else:
-        result = validate_resource_spec(resource_spec, is_mpi_enabled)
+        result = mpi_ex.validate_resource_spec(resource_spec)
         assert result is None
+
+
+@pytest.mark.local
+@pytest.mark.parametrize(
+    "resource_spec",
+    (
+        {"num_nodes": 2, "ranks_per_node": 1},
+        {"launcher_options": "--debug_foo"},
+        {"BAD_OPT": 1},
+    )
+)
+def test_mpi_resource_spec_passed_to_htex(resource_spec: dict):
+    """HTEX should reject every resource_spec"""
+
+    htex = HighThroughputExecutor()
+    htex.outgoing_q = mock.Mock(spec=queue.Queue)
+
+    with pytest.raises(InvalidResourceSpecification):
+        htex.validate_resource_spec(resource_spec)

parsl/tests/test_providers/test_local_provider.py

@@ -11,7 +11,8 @@ import time
 
 import pytest
 
-from parsl.channels import LocalChannel, SSHChannel
+from parsl.channels import LocalChannel
+from parsl.channels.ssh.ssh import DeprecatedSSHChannel
 from parsl.jobs.states import JobState
 from parsl.launchers import SingleNodeLauncher
 from parsl.providers import LocalProvider
@@ -92,10 +93,10 @@ def test_ssh_channel():
                 # already exist, so create it here.
                 pathlib.Path('{}/known.hosts'.format(config_dir)).touch(mode=0o600)
                 script_dir = tempfile.mkdtemp()
-                channel = SSHChannel('127.0.0.1', port=server_port,
-                                     script_dir=remote_script_dir,
-                                     host_keys_filename='{}/known.hosts'.format(config_dir),
-                                     key_filename=priv_key)
+                channel = DeprecatedSSHChannel('127.0.0.1', port=server_port,
+                                               script_dir=remote_script_dir,
+                                               host_keys_filename='{}/known.hosts'.format(config_dir),
+                                               key_filename=priv_key)
                 try:
                     p = LocalProvider(channel=channel,
                                       launcher=SingleNodeLauncher(debug=False))

parsl/version.py

@@ -3,4 +3,4 @@
 Year.Month.Day[alpha/beta/..]
 Alphas will be numbered like this -> 2024.12.10a0
 """
-VERSION = '2024.08.05'
+VERSION = '2024.08.19'

{parsl-2024.8.5.data → parsl-2024.8.19.data}/scripts/interchange.py

@@ -55,6 +55,7 @@ class Interchange:
                  poll_period: int,
                  cert_dir: Optional[str],
                  manager_selector: ManagerSelector,
+                 run_id: str,
                  ) -> None:
         """
         Parameters
@@ -125,6 +126,8 @@ class Interchange:
         self.command_channel.connect("tcp://{}:{}".format(client_address, client_ports[2]))
         logger.info("Connected to client")
 
+        self.run_id = run_id
+
         self.hub_address = hub_address
         self.hub_zmq_port = hub_zmq_port
 
@@ -227,6 +230,7 @@ class Interchange:
             d: Dict = cast(Dict, manager.copy())
             d['timestamp'] = datetime.datetime.now()
             d['last_heartbeat'] = datetime.datetime.fromtimestamp(d['last_heartbeat'])
+            d['run_id'] = self.run_id
 
             monitoring_radio.send((MessageType.NODE_INFO, d))
 

{parsl-2024.8.5.dist-info → parsl-2024.8.19.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2024.8.5
+Version: 2024.8.19
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2024.08.05.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2024.08.19.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0
@@ -25,7 +25,6 @@ Requires-Dist: globus-sdk
 Requires-Dist: dill
 Requires-Dist: tblib
 Requires-Dist: requests
-Requires-Dist: paramiko
 Requires-Dist: psutil>=5.5.1
 Requires-Dist: setproctitle
 Requires-Dist: filelock<4,>=3.13
@@ -57,6 +56,7 @@ Requires-Dist: jsonschema; extra == "all"
 Requires-Dist: proxystore; extra == "all"
 Requires-Dist: radical.pilot==1.60; extra == "all"
 Requires-Dist: radical.utils==1.60; extra == "all"
+Requires-Dist: paramiko; extra == "all"
 Provides-Extra: aws
 Requires-Dist: boto3; extra == "aws"
 Provides-Extra: azure
@@ -87,6 +87,8 @@ Requires-Dist: proxystore; extra == "proxystore"
 Provides-Extra: radical-pilot
 Requires-Dist: radical.pilot==1.60; extra == "radical-pilot"
 Requires-Dist: radical.utils==1.60; extra == "radical-pilot"
+Provides-Extra: ssh
+Requires-Dist: paramiko; extra == "ssh"
 Provides-Extra: visualization
 Requires-Dist: pydot; extra == "visualization"
 Requires-Dist: networkx<2.6,>=2.5; extra == "visualization"