parsl 2024.11.4__py3-none-any.whl → 2024.11.18__py3-none-any.whl

parsl/monitoring/router.py

@@ -14,6 +14,7 @@ import typeguard
 import zmq
 
 from parsl.log_utils import set_file_logger
+from parsl.monitoring.radios import MultiprocessingQueueRadioSender
 from parsl.monitoring.types import TaggedMonitoringMessage
 from parsl.process_loggers import wrap_with_logs
 from parsl.utils import setproctitle
@@ -30,7 +31,7 @@ class MonitoringRouter:
                  zmq_port_range: Tuple[int, int] = (55050, 56000),
 
                  monitoring_hub_address: str = "127.0.0.1",
-                 logdir: str = ".",
+                 run_dir: str = ".",
                  logging_level: int = logging.INFO,
                  atexit_timeout: int = 3,   # in seconds
                  resource_msgs: mpq.Queue,
@@ -47,7 +48,7 @@ class MonitoringRouter:
         zmq_port_range : tuple(int, int)
              The MonitoringHub picks ports at random from the range which will be used by Hub.
              Default: (55050, 56000)
-        logdir : str
+        run_dir : str
              Parsl log directory paths. Logs and temp files go here. Default: '.'
         logging_level : int
              Logging level as defined in the logging module. Default: logging.INFO
@@ -55,12 +56,11 @@ class MonitoringRouter:
             The amount of time in seconds to terminate the hub without receiving any messages, after the last dfk workflow message is received.
         resource_msgs : multiprocessing.Queue
             A multiprocessing queue to receive messages to be routed onwards to the database process
-
         exit_event : Event
             An event that the main Parsl process will set to signal that the monitoring router should shut down.
         """
-        os.makedirs(logdir, exist_ok=True)
-        self.logger = set_file_logger("{}/monitoring_router.log".format(logdir),
+        os.makedirs(run_dir, exist_ok=True)
+        self.logger = set_file_logger(f"{run_dir}/monitoring_router.log",
                                       name="monitoring_router",
                                       level=logging_level)
         self.logger.debug("Monitoring router starting")
@@ -98,7 +98,7 @@ class MonitoringRouter:
                                                                                min_port=zmq_port_range[0],
                                                                                max_port=zmq_port_range[1])
 
-        self.resource_msgs = resource_msgs
+        self.target_radio = MultiprocessingQueueRadioSender(resource_msgs)
         self.exit_event = exit_event
 
     @wrap_with_logs(target="monitoring_router")
@@ -125,7 +125,7 @@ class MonitoringRouter:
                     data, addr = self.udp_sock.recvfrom(2048)
                     resource_msg = pickle.loads(data)
                     self.logger.debug("Got UDP Message from {}: {}".format(addr, resource_msg))
-                    self.resource_msgs.put(resource_msg)
+                    self.target_radio.send(resource_msg)
                 except socket.timeout:
                     pass
 
@@ -136,7 +136,7 @@ class MonitoringRouter:
                     data, addr = self.udp_sock.recvfrom(2048)
                     msg = pickle.loads(data)
                     self.logger.debug("Got UDP Message from {}: {}".format(addr, msg))
-                    self.resource_msgs.put(msg)
+                    self.target_radio.send(msg)
                     last_msg_received_time = time.time()
                 except socket.timeout:
                     pass
@@ -160,7 +160,7 @@ class MonitoringRouter:
                         assert len(msg) >= 1, "ZMQ Receiver expects tuples of length at least 1, got {}".format(msg)
                         assert len(msg) == 2, "ZMQ Receiver expects message tuples of exactly length 2, got {}".format(msg)
 
-                        self.resource_msgs.put(msg)
+                        self.target_radio.send(msg)
                 except zmq.Again:
                     pass
                 except Exception:
@@ -187,14 +187,14 @@ def router_starter(*,
                    udp_port: Optional[int],
                    zmq_port_range: Tuple[int, int],
 
-                   logdir: str,
+                   run_dir: str,
                    logging_level: int) -> None:
     setproctitle("parsl: monitoring router")
     try:
         router = MonitoringRouter(hub_address=hub_address,
                                   udp_port=udp_port,
                                   zmq_port_range=zmq_port_range,
-                                  logdir=logdir,
+                                  run_dir=run_dir,
                                   logging_level=logging_level,
                                   resource_msgs=resource_msgs,
                                   exit_event=exit_event)

parsl/providers/cluster_provider.py

@@ -18,10 +18,7 @@ class ClusterProvider(ExecutionProvider):
       label : str
         Label for this provider.
       channel : Channel
-        Channel for accessing this provider. Possible channels include
-        :class:`~parsl.channels.LocalChannel` (the default),
-        :class:`~parsl.channels.SSHChannel`, or
-        :class:`~parsl.channels.SSHInteractiveLoginChannel`.
+        Channel for accessing this provider.
       walltime : str
         Walltime requested per block in HH:MM:SS.
       launcher : Launcher

parsl/providers/condor/condor.py

@@ -37,10 +37,7 @@ class CondorProvider(RepresentationMixin, ClusterProvider):
     Parameters
     ----------
     channel : Channel
-        Channel for accessing this provider. Possible channels include
-        :class:`~parsl.channels.LocalChannel` (the default),
-        :class:`~parsl.channels.SSHChannel`, or
-        :class:`~parsl.channels.SSHInteractiveLoginChannel`.
+        Channel for accessing this provider.
     nodes_per_block : int
         Nodes to provision per block.
     cores_per_slot : int

parsl/providers/grid_engine/grid_engine.py

@@ -37,10 +37,7 @@ class GridEngineProvider(ClusterProvider, RepresentationMixin):
     Parameters
     ----------
     channel : Channel
-        Channel for accessing this provider. Possible channels include
-        :class:`~parsl.channels.LocalChannel` (the default),
-        :class:`~parsl.channels.SSHChannel`, or
-        :class:`~parsl.channels.SSHInteractiveLoginChannel`.
+        Channel for accessing this provider.
     nodes_per_block : int
         Nodes to provision per block.
     min_blocks : int

parsl/providers/lsf/lsf.py

@@ -33,10 +33,7 @@ class LSFProvider(ClusterProvider, RepresentationMixin):
     Parameters
     ----------
     channel : Channel
-        Channel for accessing this provider. Possible channels include
-        :class:`~parsl.channels.LocalChannel` (the default),
-        :class:`~parsl.channels.SSHChannel`, or
-        :class:`~parsl.channels.SSHInteractiveLoginChannel`.
+        Channel for accessing this provider.
     nodes_per_block : int
         Nodes to provision per block.
         When request_by_nodes is False, it is computed by cores_per_block / cores_per_node.

parsl/providers/pbspro/pbspro.py

@@ -18,10 +18,7 @@ class PBSProProvider(TorqueProvider):
     Parameters
     ----------
     channel : Channel
-        Channel for accessing this provider. Possible channels include
-        :class:`~parsl.channels.LocalChannel` (the default),
-        :class:`~parsl.channels.SSHChannel`, or
-        :class:`~parsl.channels.SSHInteractiveLoginChannel`.
+        Channel for accessing this provider.
     account : str
         Account the job will be charged against.
     queue : str

parsl/providers/slurm/slurm.py

@@ -70,11 +70,11 @@ class SlurmProvider(ClusterProvider, RepresentationMixin):
         Slurm queue to place job in. If unspecified or ``None``, no queue slurm directive will be specified.
     constraint : str
         Slurm job constraint, often used to choose cpu or gpu type. If unspecified or ``None``, no constraint slurm directive will be added.
+    clusters : str
+        Slurm cluster name, or comma seperated cluster list, used to choose between different clusters in a federated Slurm instance.
+        If unspecified or ``None``, no slurm directive for clusters will be added.
     channel : Channel
-        Channel for accessing this provider. Possible channels include
-        :class:`~parsl.channels.LocalChannel` (the default),
-        :class:`~parsl.channels.SSHChannel`, or
-        :class:`~parsl.channels.SSHInteractiveLoginChannel`.
+        Channel for accessing this provider.
     nodes_per_block : int
         Nodes to provision per block.
     cores_per_node : int
@@ -119,6 +119,7 @@ class SlurmProvider(ClusterProvider, RepresentationMixin):
                  account: Optional[str] = None,
                  qos: Optional[str] = None,
                  constraint: Optional[str] = None,
+                 clusters: Optional[str] = None,
                  channel: Channel = LocalChannel(),
                  nodes_per_block: int = 1,
                  cores_per_node: Optional[int] = None,
@@ -155,6 +156,7 @@ class SlurmProvider(ClusterProvider, RepresentationMixin):
         self.account = account
         self.qos = qos
         self.constraint = constraint
+        self.clusters = clusters
         self.scheduler_options = scheduler_options + '\n'
         if exclusive:
             self.scheduler_options += "#SBATCH --exclusive\n"
@@ -166,6 +168,8 @@ class SlurmProvider(ClusterProvider, RepresentationMixin):
             self.scheduler_options += "#SBATCH --qos={}\n".format(qos)
         if constraint:
             self.scheduler_options += "#SBATCH --constraint={}\n".format(constraint)
+        if clusters:
+            self.scheduler_options += "#SBATCH --clusters={}\n".format(clusters)
 
         self.regex_job_id = regex_job_id
         self.worker_init = worker_init + '\n'
@@ -177,14 +181,22 @@ class SlurmProvider(ClusterProvider, RepresentationMixin):
         logger.debug(f"sacct returned retcode={retcode} stderr={stderr}")
         if retcode == 0:
             logger.debug("using sacct to get job status")
+            _cmd = "sacct"
+            # Add clusters option to sacct if provided
+            if self.clusters:
+                _cmd += f" --clusters={self.clusters}"
             # Using state%20 to get enough characters to not truncate output
             # of the state. Without output can look like "<job_id>     CANCELLED+"
-            self._cmd = "sacct -X --noheader --format=jobid,state%20 --job '{0}'"
+            self._cmd = _cmd + " -X --noheader --format=jobid,state%20 --job '{0}'"
             self._translate_table = sacct_translate_table
         else:
             logger.debug(f"sacct failed with retcode={retcode}")
             logger.debug("falling back to using squeue to get job status")
-            self._cmd = "squeue --noheader --format='%i %t' --job '{0}'"
+            _cmd = "squeue"
+            # Add clusters option to squeue if provided
+            if self.clusters:
+                _cmd += f" --clusters={self.clusters}"
+            self._cmd = _cmd + " --noheader --format='%i %t' --job '{0}'"
             self._translate_table = squeue_translate_table
 
     def _status(self):
@@ -347,7 +359,14 @@ class SlurmProvider(ClusterProvider, RepresentationMixin):
         '''
 
         job_id_list = ' '.join(job_ids)
-        retcode, stdout, stderr = self.execute_wait("scancel {0}".format(job_id_list))
+
+        # Make the command to cancel jobs
+        _cmd = "scancel"
+        if self.clusters:
+            _cmd += f" --clusters={self.clusters}"
+        _cmd += " {0}"
+
+        retcode, stdout, stderr = self.execute_wait(_cmd.format(job_id_list))
         rets = None
         if retcode == 0:
             for jid in job_ids:

parsl/providers/torque/torque.py

@@ -34,10 +34,7 @@ class TorqueProvider(ClusterProvider, RepresentationMixin):
     Parameters
     ----------
     channel : Channel
-        Channel for accessing this provider. Possible channels include
-        :class:`~parsl.channels.LocalChannel` (the default),
-        :class:`~parsl.channels.SSHChannel`, or
-        :class:`~parsl.channels.SSHInteractiveLoginChannel`.
+        Channel for accessing this provider.
     account : str
         Account the job will be charged against.
     queue : str

parsl/tests/configs/user_opts.py

@@ -135,13 +135,6 @@ user_opts = {
     #     # For example:
     #     'remote_writeable': 'globus://af7bda53-6d04-11e5-ba46-22000b92c6ec/home/bzc/'
     # },
-    # 'adhoc': {
-    #    # This specifies configuration parameters when testing an ad-hoc SSH based cluster
-    #    'username': 'fixme', # username on remote systems
-    #    'remote_hostnames': ['hostname1', 'hostname2'], # addresses of remote systems
-    #    'worker_init': 'init commands',  # worker_init for remote systems
-    #    'script_dir': "/path"  # script directory on remote systems
-    # }
     #
 }  # type: Dict[str, Any]
 

parsl/tests/conftest.py

@@ -143,10 +143,6 @@ def pytest_configure(config):
         'markers',
         'staging_required: Marks tests that require a staging provider, when there is no sharedFS'
     )
-    config.addinivalue_line(
-        'markers',
-        'sshd_required: Marks tests that require a SSHD'
-    )
     config.addinivalue_line(
         'markers',
         'multiple_cores_required: Marks tests that require multiple cores, such as htex affinity'

parsl/tests/test_channels/test_local_channel.py

@@ -17,22 +17,3 @@ def test_env():
 
     x = [s for s in stdout if s.startswith("HOME=")]
     assert x, "HOME not found"
-
-
-@pytest.mark.local
-def test_env_mod():
-    ''' Testing for env update at execute time.
-    '''
-
-    lc = LocalChannel()
-    rc, stdout, stderr = lc.execute_wait("env", 1, {'TEST_ENV': 'fooo'})
-
-    stdout = stdout.split('\n')
-    x = [s for s in stdout if s.startswith("PATH=")]
-    assert x, "PATH not found"
-
-    x = [s for s in stdout if s.startswith("HOME=")]
-    assert x, "HOME not found"
-
-    x = [s for s in stdout if s.startswith("TEST_ENV=fooo")]
-    assert x, "User set env missing"

parsl/tests/test_providers/test_local_provider.py

@@ -12,7 +12,6 @@ import time
 import pytest
 
 from parsl.channels import LocalChannel
-from parsl.channels.ssh.ssh import DeprecatedSSHChannel
 from parsl.jobs.states import JobState
 from parsl.launchers import SingleNodeLauncher
 from parsl.providers import LocalProvider
@@ -69,140 +68,6 @@ def test_local_channel():
         _run_tests(p)
 
 
-SSHD_CONFIG = """
-Port {port}
-ListenAddress 127.0.0.1
-HostKey {hostkey}
-AuthorizedKeysFile {connpubkey}
-AuthenticationMethods publickey
-StrictModes no
-Subsystem sftp {sftp_path}
-"""
-
-
-# It would probably be better, when more formalized site testing comes into existence, to
-# use a site-testing provided server/configuration instead of the current scheme
-@pytest.mark.local
-@pytest.mark.sshd_required
-def test_ssh_channel():
-    with tempfile.TemporaryDirectory() as config_dir:
-        sshd_thread, priv_key, server_port = _start_sshd(config_dir)
-        try:
-            with tempfile.TemporaryDirectory() as remote_script_dir:
-                # The SSH library fails to add the new host key to the file if the file does not
-                # already exist, so create it here.
-                pathlib.Path('{}/known.hosts'.format(config_dir)).touch(mode=0o600)
-                script_dir = tempfile.mkdtemp()
-                channel = DeprecatedSSHChannel('127.0.0.1', port=server_port,
-                                               script_dir=remote_script_dir,
-                                               host_keys_filename='{}/known.hosts'.format(config_dir),
-                                               key_filename=priv_key)
-                try:
-                    p = LocalProvider(channel=channel,
-                                      launcher=SingleNodeLauncher(debug=False))
-                    p.script_dir = script_dir
-                    _run_tests(p)
-                finally:
-                    channel.close()
-        finally:
-            _stop_sshd(sshd_thread)
-
-
-def _stop_sshd(sshd_thread):
-    sshd_thread.stop()
-    sshd_thread.join()
-
-
-class SSHDThread(threading.Thread):
-    def __init__(self, config_file):
-        threading.Thread.__init__(self, daemon=True)
-        self.config_file = config_file
-        self.stop_flag = False
-        self.error = None
-
-    def run(self):
-        try:
-            # sshd needs to be run with an absolute path, hence the call to which()
-            sshpath = shutil.which('sshd')
-            assert sshpath is not None, "can find sshd executable"
-            p = subprocess.Popen([sshpath, '-D', '-f', self.config_file],
-                                 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-            while True:
-                ec = p.poll()
-                if self.stop_flag:
-                    p.terminate()
-                    break
-                elif ec is None:
-                    time.sleep(0.1)
-                elif ec == 0:
-                    self.error = Exception('sshd exited prematurely: {}{}'.format(p.stdout.read(),
-                                                                                  p.stderr.read()))
-                    break
-                else:
-                    self.error = Exception('sshd failed: {}{}'.format(p.stdout.read(),
-                                                                      p.stderr.read()))
-                    break
-        except Exception as ex:
-            logger.exception("SSHDThread exception from run loop")
-            self.error = ex
-
-    def stop(self):
-        self.stop_flag = True
-
-
-def _start_sshd(config_dir: str):
-    server_config, priv_key, port = _init_sshd(config_dir)
-    sshd_thread = SSHDThread(server_config)
-    sshd_thread.start()
-    time.sleep(1.0)
-    if not sshd_thread.is_alive():
-        raise Exception('Failed to start sshd: {}'.format(sshd_thread.error))
-    return sshd_thread, priv_key, port
-
-
-def _init_sshd(config_dir):
-    hostkey = '{}/hostkey'.format(config_dir)
-    connkey = '{}/connkey'.format(config_dir)
-    os.system('ssh-keygen -b 2048 -t rsa -q -N "" -f {}'.format(hostkey))
-    os.system('ssh-keygen -b 2048 -t rsa -q -N "" -f {}'.format(connkey))
-    port = _find_free_port(22222)
-    server_config_str = SSHD_CONFIG.format(port=port, hostkey=hostkey,
-                                           connpubkey='{}.pub'.format(connkey),
-                                           sftp_path=_get_system_sftp_path())
-    server_config = '{}/sshd_config'.format(config_dir)
-    with open(server_config, 'w') as f:
-        f.write(server_config_str)
-    return server_config, connkey, port
-
-
-def _get_system_sftp_path():
-    try:
-        with open('/etc/ssh/sshd_config') as f:
-            line = f.readline()
-            while line:
-                tokens = line.split()
-                if tokens[0] == 'Subsystem' and tokens[1] == 'sftp':
-                    return tokens[2]
-                line = f.readline()
-    except Exception:
-        pass
-    return '/usr/lib/openssh/sftp-server'
-
-
-def _find_free_port(start: int):
-    port = start
-    while port < 65535:
-        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        try:
-            s.bind(('127.0.0.1', port))
-            s.close()
-            return port
-        except Exception:
-            pass
-        port += random.randint(1, 20)
-    raise Exception('Could not find free port')
-
-
 def _run(p: LocalProvider, command: str, np: int = 1):
     id = p.submit(command, np)
     return _wait(p, id)

parsl/tests/test_providers/test_pbspro_template.py

@@ -12,9 +12,10 @@ def test_submit_script_basic(tmp_path):
     """Test slurm resources table"""
 
     provider = PBSProProvider(
-        queue="debug", channel=LocalChannel(script_dir=tmp_path)
+        queue="debug", channel=LocalChannel()
     )
     provider.script_dir = tmp_path
+    provider.channel.script_dir = tmp_path
     job_id = str(random.randint(55000, 59000))
     provider.execute_wait = mock.Mock(spec=PBSProProvider.execute_wait)
     provider.execute_wait.return_value = (0, job_id, "")

parsl/tests/test_providers/test_slurm_template.py

@@ -13,9 +13,10 @@ def test_submit_script_basic(tmp_path):
     """Test slurm resources table"""
 
     provider = SlurmProvider(
-        partition="debug", channel=LocalChannel(script_dir=tmp_path)
+        partition="debug", channel=LocalChannel()
     )
     provider.script_dir = tmp_path
+    provider.channel.script_dir = tmp_path
     job_id = str(random.randint(55000, 59000))
     provider.execute_wait = mock.MagicMock(spec=SlurmProvider.execute_wait)
     provider.execute_wait.return_value = (0, f"Submitted batch job {job_id}", "")

parsl/version.py

@@ -3,4 +3,4 @@
 Year.Month.Day[alpha/beta/..]
 Alphas will be numbered like this -> 2024.12.10a0
 """
-VERSION = '2024.11.04'
+VERSION = '2024.11.18'

{parsl-2024.11.4.dist-info → parsl-2024.11.18.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2024.11.4
+Version: 2024.11.18
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2024.11.04.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2024.11.18.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0
@@ -38,7 +38,6 @@ Requires-Dist: plotly; extra == "all"
 Requires-Dist: python-daemon; extra == "all"
 Requires-Dist: boto3; extra == "all"
 Requires-Dist: kubernetes; extra == "all"
-Requires-Dist: oauth-ssh>=0.9; extra == "all"
 Requires-Dist: ipython<=8.6.0; extra == "all"
 Requires-Dist: nbsphinx; extra == "all"
 Requires-Dist: sphinx<7.2,>=7.1; extra == "all"
@@ -55,7 +54,6 @@ Requires-Dist: jsonschema; extra == "all"
 Requires-Dist: proxystore; extra == "all"
 Requires-Dist: radical.pilot==1.60; extra == "all"
 Requires-Dist: radical.utils==1.60; extra == "all"
-Requires-Dist: paramiko; extra == "all"
 Provides-Extra: aws
 Requires-Dist: boto3; extra == "aws"
 Provides-Extra: azure
@@ -79,15 +77,11 @@ Provides-Extra: kubernetes
 Requires-Dist: kubernetes; extra == "kubernetes"
 Provides-Extra: monitoring
 Requires-Dist: sqlalchemy<2,>=1.4; extra == "monitoring"
-Provides-Extra: oauth_ssh
-Requires-Dist: oauth-ssh>=0.9; extra == "oauth-ssh"
 Provides-Extra: proxystore
 Requires-Dist: proxystore; extra == "proxystore"
 Provides-Extra: radical-pilot
 Requires-Dist: radical.pilot==1.60; extra == "radical-pilot"
 Requires-Dist: radical.utils==1.60; extra == "radical-pilot"
-Provides-Extra: ssh
-Requires-Dist: paramiko; extra == "ssh"
 Provides-Extra: visualization
 Requires-Dist: pydot; extra == "visualization"
 Requires-Dist: networkx<2.6,>=2.5; extra == "visualization"