parsl 2024.3.18__py3-none-any.whl → 2025.1.13__py3-none-any.whl

parsl/channels/ssh_il/ssh_il.py

@@ -1,74 +0,0 @@
-import getpass
-import logging
-
-import paramiko
-from parsl.channels.ssh.ssh import SSHChannel
-
-logger = logging.getLogger(__name__)
-
-
-class SSHInteractiveLoginChannel(SSHChannel):
-    """SSH persistent channel. This enables remote execution on sites
-    accessible via ssh. This channel supports interactive login and is appropriate when
-    keys are not set up.
-    """
-
-    def __init__(self, hostname, username=None, password=None, script_dir=None, envs=None):
-        ''' Initialize a persistent connection to the remote system.
-        We should know at this point whether ssh connectivity is possible
-
-        Args:
-            - hostname (String) : Hostname
-
-        KWargs:
-            - username (string) : Username on remote system
-            - password (string) : Password for remote system
-            - script_dir (string) : Full path to a script dir where
-              generated scripts could be sent to.
-            - envs (dict) : A dictionary of env variables to be set when executing commands
-
-        Raises:
-        '''
-        self.hostname = hostname
-        self.username = username
-        self.password = password
-
-        self.ssh_client = paramiko.SSHClient()
-        self.ssh_client.load_system_host_keys()
-        self.ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-
-        self.script_dir = script_dir
-
-        self.envs = {}
-        if envs is not None:
-            self.envs = envs
-
-        try:
-            self.ssh_client.connect(
-                hostname, username=username, password=password, allow_agent=True
-            )
-
-        except Exception:
-            logger.debug("Caught the SSHException in SSHInteractive")
-            pass
-        '''
-        except paramiko.BadHostKeyException as e:
-            raise BadHostKeyException(e, self.hostname)
-
-        except paramiko.AuthenticationException as e:
-            raise AuthException(e, self.hostname)
-
-        except paramiko.SSHException as e:
-            logger.debug("Caught the SSHException in SSHInteractive")
-            pass
-
-        except Exception as e:
-            raise SSHException(e, self.hostname)
-        '''
-
-        transport = self.ssh_client.get_transport()
-
-        il_password = getpass.getpass('Enter {0} Logon password :'.format(hostname))
-        transport.auth_password(username, il_password)
-
-        self.sftp_client = paramiko.SFTPClient.from_transport(transport)

parsl/configs/ad_hoc.py

@@ -1,35 +0,0 @@
-from parsl.providers import AdHocProvider
-from parsl.channels import SSHChannel
-from parsl.executors import HighThroughputExecutor
-from parsl.config import Config
-from typing import Any, Dict
-
-user_opts: Dict[str, Dict[str, Any]]
-user_opts = {'adhoc':
-             {'username': 'YOUR_USERNAME',
-              'script_dir': 'YOUR_SCRIPT_DIR',
-              'remote_hostnames': ['REMOTE_HOST_URL_1', 'REMOTE_HOST_URL_2']
-              }
-             }
-
-
-config = Config(
-    executors=[
-        HighThroughputExecutor(
-            label='remote_htex',
-            max_workers_per_node=2,
-            worker_logdir_root=user_opts['adhoc']['script_dir'],
-            provider=AdHocProvider(
-                # Command to be run before starting a worker, such as:
-                # 'module load Anaconda; source activate parsl_env'.
-                worker_init='',
-                channels=[SSHChannel(hostname=m,
-                                     username=user_opts['adhoc']['username'],
-                                     script_dir=user_opts['adhoc']['script_dir'],
-                                     ) for m in user_opts['adhoc']['remote_hostnames']]
-            )
-        )
-    ],
-    #  AdHoc Clusters should not be setup with scaling strategy.
-    strategy='none',
-)

parsl/executors/radical/rpex_master.py

@@ -1,42 +0,0 @@
-#!/usr/bin/env python3
-
-import sys
-
-import radical.utils as ru
-import radical.pilot as rp
-
-
-# ------------------------------------------------------------------------------
-#
-if __name__ == '__main__':
-
-    # The purpose of this master is to (a) spawn a set or workers
-    # within the same allocation, (b) to distribute work items to
-    # those workers, and (c) to collect the responses again.
-    cfg_fname = str(sys.argv[1])
-    cfg = ru.Config(cfg=ru.read_json(cfg_fname))
-    cfg.rank = int(sys.argv[2])
-
-    worker_descr = cfg.worker_descr
-    n_workers = cfg.n_workers
-    gpus_per_node = cfg.gpus_per_node
-    cores_per_node = cfg.cores_per_node
-    nodes_per_worker = cfg.nodes_per_worker
-
-    # create a master class instance - this will establish communication
-    # to the pilot agent
-    master = rp.raptor.Master(cfg)
-
-    # insert `n` worker into the agent.  The agent will schedule (place)
-    # those workers and execute them.
-    worker_descr['ranks'] = nodes_per_worker * cores_per_node
-    worker_descr['gpus_per_rank'] = nodes_per_worker * gpus_per_node
-    worker_ids = master.submit_workers(
-                 [rp.TaskDescription(worker_descr) for _ in range(n_workers)])
-
-    # wait for all workers
-    master.wait_workers()
-    master.start()
-    master.join()
-
-# ------------------------------------------------------------------------------

parsl/monitoring/radios.py

@@ -1,175 +0,0 @@
-import os
-import socket
-import pickle
-import uuid
-import logging
-
-from abc import ABCMeta, abstractmethod
-
-from typing import Optional
-
-from parsl.serialize import serialize
-
-_db_manager_excepts: Optional[Exception]
-
-
-logger = logging.getLogger(__name__)
-
-
-class MonitoringRadio(metaclass=ABCMeta):
-    @abstractmethod
-    def send(self, message: object) -> None:
-        pass
-
-
-class FilesystemRadio(MonitoringRadio):
-    """A MonitoringRadio that sends messages over a shared filesystem.
-
-    The messsage directory structure is based on maildir,
-    https://en.wikipedia.org/wiki/Maildir
-
-    The writer creates a message in tmp/ and then when it is fully
-    written, moves it atomically into new/
-
-    The reader ignores tmp/ and only reads and deletes messages from
-    new/
-
-    This avoids a race condition of reading partially written messages.
-
-    This radio is likely to give higher shared filesystem load compared to
-    the UDPRadio, but should be much more reliable.
-    """
-
-    def __init__(self, *, monitoring_url: str, source_id: int, timeout: int = 10, run_dir: str):
-        logger.info("filesystem based monitoring channel initializing")
-        self.source_id = source_id
-        self.base_path = f"{run_dir}/monitor-fs-radio/"
-        self.tmp_path = f"{self.base_path}/tmp"
-        self.new_path = f"{self.base_path}/new"
-
-        os.makedirs(self.tmp_path, exist_ok=True)
-        os.makedirs(self.new_path, exist_ok=True)
-
-    def send(self, message: object) -> None:
-        logger.info("Sending a monitoring message via filesystem")
-
-        unique_id = str(uuid.uuid4())
-
-        tmp_filename = f"{self.tmp_path}/{unique_id}"
-        new_filename = f"{self.new_path}/{unique_id}"
-        buffer = (message, "NA")
-
-        # this will write the message out then atomically
-        # move it into new/, so that a partially written
-        # file will never be observed in new/
-        with open(tmp_filename, "wb") as f:
-            f.write(serialize(buffer))
-        os.rename(tmp_filename, new_filename)
-
-
-class HTEXRadio(MonitoringRadio):
-
-    def __init__(self, monitoring_url: str, source_id: int, timeout: int = 10):
-        """
-        Parameters
-        ----------
-
-        monitoring_url : str
-            URL of the form <scheme>://<IP>:<PORT>
-        source_id : str
-            String identifier of the source
-        timeout : int
-            timeout, default=10s
-        """
-        self.source_id = source_id
-        logger.info("htex-based monitoring channel initialising")
-
-    def send(self, message: object) -> None:
-        """ Sends a message to the UDP receiver
-
-        Parameter
-        ---------
-
-        message: object
-            Arbitrary pickle-able object that is to be sent
-
-        Returns:
-            None
-        """
-
-        import parsl.executors.high_throughput.monitoring_info
-
-        result_queue = parsl.executors.high_throughput.monitoring_info.result_queue
-
-        # this message needs to go in the result queue tagged so that it is treated
-        # i) as a monitoring message by the interchange, and then further more treated
-        # as a RESOURCE_INFO message when received by monitoring (rather than a NODE_INFO
-        # which is the implicit default for messages from the interchange)
-
-        # for the interchange, the outer wrapper, this needs to be a dict:
-
-        interchange_msg = {
-            'type': 'monitoring',
-            'payload': message
-        }
-
-        if result_queue:
-            result_queue.put(pickle.dumps(interchange_msg))
-        else:
-            logger.error("result_queue is uninitialized - cannot put monitoring message")
-
-        return
-
-
-class UDPRadio(MonitoringRadio):
-
-    def __init__(self, monitoring_url: str, source_id: int, timeout: int = 10):
-        """
-        Parameters
-        ----------
-
-        monitoring_url : str
-            URL of the form <scheme>://<IP>:<PORT>
-        source_id : str
-            String identifier of the source
-        timeout : int
-            timeout, default=10s
-        """
-        self.monitoring_url = monitoring_url
-        self.sock_timeout = timeout
-        self.source_id = source_id
-        try:
-            self.scheme, self.ip, port = (x.strip('/') for x in monitoring_url.split(':'))
-            self.port = int(port)
-        except Exception:
-            raise Exception("Failed to parse monitoring url: {}".format(monitoring_url))
-
-        self.sock = socket.socket(socket.AF_INET,
-                                  socket.SOCK_DGRAM,
-                                  socket.IPPROTO_UDP)  # UDP
-        self.sock.settimeout(self.sock_timeout)
-
-    def send(self, message: object) -> None:
-        """ Sends a message to the UDP receiver
-
-        Parameter
-        ---------
-
-        message: object
-            Arbitrary pickle-able object that is to be sent
-
-        Returns:
-            None
-        """
-        try:
-            buffer = pickle.dumps(message)
-        except Exception:
-            logging.exception("Exception during pickling", exc_info=True)
-            return
-
-        try:
-            self.sock.sendto(buffer, (self.ip, self.port))
-        except socket.timeout:
-            logging.error("Could not send message within timeout limit")
-            return
-        return

parsl/providers/ad_hoc/ad_hoc.py

@@ -1,248 +0,0 @@
-import logging
-import os
-import time
-
-from parsl.channels import LocalChannel
-from parsl.jobs.states import JobStatus, JobState
-from parsl.launchers import SimpleLauncher
-from parsl.providers.base import ExecutionProvider
-from parsl.providers.errors import ScriptPathError
-from parsl.utils import RepresentationMixin
-
-logger = logging.getLogger(__name__)
-
-
-class AdHocProvider(ExecutionProvider, RepresentationMixin):
-    """ Ad-hoc execution provider
-
-    This provider is used to provision execution resources over one or more ad hoc nodes
-    that are each accessible over a Channel (say, ssh) but otherwise lack a cluster scheduler.
-
-    Parameters
-    ----------
-
-    channels : list of Channel ojects
-      Each channel represents a connection to a remote node
-
-    worker_init : str
-      Command to be run before starting a worker, such as 'module load Anaconda; source activate env'.
-      Since this provider calls the same worker_init across all nodes in the ad-hoc cluster, it is
-      recommended that a single script is made available across nodes such as ~/setup.sh that can
-      be invoked.
-
-    cmd_timeout : int
-      Duration for which the provider will wait for a command to be invoked on a remote system.
-      Defaults to 30s
-
-    parallelism : float
-      Determines the ratio of workers to tasks as managed by the strategy component
-
-    """
-
-    def __init__(self,
-                 channels=[],
-                 worker_init='',
-                 cmd_timeout=30,
-                 parallelism=1,
-                 move_files=None):
-
-        self.channels = channels
-        self._label = 'ad-hoc'
-        self.worker_init = worker_init
-        self.cmd_timeout = cmd_timeout
-        self.parallelism = 1
-        self.move_files = move_files
-        self.launcher = SimpleLauncher()
-        self.init_blocks = self.min_blocks = self.max_blocks = len(channels)
-
-        # This will be overridden by the DFK to the rundirs.
-        self.script_dir = "."
-
-        # In ad-hoc mode, nodes_per_block should be 1
-        self.nodes_per_block = 1
-
-        # Dictionary that keeps track of jobs, keyed on job_id
-        self.resources = {}
-
-        self.least_loaded = self._least_loaded()
-        logger.debug("AdHoc provider initialized")
-
-    def _write_submit_script(self, script_string, script_filename):
-        '''
-        Load the template string with config values and write the generated submit script to
-        a submit script file.
-
-        Parameters
-        ----------
-        script_string: (string)
-          The template string to be used for the writing submit script
-
-        script_filename: (string)
-          Name of the submit script
-
-        Returns
-        -------
-        None: on success
-
-        Raises
-        ------
-        ScriptPathError
-          Unable to write submit script out
-        '''
-
-        try:
-            with open(script_filename, 'w') as f:
-                f.write(script_string)
-
-        except IOError as e:
-            logger.error("Failed writing to submit script: %s", script_filename)
-            raise ScriptPathError(script_filename, e)
-
-        return None
-
-    def _least_loaded(self):
-        """ Find channels that are not in use
-
-        Returns
-        -------
-        channel : Channel object
-        None : When there are no more available channels
-        """
-        while True:
-            channel_counts = {channel: 0 for channel in self.channels}
-            for job_id in self.resources:
-                channel = self.resources[job_id]['channel']
-                if self.resources[job_id]['status'].state == JobState.RUNNING:
-                    channel_counts[channel] = channel_counts.get(channel, 0) + 1
-                else:
-                    channel_counts[channel] = channel_counts.get(channel, 0)
-
-            logger.debug("Channel_counts : {}".format(channel_counts))
-            if 0 not in channel_counts.values():
-                yield None
-
-            for channel in channel_counts:
-                if channel_counts[channel] == 0:
-                    yield channel
-
-    def submit(self, command, tasks_per_node, job_name="parsl.adhoc"):
-        ''' Submits the command onto a channel from the list of channels
-
-        Submit returns an ID that corresponds to the task that was just submitted.
-
-        Parameters
-        ----------
-        command: (String)
-          Commandline invocation to be made on the remote side.
-
-        tasks_per_node: (int)
-          command invocations to be launched per node
-
-        job_name: (String)
-          Name of the job. Default : parsl.adhoc
-
-
-        Returns
-        -------
-        None
-          At capacity, cannot provision more
-
-        job_id: (string)
-          Identifier for the job
-
-        '''
-        channel = next(self.least_loaded)
-        if channel is None:
-            logger.warning("All Channels in Ad-Hoc provider are in use")
-            return None
-
-        job_name = "{0}.{1}".format(job_name, time.time())
-
-        # Set script path
-        script_path = "{0}/{1}.sh".format(self.script_dir, job_name)
-        script_path = os.path.abspath(script_path)
-
-        wrap_command = self.worker_init + '\n' + self.launcher(command, tasks_per_node, self.nodes_per_block)
-
-        self._write_submit_script(wrap_command, script_path)
-
-        job_id = None
-        remote_pid = None
-        final_cmd = None
-
-        if (self.move_files is None and not isinstance(channel, LocalChannel)) or (self.move_files):
-            logger.debug("Pushing start script")
-            script_path = channel.push_file(script_path, channel.script_dir)
-
-        # Bash would return until the streams are closed. So we redirect to a outs file
-        final_cmd = 'bash {0} > {0}.out 2>&1 & \n echo "PID:$!" '.format(script_path)
-        retcode, stdout, stderr = channel.execute_wait(final_cmd, self.cmd_timeout)
-        for line in stdout.split('\n'):
-            if line.startswith("PID:"):
-                remote_pid = line.split("PID:")[1].strip()
-                job_id = remote_pid
-        if job_id is None:
-            logger.warning("Channel failed to start remote command/retrieve PID")
-
-        self.resources[job_id] = {'job_id': job_id,
-                                  'status': JobStatus(JobState.RUNNING),
-                                  'cmd': final_cmd,
-                                  'channel': channel,
-                                  'remote_pid': remote_pid}
-
-        return job_id
-
-    def status(self, job_ids):
-        """ Get status of the list of jobs with job_ids
-
-        Parameters
-        ----------
-        job_ids : list of strings
-          List of job id strings
-
-        Returns
-        -------
-        list of JobStatus objects
-        """
-        for job_id in job_ids:
-            channel = self.resources[job_id]['channel']
-            status_command = "ps --pid {} | grep {}".format(self.resources[job_id]['job_id'],
-                                                            self.resources[job_id]['cmd'].split()[0])
-            retcode, stdout, stderr = channel.execute_wait(status_command)
-            if retcode != 0 and self.resources[job_id]['status'].state == JobState.RUNNING:
-                self.resources[job_id]['status'] = JobStatus(JobState.FAILED)
-
-        return [self.resources[job_id]['status'] for job_id in job_ids]
-
-    def cancel(self, job_ids):
-        """ Cancel a list of jobs with job_ids
-
-        Parameters
-        ----------
-        job_ids : list of strings
-          List of job id strings
-
-        Returns
-        -------
-        list of confirmation bools: [True, False...]
-        """
-        logger.debug("Cancelling jobs: {}".format(job_ids))
-        rets = []
-        for job_id in job_ids:
-            channel = self.resources[job_id]['channel']
-            cmd = "kill -TERM -$(ps -o pgid= {} | grep -o '[0-9]*')".format(self.resources[job_id]['job_id'])
-            retcode, stdout, stderr = channel.execute_wait(cmd)
-            if retcode == 0:
-                rets.append(True)
-            else:
-                rets.append(False)
-            self.resources[job_id]['status'] = JobStatus(JobState.COMPLETED)
-        return rets
-
-    @property
-    def label(self):
-        return self._label
-
-    @property
-    def status_polling_interval(self):
-        return 10