parsl 2024.11.4__py3-none-any.whl → 2024.11.11__py3-none-any.whl

parsl/channels/ssh/ssh.py

@@ -1,295 +0,0 @@
-import errno
-import logging
-import os
-
-from parsl.channels.base import Channel
-from parsl.channels.errors import (
-    AuthException,
-    BadHostKeyException,
-    BadPermsScriptPath,
-    BadScriptPath,
-    FileCopyException,
-    SSHException,
-)
-from parsl.errors import OptionalModuleMissing
-from parsl.utils import RepresentationMixin
-
-try:
-    import paramiko
-    _ssh_enabled = True
-except (ImportError, NameError, FileNotFoundError):
-    _ssh_enabled = False
-
-
-logger = logging.getLogger(__name__)
-
-
-if _ssh_enabled:
-    class NoAuthSSHClient(paramiko.SSHClient):
-        def _auth(self, username, *args):
-            self._transport.auth_none(username)
-            return
-
-
-class DeprecatedSSHChannel(Channel, RepresentationMixin):
-    ''' SSH persistent channel. This enables remote execution on sites
-    accessible via ssh. It is assumed that the user has setup host keys
-    so as to ssh to the remote host. Which goes to say that the following
-    test on the commandline should work:
-
-    >>> ssh <username>@<hostname>
-
-    '''
-
-    def __init__(self, hostname, username=None, password=None, script_dir=None, envs=None,
-                 gssapi_auth=False, skip_auth=False, port=22, key_filename=None, host_keys_filename=None):
-        ''' Initialize a persistent connection to the remote system.
-        We should know at this point whether ssh connectivity is possible
-
-        Args:
-            - hostname (String) : Hostname
-
-        KWargs:
-            - username (string) : Username on remote system
-            - password (string) : Password for remote system
-            - port : The port designated for the ssh connection. Default is 22.
-            - script_dir (string) : Full path to a script dir where
-              generated scripts could be sent to.
-            - envs (dict) : A dictionary of environment variables to be set when executing commands
-            - key_filename (string or list): the filename, or list of filenames, of optional private key(s)
-
-        Raises:
-        '''
-        if not _ssh_enabled:
-            raise OptionalModuleMissing(['ssh'],
-                                        "SSHChannel requires the ssh module and config.")
-
-        self.hostname = hostname
-        self.username = username
-        self.password = password
-        self.port = port
-        self.script_dir = script_dir
-        self.skip_auth = skip_auth
-        self.gssapi_auth = gssapi_auth
-        self.key_filename = key_filename
-        self.host_keys_filename = host_keys_filename
-
-        if self.skip_auth:
-            self.ssh_client = NoAuthSSHClient()
-        else:
-            self.ssh_client = paramiko.SSHClient()
-        self.ssh_client.load_system_host_keys(filename=host_keys_filename)
-        self.ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-        self.sftp_client = None
-
-        self.envs = {}
-        if envs is not None:
-            self.envs = envs
-
-    def _is_connected(self):
-        transport = self.ssh_client.get_transport() if self.ssh_client else None
-        return transport and transport.is_active()
-
-    def _connect(self):
-        if not self._is_connected():
-            logger.debug(f"connecting to {self.hostname}:{self.port}")
-            try:
-                self.ssh_client.connect(
-                    self.hostname,
-                    username=self.username,
-                    password=self.password,
-                    port=self.port,
-                    allow_agent=True,
-                    gss_auth=self.gssapi_auth,
-                    gss_kex=self.gssapi_auth,
-                    key_filename=self.key_filename
-                )
-                transport = self.ssh_client.get_transport()
-                self.sftp_client = paramiko.SFTPClient.from_transport(transport)
-
-            except paramiko.BadHostKeyException as e:
-                raise BadHostKeyException(e, self.hostname)
-
-            except paramiko.AuthenticationException as e:
-                raise AuthException(e, self.hostname)
-
-            except paramiko.SSHException as e:
-                raise SSHException(e, self.hostname)
-
-            except Exception as e:
-                raise SSHException(e, self.hostname)
-
-    def _valid_sftp_client(self):
-        self._connect()
-        return self.sftp_client
-
-    def _valid_ssh_client(self):
-        self._connect()
-        return self.ssh_client
-
-    def prepend_envs(self, cmd, env={}):
-        env.update(self.envs)
-
-        if len(env.keys()) > 0:
-            env_vars = ' '.join(['{}={}'.format(key, value) for key, value in env.items()])
-            return 'env {0} {1}'.format(env_vars, cmd)
-        return cmd
-
-    def execute_wait(self, cmd, walltime=2, envs={}):
-        ''' Synchronously execute a commandline string on the shell.
-
-        Args:
-            - cmd (string) : Commandline string to execute
-            - walltime (int) : walltime in seconds
-
-        Kwargs:
-            - envs (dict) : Dictionary of env variables
-
-        Returns:
-            - retcode : Return code from the execution, -1 on fail
-            - stdout  : stdout string
-            - stderr  : stderr string
-
-        Raises:
-        None.
-        '''
-
-        # Execute the command
-        stdin, stdout, stderr = self._valid_ssh_client().exec_command(
-            self.prepend_envs(cmd, envs), bufsize=-1, timeout=walltime
-        )
-        # Block on exit status from the command
-        exit_status = stdout.channel.recv_exit_status()
-        return exit_status, stdout.read().decode("utf-8"), stderr.read().decode("utf-8")
-
-    def push_file(self, local_source, remote_dir):
-        ''' Transport a local file to a directory on a remote machine
-
-        Args:
-            - local_source (string): Path
-            - remote_dir (string): Remote path
-
-        Returns:
-            - str: Path to copied file on remote machine
-
-        Raises:
-            - BadScriptPath : if script path on the remote side is bad
-            - BadPermsScriptPath : You do not have perms to make the channel script dir
-            - FileCopyException : FileCopy failed.
-
-        '''
-        remote_dest = os.path.join(remote_dir, os.path.basename(local_source))
-
-        try:
-            self.makedirs(remote_dir, exist_ok=True)
-        except IOError as e:
-            logger.exception("Pushing {0} to {1} failed".format(local_source, remote_dir))
-            if e.errno == 2:
-                raise BadScriptPath(e, self.hostname)
-            elif e.errno == 13:
-                raise BadPermsScriptPath(e, self.hostname)
-            else:
-                logger.exception("File push failed due to SFTP client failure")
-                raise FileCopyException(e, self.hostname)
-        try:
-            self._valid_sftp_client().put(local_source, remote_dest, confirm=True)
-            # Set perm because some systems require the script to be executable
-            self._valid_sftp_client().chmod(remote_dest, 0o700)
-        except Exception as e:
-            logger.exception("File push from local source {} to remote destination {} failed".format(
-                local_source, remote_dest))
-            raise FileCopyException(e, self.hostname)
-
-        return remote_dest
-
-    def pull_file(self, remote_source, local_dir):
-        ''' Transport file on the remote side to a local directory
-
-        Args:
-            - remote_source (string): remote_source
-            - local_dir (string): Local directory to copy to
-
-
-        Returns:
-            - str: Local path to file
-
-        Raises:
-            - FileCopyException : FileCopy failed.
-        '''
-
-        local_dest = os.path.join(local_dir, os.path.basename(remote_source))
-
-        try:
-            os.makedirs(local_dir)
-        except OSError as e:
-            if e.errno != errno.EEXIST:
-                logger.exception("Failed to create local_dir: {0}".format(local_dir))
-                raise BadScriptPath(e, self.hostname)
-
-        try:
-            self._valid_sftp_client().get(remote_source, local_dest)
-        except Exception as e:
-            logger.exception("File pull failed")
-            raise FileCopyException(e, self.hostname)
-
-        return local_dest
-
-    def close(self) -> None:
-        if self._is_connected():
-            transport = self.ssh_client.get_transport()
-            self.ssh_client.close()
-
-            # ssh_client.close calls transport.close, but transport.close does
-            # not always wait for the transport thread to be stopped. See impl
-            # of Transport.close in paramiko and issue
-            # https://github.com/paramiko/paramiko/issues/520
-            logger.debug("Waiting for transport thread to stop")
-            transport.join(30)
-            if transport.is_alive():
-                logger.warning("SSH transport thread did not shut down")
-            else:
-                logger.debug("SSH transport thread stopped")
-
-    def isdir(self, path):
-        """Return true if the path refers to an existing directory.
-
-        Parameters
-        ----------
-        path : str
-            Path of directory on the remote side to check.
-        """
-        result = True
-        try:
-            self._valid_sftp_client().lstat(path)
-        except FileNotFoundError:
-            result = False
-
-        return result
-
-    def makedirs(self, path, mode=0o700, exist_ok=False):
-        """Create a directory on the remote side.
-
-        If intermediate directories do not exist, they will be created.
-
-        Parameters
-        ----------
-        path : str
-            Path of directory on the remote side to create.
-        mode : int
-            Permissions (posix-style) for the newly-created directory.
-        exist_ok : bool
-            If False, raise an OSError if the target directory already exists.
-        """
-        if exist_ok is False and self.isdir(path):
-            raise OSError('Target directory {} already exists'.format(path))
-
-        self.execute_wait('mkdir -p {}'.format(path))
-        self._valid_sftp_client().chmod(path, mode)
-
-    @property
-    def script_dir(self):
-        return self._script_dir
-
-    @script_dir.setter
-    def script_dir(self, value):
-        self._script_dir = value

parsl/channels/ssh_il/ssh_il.py

@@ -1,85 +0,0 @@
-import getpass
-import logging
-
-from parsl.channels.ssh.ssh import DeprecatedSSHChannel
-from parsl.errors import OptionalModuleMissing
-
-try:
-    import paramiko
-    _ssh_enabled = True
-except (ImportError, NameError, FileNotFoundError):
-    _ssh_enabled = False
-
-
-logger = logging.getLogger(__name__)
-
-
-class DeprecatedSSHInteractiveLoginChannel(DeprecatedSSHChannel):
-    """SSH persistent channel. This enables remote execution on sites
-    accessible via ssh. This channel supports interactive login and is appropriate when
-    keys are not set up.
-    """
-
-    def __init__(self, hostname, username=None, password=None, script_dir=None, envs=None):
-        ''' Initialize a persistent connection to the remote system.
-        We should know at this point whether ssh connectivity is possible
-
-        Args:
-            - hostname (String) : Hostname
-
-        KWargs:
-            - username (string) : Username on remote system
-            - password (string) : Password for remote system
-            - script_dir (string) : Full path to a script dir where
-              generated scripts could be sent to.
-            - envs (dict) : A dictionary of env variables to be set when executing commands
-
-        Raises:
-        '''
-        if not _ssh_enabled:
-            raise OptionalModuleMissing(['ssh'],
-                                        "SSHInteractiveLoginChannel requires the ssh module and config.")
-
-        self.hostname = hostname
-        self.username = username
-        self.password = password
-
-        self.ssh_client = paramiko.SSHClient()
-        self.ssh_client.load_system_host_keys()
-        self.ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-
-        self.script_dir = script_dir
-
-        self.envs = {}
-        if envs is not None:
-            self.envs = envs
-
-        try:
-            self.ssh_client.connect(
-                hostname, username=username, password=password, allow_agent=True
-            )
-
-        except Exception:
-            logger.debug("Caught the SSHException in SSHInteractive")
-            pass
-        '''
-        except paramiko.BadHostKeyException as e:
-            raise BadHostKeyException(e, self.hostname)
-
-        except paramiko.AuthenticationException as e:
-            raise AuthException(e, self.hostname)
-
-        except paramiko.SSHException as e:
-            logger.debug("Caught the SSHException in SSHInteractive")
-            pass
-
-        except Exception as e:
-            raise SSHException(e, self.hostname)
-        '''
-
-        transport = self.ssh_client.get_transport()
-
-        il_password = getpass.getpass('Enter {0} Logon password :'.format(hostname))
-        transport.auth_password(username, il_password)
-
-        self.sftp_client = paramiko.SFTPClient.from_transport(transport)

parsl/providers/ad_hoc/ad_hoc.py

@@ -1,252 +0,0 @@
-import logging
-import os
-import time
-
-from parsl.channels import LocalChannel
-from parsl.jobs.states import JobState, JobStatus
-from parsl.launchers import SimpleLauncher
-from parsl.providers.base import ExecutionProvider
-from parsl.providers.errors import ScriptPathError
-from parsl.utils import RepresentationMixin
-
-logger = logging.getLogger(__name__)
-
-
-class DeprecatedAdHocProvider(ExecutionProvider, RepresentationMixin):
-    """ Deprecated ad-hoc execution provider
-
-    The (former) AdHocProvider is deprecated. See
-    `issue #3515 <https://github.com/Parsl/parsl/issues/3515>`_
-    for further discussion.
-
-    This provider is used to provision execution resources over one or more ad hoc nodes
-    that are each accessible over a Channel (say, ssh) but otherwise lack a cluster scheduler.
-
-    Parameters
-    ----------
-
-    channels : list of Channel ojects
-      Each channel represents a connection to a remote node
-
-    worker_init : str
-      Command to be run before starting a worker, such as 'module load Anaconda; source activate env'.
-      Since this provider calls the same worker_init across all nodes in the ad-hoc cluster, it is
-      recommended that a single script is made available across nodes such as ~/setup.sh that can
-      be invoked.
-
-    cmd_timeout : int
-      Duration for which the provider will wait for a command to be invoked on a remote system.
-      Defaults to 30s
-
-    parallelism : float
-      Determines the ratio of workers to tasks as managed by the strategy component
-
-    """
-
-    def __init__(self,
-                 channels=[],
-                 worker_init='',
-                 cmd_timeout=30,
-                 parallelism=1,
-                 move_files=None):
-
-        self.channels = channels
-        self._label = 'ad-hoc'
-        self.worker_init = worker_init
-        self.cmd_timeout = cmd_timeout
-        self.parallelism = 1
-        self.move_files = move_files
-        self.launcher = SimpleLauncher()
-        self.init_blocks = self.min_blocks = self.max_blocks = len(channels)
-
-        # This will be overridden by the DFK to the rundirs.
-        self.script_dir = "."
-
-        # In ad-hoc mode, nodes_per_block should be 1
-        self.nodes_per_block = 1
-
-        # Dictionary that keeps track of jobs, keyed on job_id
-        self.resources = {}
-
-        self.least_loaded = self._least_loaded()
-        logger.debug("AdHoc provider initialized")
-
-    def _write_submit_script(self, script_string, script_filename):
-        '''
-        Load the template string with config values and write the generated submit script to
-        a submit script file.
-
-        Parameters
-        ----------
-        script_string: (string)
-          The template string to be used for the writing submit script
-
-        script_filename: (string)
-          Name of the submit script
-
-        Returns
-        -------
-        None: on success
-
-        Raises
-        ------
-        ScriptPathError
-          Unable to write submit script out
-        '''
-
-        try:
-            with open(script_filename, 'w') as f:
-                f.write(script_string)
-
-        except IOError as e:
-            logger.error("Failed writing to submit script: %s", script_filename)
-            raise ScriptPathError(script_filename, e)
-
-        return None
-
-    def _least_loaded(self):
-        """ Find channels that are not in use
-
-        Returns
-        -------
-        channel : Channel object
-        None : When there are no more available channels
-        """
-        while True:
-            channel_counts = {channel: 0 for channel in self.channels}
-            for job_id in self.resources:
-                channel = self.resources[job_id]['channel']
-                if self.resources[job_id]['status'].state == JobState.RUNNING:
-                    channel_counts[channel] = channel_counts.get(channel, 0) + 1
-                else:
-                    channel_counts[channel] = channel_counts.get(channel, 0)
-
-            logger.debug("Channel_counts : {}".format(channel_counts))
-            if 0 not in channel_counts.values():
-                yield None
-
-            for channel in channel_counts:
-                if channel_counts[channel] == 0:
-                    yield channel
-
-    def submit(self, command, tasks_per_node, job_name="parsl.adhoc"):
-        ''' Submits the command onto a channel from the list of channels
-
-        Submit returns an ID that corresponds to the task that was just submitted.
-
-        Parameters
-        ----------
-        command: (String)
-          Commandline invocation to be made on the remote side.
-
-        tasks_per_node: (int)
-          command invocations to be launched per node
-
-        job_name: (String)
-          Name of the job. Default : parsl.adhoc
-
-
-        Returns
-        -------
-        None
-          At capacity, cannot provision more
-
-        job_id: (string)
-          Identifier for the job
-
-        '''
-        channel = next(self.least_loaded)
-        if channel is None:
-            logger.warning("All Channels in Ad-Hoc provider are in use")
-            return None
-
-        job_name = "{0}.{1}".format(job_name, time.time())
-
-        # Set script path
-        script_path = "{0}/{1}.sh".format(self.script_dir, job_name)
-        script_path = os.path.abspath(script_path)
-
-        wrap_command = self.worker_init + '\n' + self.launcher(command, tasks_per_node, self.nodes_per_block)
-
-        self._write_submit_script(wrap_command, script_path)
-
-        job_id = None
-        remote_pid = None
-        final_cmd = None
-
-        if (self.move_files is None and not isinstance(channel, LocalChannel)) or (self.move_files):
-            logger.debug("Pushing start script")
-            script_path = channel.push_file(script_path, channel.script_dir)
-
-        # Bash would return until the streams are closed. So we redirect to a outs file
-        final_cmd = 'bash {0} > {0}.out 2>&1 & \n echo "PID:$!" '.format(script_path)
-        retcode, stdout, stderr = channel.execute_wait(final_cmd, self.cmd_timeout)
-        for line in stdout.split('\n'):
-            if line.startswith("PID:"):
-                remote_pid = line.split("PID:")[1].strip()
-                job_id = remote_pid
-        if job_id is None:
-            logger.warning("Channel failed to start remote command/retrieve PID")
-
-        self.resources[job_id] = {'job_id': job_id,
-                                  'status': JobStatus(JobState.RUNNING),
-                                  'cmd': final_cmd,
-                                  'channel': channel,
-                                  'remote_pid': remote_pid}
-
-        return job_id
-
-    def status(self, job_ids):
-        """ Get status of the list of jobs with job_ids
-
-        Parameters
-        ----------
-        job_ids : list of strings
-          List of job id strings
-
-        Returns
-        -------
-        list of JobStatus objects
-        """
-        for job_id in job_ids:
-            channel = self.resources[job_id]['channel']
-            status_command = "ps --pid {} | grep {}".format(self.resources[job_id]['job_id'],
-                                                            self.resources[job_id]['cmd'].split()[0])
-            retcode, stdout, stderr = channel.execute_wait(status_command)
-            if retcode != 0 and self.resources[job_id]['status'].state == JobState.RUNNING:
-                self.resources[job_id]['status'] = JobStatus(JobState.FAILED)
-
-        return [self.resources[job_id]['status'] for job_id in job_ids]
-
-    def cancel(self, job_ids):
-        """ Cancel a list of jobs with job_ids
-
-        Parameters
-        ----------
-        job_ids : list of strings
-          List of job id strings
-
-        Returns
-        -------
-        list of confirmation bools: [True, False...]
-        """
-        logger.debug("Cancelling jobs: {}".format(job_ids))
-        rets = []
-        for job_id in job_ids:
-            channel = self.resources[job_id]['channel']
-            cmd = "kill -TERM -$(ps -o pgid= {} | grep -o '[0-9]*')".format(self.resources[job_id]['job_id'])
-            retcode, stdout, stderr = channel.execute_wait(cmd)
-            if retcode == 0:
-                rets.append(True)
-            else:
-                rets.append(False)
-            self.resources[job_id]['status'] = JobStatus(JobState.COMPLETED)
-        return rets
-
-    @property
-    def label(self):
-        return self._label
-
-    @property
-    def status_polling_interval(self):
-        return 10

parsl/tests/configs/local_adhoc.py

@@ -1,18 +0,0 @@
-from parsl.channels import LocalChannel
-from parsl.config import Config
-from parsl.executors import HighThroughputExecutor
-from parsl.providers.ad_hoc.ad_hoc import DeprecatedAdHocProvider
-
-
-def fresh_config():
-    return Config(
-        executors=[
-            HighThroughputExecutor(
-                label='AdHoc',
-                encrypted=True,
-                provider=DeprecatedAdHocProvider(
-                    channels=[LocalChannel(), LocalChannel()]
-                )
-            )
-        ]
-    )