parsl 2024.3.18__py3-none-any.whl → 2025.1.13__py3-none-any.whl

parsl/channels/errors.py

@@ -1,113 +0,0 @@
-''' Exceptions raise by Apps.
-'''
-from parsl.errors import ParslError
-from typing import Optional
-
-
-class ChannelError(ParslError):
-    """ Base class for all exceptions
-
-    Only to be invoked when only a more specific error is not available.
-    """
-    def __init__(self, reason: str, e: Exception, hostname: str) -> None:
-        self.reason = reason
-        self.e = e
-        self.hostname = hostname
-
-    def __str__(self) -> str:
-        return "Hostname:{0}, Reason:{1}".format(self.hostname, self.reason)
-
-
-class BadHostKeyException(ChannelError):
-    ''' SSH channel could not be created since server's host keys could not
-    be verified
-
-    Contains:
-    reason(string)
-    e (paramiko exception object)
-    hostname (string)
-    '''
-
-    def __init__(self, e: Exception, hostname: str) -> None:
-        super().__init__("SSH channel could not be created since server's host keys could not be "
-                         "verified", e, hostname)
-
-
-class BadScriptPath(ChannelError):
-    ''' An error raised during execution of an app.
-    What this exception contains depends entirely on context
-    Contains:
-    reason(string)
-    e (paramiko exception object)
-    hostname (string)
-    '''
-
-    def __init__(self, e: Exception, hostname: str) -> None:
-        super().__init__("Inaccessible remote script dir. Specify script_dir", e, hostname)
-
-
-class BadPermsScriptPath(ChannelError):
-    ''' User does not have permissions to access the script_dir on the remote site
-
-    Contains:
-    reason(string)
-    e (paramiko exception object)
-    hostname (string)
-    '''
-
-    def __init__(self, e: Exception, hostname: str) -> None:
-        super().__init__("User does not have permissions to access the script_dir", e, hostname)
-
-
-class FileExists(ChannelError):
-    ''' Push or pull of file over channel fails since a file of the name already
-    exists on the destination.
-
-    Contains:
-    reason(string)
-    e (paramiko exception object)
-    hostname (string)
-    '''
-
-    def __init__(self, e: Exception, hostname: str, filename: Optional[str] = None) -> None:
-        super().__init__("File name collision in channel transport phase: {}".format(filename),
-                         e, hostname)
-
-
-class AuthException(ChannelError):
-    ''' An error raised during execution of an app.
-    What this exception contains depends entirely on context
-    Contains:
-    reason(string)
-    e (paramiko exception object)
-    hostname (string)
-    '''
-
-    def __init__(self, e: Exception, hostname: str) -> None:
-        super().__init__("Authentication to remote server failed", e, hostname)
-
-
-class SSHException(ChannelError):
-    ''' if there was any other error connecting or establishing an SSH session
-
-    Contains:
-    reason(string)
-    e (paramiko exception object)
-    hostname (string)
-    '''
-
-    def __init__(self, e: Exception, hostname: str) -> None:
-        super().__init__("Error connecting or establishing an SSH session", e, hostname)
-
-
-class FileCopyException(ChannelError):
-    ''' File copy operation failed
-
-    Contains:
-    reason(string)
-    e (paramiko exception object)
-    hostname (string)
-    '''
-
-    def __init__(self, e: Exception, hostname: str) -> None:
-        super().__init__("File copy failed due to {0}".format(e), e, hostname)

parsl/channels/local/local.py

@@ -1,164 +0,0 @@
-import copy
-import logging
-import os
-import shutil
-import subprocess
-
-from parsl.channels.base import Channel
-from parsl.channels.errors import FileCopyException
-from parsl.utils import RepresentationMixin
-
-logger = logging.getLogger(__name__)
-
-
-class LocalChannel(Channel, RepresentationMixin):
-    ''' This is not even really a channel, since opening a local shell is not heavy
-    and done so infrequently that they do not need a persistent channel
-    '''
-
-    def __init__(self, userhome=".", envs={}, script_dir=None):
-        ''' Initialize the local channel. script_dir is required by set to a default.
-
-        KwArgs:
-            - userhome (string): (default='.') This is provided as a way to override and set a specific userhome
-            - envs (dict) : A dictionary of env variables to be set when launching the shell
-            - script_dir (string): Directory to place scripts
-        '''
-        self.userhome = os.path.abspath(userhome)
-        self.hostname = "localhost"
-        self.envs = envs
-        local_env = os.environ.copy()
-        self._envs = copy.deepcopy(local_env)
-        self._envs.update(envs)
-        self.script_dir = script_dir
-
-    def execute_wait(self, cmd, walltime=None, envs={}):
-        ''' Synchronously execute a commandline string on the shell.
-
-        Args:
-            - cmd (string) : Commandline string to execute
-            - walltime (int) : walltime in seconds, this is not really used now.
-
-        Kwargs:
-            - envs (dict) : Dictionary of env variables. This will be used
-              to override the envs set at channel initialization.
-
-        Returns:
-            - retcode : Return code from the execution, -1 on fail
-            - stdout  : stdout string
-            - stderr  : stderr string
-
-        Raises:
-        None.
-        '''
-        current_env = copy.deepcopy(self._envs)
-        current_env.update(envs)
-
-        try:
-            proc = subprocess.Popen(
-                cmd,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.PIPE,
-                cwd=self.userhome,
-                env=current_env,
-                shell=True,
-                preexec_fn=os.setpgrp
-            )
-            (stdout, stderr) = proc.communicate(timeout=walltime)
-            retcode = proc.returncode
-
-        except Exception as e:
-            logger.warning("Execution of command '{}' failed due to \n{}".format(cmd, e))
-            raise
-
-        return (retcode, stdout.decode("utf-8"), stderr.decode("utf-8"))
-
-    def push_file(self, source, dest_dir):
-        ''' If the source files dirpath is the same as dest_dir, a copy
-        is not necessary, and nothing is done. Else a copy is made.
-
-        Args:
-            - source (string) : Path to the source file
-            - dest_dir (string) : Path to the directory to which the files is to be copied
-
-        Returns:
-            - destination_path (String) : Absolute path of the destination file
-
-        Raises:
-            - FileCopyException : If file copy failed.
-        '''
-
-        local_dest = os.path.join(dest_dir, os.path.basename(source))
-
-        # Only attempt to copy if the target dir and source dir are different
-        if os.path.dirname(source) != dest_dir:
-            try:
-                shutil.copyfile(source, local_dest)
-                os.chmod(local_dest, 0o700)
-
-            except OSError as e:
-                raise FileCopyException(e, self.hostname)
-
-        else:
-            os.chmod(local_dest, 0o700)
-
-        return local_dest
-
-    def pull_file(self, remote_source, local_dir):
-        return self.push_file(remote_source, local_dir)
-
-    def close(self):
-        ''' There's nothing to close here, and this really doesn't do anything
-
-        Returns:
-             - False, because it really did not "close" this channel.
-        '''
-        return False
-
-    def isdir(self, path):
-        """Return true if the path refers to an existing directory.
-
-        Parameters
-        ----------
-        path : str
-            Path of directory to check.
-        """
-
-        return os.path.isdir(path)
-
-    def makedirs(self, path, mode=0o700, exist_ok=False):
-        """Create a directory.
-
-        If intermediate directories do not exist, they will be created.
-
-        Parameters
-        ----------
-        path : str
-            Path of directory to create.
-        mode : int
-            Permissions (posix-style) for the newly-created directory.
-        exist_ok : bool
-            If False, raise an OSError if the target directory already exists.
-        """
-
-        return os.makedirs(path, mode, exist_ok)
-
-    def abspath(self, path):
-        """Return the absolute path.
-
-        Parameters
-        ----------
-        path : str
-            Path for which the absolute path will be returned.
-        """
-        return os.path.abspath(path)
-
-    @property
-    def script_dir(self):
-        return self._script_dir
-
-    @script_dir.setter
-    def script_dir(self, value):
-        if value is not None:
-            value = self.abspath(value)
-        self._script_dir = value

parsl/channels/oauth_ssh/oauth_ssh.py

@@ -1,110 +0,0 @@
-import logging
-import paramiko
-import socket
-
-from parsl.errors import OptionalModuleMissing
-from parsl.channels.ssh.ssh import SSHChannel
-
-try:
-    from oauth_ssh.ssh_service import SSHService
-    from oauth_ssh.oauth_ssh_token import find_access_token
-    _oauth_ssh_enabled = True
-except (ImportError, NameError):
-    _oauth_ssh_enabled = False
-
-
-logger = logging.getLogger(__name__)
-
-
-class OAuthSSHChannel(SSHChannel):
-    """SSH persistent channel. This enables remote execution on sites
-    accessible via ssh. This channel uses Globus based OAuth tokens for authentication.
-    """
-
-    def __init__(self, hostname, username=None, script_dir=None, envs=None, port=22):
-        ''' Initialize a persistent connection to the remote system.
-        We should know at this point whether ssh connectivity is possible
-
-        Args:
-            - hostname (String) : Hostname
-
-        KWargs:
-            - username (string) : Username on remote system
-            - script_dir (string) : Full path to a script dir where
-              generated scripts could be sent to.
-            - envs (dict) : A dictionary of env variables to be set when executing commands
-            - port (int) : Port at which the SSHService is running
-
-        Raises:
-        '''
-        if not _oauth_ssh_enabled:
-            raise OptionalModuleMissing(['oauth_ssh'],
-                                        "OauthSSHChannel requires oauth_ssh module and config.")
-
-        self.hostname = hostname
-        self.username = username
-        self.script_dir = script_dir
-        self.port = port
-        self.envs = {}
-        if envs is not None:
-            self.envs = envs
-
-        try:
-            access_token = find_access_token(hostname)
-        except Exception:
-            logger.exception("Failed to find the access token for {}".format(hostname))
-            raise
-
-        try:
-            self.service = SSHService(hostname, port)
-            self.transport = self.service.login(access_token, username)
-
-        except Exception:
-            logger.exception("Caught an exception in the OAuth authentication step with {}".format(hostname))
-            raise
-
-        self.sftp_client = paramiko.SFTPClient.from_transport(self.transport)
-
-    def execute_wait(self, cmd, walltime=60, envs={}):
-        ''' Synchronously execute a commandline string on the shell.
-
-        This command does *NOT* honor walltime currently.
-
-        Args:
-            - cmd (string) : Commandline string to execute
-
-        Kwargs:
-            - walltime (int) : walltime in seconds
-            - envs (dict) : Dictionary of env variables
-
-        Returns:
-            - retcode : Return code from the execution, -1 on fail
-            - stdout  : stdout string
-            - stderr  : stderr string
-
-        Raises:
-        None.
-        '''
-
-        session = self.transport.open_session()
-        session.setblocking(0)
-
-        nbytes = 1024
-        session.exec_command(self.prepend_envs(cmd, envs))
-        session.settimeout(walltime)
-
-        try:
-            # Wait until command is executed
-            exit_status = session.recv_exit_status()
-
-            stdout = session.recv(nbytes).decode('utf-8')
-            stderr = session.recv_stderr(nbytes).decode('utf-8')
-
-        except socket.timeout:
-            logger.exception("Command failed to execute without timeout limit on {}".format(self))
-            raise
-
-        return exit_status, stdout, stderr
-
-    def close(self):
-        return self.transport.close()

parsl/channels/ssh/ssh.py

@@ -1,276 +0,0 @@
-import errno
-import logging
-import os
-
-import paramiko
-from parsl.channels.base import Channel
-from parsl.channels.errors import BadHostKeyException, AuthException, SSHException, BadScriptPath, BadPermsScriptPath, FileCopyException
-from parsl.utils import RepresentationMixin
-
-logger = logging.getLogger(__name__)
-
-
-class NoAuthSSHClient(paramiko.SSHClient):
-    def _auth(self, username, *args):
-        self._transport.auth_none(username)
-        return
-
-
-class SSHChannel(Channel, RepresentationMixin):
-    ''' SSH persistent channel. This enables remote execution on sites
-    accessible via ssh. It is assumed that the user has setup host keys
-    so as to ssh to the remote host. Which goes to say that the following
-    test on the commandline should work:
-
-    >>> ssh <username>@<hostname>
-
-    '''
-
-    def __init__(self, hostname, username=None, password=None, script_dir=None, envs=None,
-                 gssapi_auth=False, skip_auth=False, port=22, key_filename=None, host_keys_filename=None):
-        ''' Initialize a persistent connection to the remote system.
-        We should know at this point whether ssh connectivity is possible
-
-        Args:
-            - hostname (String) : Hostname
-
-        KWargs:
-            - username (string) : Username on remote system
-            - password (string) : Password for remote system
-            - port : The port designated for the ssh connection. Default is 22.
-            - script_dir (string) : Full path to a script dir where
-              generated scripts could be sent to.
-            - envs (dict) : A dictionary of environment variables to be set when executing commands
-            - key_filename (string or list): the filename, or list of filenames, of optional private key(s)
-
-        Raises:
-        '''
-
-        self.hostname = hostname
-        self.username = username
-        self.password = password
-        self.port = port
-        self.script_dir = script_dir
-        self.skip_auth = skip_auth
-        self.gssapi_auth = gssapi_auth
-        self.key_filename = key_filename
-        self.host_keys_filename = host_keys_filename
-
-        if self.skip_auth:
-            self.ssh_client = NoAuthSSHClient()
-        else:
-            self.ssh_client = paramiko.SSHClient()
-        self.ssh_client.load_system_host_keys(filename=host_keys_filename)
-        self.ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-        self.sftp_client = None
-
-        self.envs = {}
-        if envs is not None:
-            self.envs = envs
-
-    def _is_connected(self):
-        transport = self.ssh_client.get_transport() if self.ssh_client else None
-        return transport and transport.is_active()
-
-    def _connect(self):
-        if not self._is_connected():
-            logger.debug(f"connecting to {self.hostname}:{self.port}")
-            try:
-                self.ssh_client.connect(
-                    self.hostname,
-                    username=self.username,
-                    password=self.password,
-                    port=self.port,
-                    allow_agent=True,
-                    gss_auth=self.gssapi_auth,
-                    gss_kex=self.gssapi_auth,
-                    key_filename=self.key_filename
-                )
-                transport = self.ssh_client.get_transport()
-                self.sftp_client = paramiko.SFTPClient.from_transport(transport)
-
-            except paramiko.BadHostKeyException as e:
-                raise BadHostKeyException(e, self.hostname)
-
-            except paramiko.AuthenticationException as e:
-                raise AuthException(e, self.hostname)
-
-            except paramiko.SSHException as e:
-                raise SSHException(e, self.hostname)
-
-            except Exception as e:
-                raise SSHException(e, self.hostname)
-
-    def _valid_sftp_client(self):
-        self._connect()
-        return self.sftp_client
-
-    def _valid_ssh_client(self):
-        self._connect()
-        return self.ssh_client
-
-    def prepend_envs(self, cmd, env={}):
-        env.update(self.envs)
-
-        if len(env.keys()) > 0:
-            env_vars = ' '.join(['{}={}'.format(key, value) for key, value in env.items()])
-            return 'env {0} {1}'.format(env_vars, cmd)
-        return cmd
-
-    def execute_wait(self, cmd, walltime=2, envs={}):
-        ''' Synchronously execute a commandline string on the shell.
-
-        Args:
-            - cmd (string) : Commandline string to execute
-            - walltime (int) : walltime in seconds
-
-        Kwargs:
-            - envs (dict) : Dictionary of env variables
-
-        Returns:
-            - retcode : Return code from the execution, -1 on fail
-            - stdout  : stdout string
-            - stderr  : stderr string
-
-        Raises:
-        None.
-        '''
-
-        # Execute the command
-        stdin, stdout, stderr = self._valid_ssh_client().exec_command(
-            self.prepend_envs(cmd, envs), bufsize=-1, timeout=walltime
-        )
-        # Block on exit status from the command
-        exit_status = stdout.channel.recv_exit_status()
-        return exit_status, stdout.read().decode("utf-8"), stderr.read().decode("utf-8")
-
-    def push_file(self, local_source, remote_dir):
-        ''' Transport a local file to a directory on a remote machine
-
-        Args:
-            - local_source (string): Path
-            - remote_dir (string): Remote path
-
-        Returns:
-            - str: Path to copied file on remote machine
-
-        Raises:
-            - BadScriptPath : if script path on the remote side is bad
-            - BadPermsScriptPath : You do not have perms to make the channel script dir
-            - FileCopyException : FileCopy failed.
-
-        '''
-        remote_dest = os.path.join(remote_dir, os.path.basename(local_source))
-
-        try:
-            self.makedirs(remote_dir, exist_ok=True)
-        except IOError as e:
-            logger.exception("Pushing {0} to {1} failed".format(local_source, remote_dir))
-            if e.errno == 2:
-                raise BadScriptPath(e, self.hostname)
-            elif e.errno == 13:
-                raise BadPermsScriptPath(e, self.hostname)
-            else:
-                logger.exception("File push failed due to SFTP client failure")
-                raise FileCopyException(e, self.hostname)
-        try:
-            self._valid_sftp_client().put(local_source, remote_dest, confirm=True)
-            # Set perm because some systems require the script to be executable
-            self._valid_sftp_client().chmod(remote_dest, 0o700)
-        except Exception as e:
-            logger.exception("File push from local source {} to remote destination {} failed".format(
-                local_source, remote_dest))
-            raise FileCopyException(e, self.hostname)
-
-        return remote_dest
-
-    def pull_file(self, remote_source, local_dir):
-        ''' Transport file on the remote side to a local directory
-
-        Args:
-            - remote_source (string): remote_source
-            - local_dir (string): Local directory to copy to
-
-
-        Returns:
-            - str: Local path to file
-
-        Raises:
-            - FileExists : Name collision at local directory.
-            - FileCopyException : FileCopy failed.
-        '''
-
-        local_dest = os.path.join(local_dir, os.path.basename(remote_source))
-
-        try:
-            os.makedirs(local_dir)
-        except OSError as e:
-            if e.errno != errno.EEXIST:
-                logger.exception("Failed to create local_dir: {0}".format(local_dir))
-                raise BadScriptPath(e, self.hostname)
-
-        try:
-            self._valid_sftp_client().get(remote_source, local_dest)
-        except Exception as e:
-            logger.exception("File pull failed")
-            raise FileCopyException(e, self.hostname)
-
-        return local_dest
-
-    def close(self):
-        if self._is_connected():
-            return self.ssh_client.close()
-
-    def isdir(self, path):
-        """Return true if the path refers to an existing directory.
-
-        Parameters
-        ----------
-        path : str
-            Path of directory on the remote side to check.
-        """
-        result = True
-        try:
-            self._valid_sftp_client().lstat(path)
-        except FileNotFoundError:
-            result = False
-
-        return result
-
-    def makedirs(self, path, mode=0o700, exist_ok=False):
-        """Create a directory on the remote side.
-
-        If intermediate directories do not exist, they will be created.
-
-        Parameters
-        ----------
-        path : str
-            Path of directory on the remote side to create.
-        mode : int
-            Permissions (posix-style) for the newly-created directory.
-        exist_ok : bool
-            If False, raise an OSError if the target directory already exists.
-        """
-        if exist_ok is False and self.isdir(path):
-            raise OSError('Target directory {} already exists'.format(path))
-
-        self.execute_wait('mkdir -p {}'.format(path))
-        self._valid_sftp_client().chmod(path, mode)
-
-    def abspath(self, path):
-        """Return the absolute path on the remote side.
-
-        Parameters
-        ----------
-        path : str
-            Path for which the absolute path will be returned.
-        """
-        return self._valid_sftp_client().normalize(path)
-
-    @property
-    def script_dir(self):
-        return self._script_dir
-
-    @script_dir.setter
-    def script_dir(self, value):
-        self._script_dir = value