pyinfra 0.11.dev3__py3-none-any.whl → 3.5.1__py3-none-any.whl

pyinfra/api/connectors/ssh.py

@@ -1,402 +0,0 @@
-from getpass import getpass
-from os import path
-from socket import (
-    error as socket_error,
-    gaierror,
-)
-
-import click
-
-from paramiko import (
-    AuthenticationException,
-    MissingHostKeyPolicy,
-    PasswordRequiredException,
-    RSAKey,
-    SFTPClient,
-    SSHException,
-)
-from paramiko.agent import AgentRequestHandler
-
-import pyinfra
-
-from pyinfra import logger
-from pyinfra.api.exceptions import PyinfraError
-from pyinfra.api.util import get_file_io, make_command, memoize
-
-from .sshuserclient import SSHClient
-from .util import read_buffers_into_queue, split_combined_output
-
-
-def _log_connect_error(host, message, data):
-    logger.error('{0}{1} ({2})'.format(
-        host.print_prefix,
-        click.style(message, 'red'),
-        data,
-    ))
-
-
-def _get_private_key(state, key_filename, key_password):
-    if key_filename in state.private_keys:
-        return state.private_keys[key_filename]
-
-    ssh_key_filenames = [
-        # Global from executed directory
-        path.expanduser(key_filename),
-    ]
-
-    # Relative to the deploy
-    if state.deploy_dir:
-        ssh_key_filenames.append(
-            path.join(state.deploy_dir, key_filename),
-        )
-
-    for filename in ssh_key_filenames:
-        if not path.isfile(filename):
-            continue
-
-        # First, lets try the key without a password
-        try:
-            key = RSAKey.from_private_key_file(
-                filename=filename,
-            )
-            break
-
-        # Key is encrypted!
-        except PasswordRequiredException:
-            # If password is not provided, but we're in CLI mode, ask for it. I'm not a
-            # huge fan of having CLI specific code in here, but it doesn't really fit
-            # anywhere else without duplicating lots of key related code into cli.py.
-            if not key_password:
-                if pyinfra.is_cli:
-                    key_password = getpass(
-                        'Enter password for private key: {0}: '.format(
-                            key_filename,
-                        ),
-                    )
-
-            # API mode and no password? We can't continue!
-                else:
-                    raise PyinfraError(
-                        'Private key file ({0}) is encrypted, set ssh_key_password to '
-                        'use this key'.format(key_filename),
-                    )
-
-            # Now, try opening the key with the password
-            try:
-                key = RSAKey.from_private_key_file(
-                    filename=filename,
-                    password=key_password,
-                )
-                break
-
-            except SSHException:
-                raise PyinfraError(
-                    'Incorrect password for private key: {0}'.format(
-                        key_filename,
-                    ),
-                )
-
-    # No break, so no key found
-    else:
-        raise IOError('No such private key file: {0}'.format(key_filename))
-
-    state.private_keys[key_filename] = key
-    return key
-
-
-def _make_paramiko_kwargs(state, host):
-    kwargs = {
-        'allow_agent': False,
-        'look_for_keys': False,
-    }
-
-    for key, value in (
-        ('username', host.data.ssh_user),
-        ('port', int(host.data.ssh_port or 0)),
-        ('timeout', state.config.CONNECT_TIMEOUT),
-    ):
-        if value:
-            kwargs[key] = value
-
-    # Password auth (boo!)
-    if host.data.ssh_password:
-        kwargs['password'] = host.data.ssh_password
-
-    # Key auth!
-    elif host.data.ssh_key:
-        kwargs['pkey'] = _get_private_key(
-            state,
-            key_filename=host.data.ssh_key,
-            key_password=host.data.ssh_key_password,
-        )
-
-    # No key or password, so let's have paramiko look for SSH agents and user keys
-    else:
-        kwargs['allow_agent'] = True
-        kwargs['look_for_keys'] = True
-
-    return kwargs
-
-
-def connect(state, host, for_fact=None):
-    '''
-    Connect to a single host. Returns the SSH client if succesful. Stateless by
-    design so can be run in parallel.
-    '''
-
-    kwargs = _make_paramiko_kwargs(state, host)
-    logger.debug('Connecting to: {0} ({1})'.format(host.name, kwargs))
-
-    # Hostname can be provided via SSH config (alias), data, or the hosts name
-    hostname = kwargs.pop(
-        'hostname',
-        host.data.ssh_hostname or host.name,
-    )
-
-    try:
-        # Create new client & connect to the host
-        client = SSHClient()
-        client.set_missing_host_key_policy(MissingHostKeyPolicy())
-        client.connect(hostname, **kwargs)
-
-        # Enable SSH forwarding
-        session = client.get_transport().open_session()
-        AgentRequestHandler(session)
-
-        # Log
-        log_message = '{0}{1}'.format(
-            host.print_prefix,
-            click.style('Connected', 'green'),
-        )
-
-        if for_fact:
-            log_message = '{0}{1}'.format(
-                log_message,
-                ' (for {0} fact)'.format(for_fact),
-            )
-
-        logger.info(log_message)
-
-        return client
-
-    except AuthenticationException:
-        auth_kwargs = {}
-
-        for key, value in kwargs.items():
-            if key in ('username', 'password'):
-                auth_kwargs[key] = value
-                continue
-
-            if key == 'pkey' and value:
-                auth_kwargs['key'] = host.data.ssh_key
-
-        auth_args = ', '.join(
-            '{0}={1}'.format(key, value)
-            for key, value in auth_kwargs.items()
-        )
-
-        _log_connect_error(host, 'Authentication error', auth_args)
-
-    except SSHException as e:
-        _log_connect_error(host, 'SSH error', e)
-
-    except gaierror:
-        _log_connect_error(host, 'Could not resolve hostname', hostname)
-
-    except socket_error as e:
-        _log_connect_error(host, 'Could not connect', e)
-
-    except EOFError as e:
-        _log_connect_error(host, 'EOF error', e)
-
-
-def run_shell_command(
-    state, host, command,
-    get_pty=False, timeout=None, print_output=False,
-    return_combined_output=False,
-    **command_kwargs
-):
-    '''
-    Execute a command on the specified host.
-
-    Args:
-        state (``pyinfra.api.State`` obj): state object for this command
-        hostname (string): hostname of the target
-        command (string): actual command to execute
-        sudo (boolean): whether to wrap the command with sudo
-        sudo_user (string): user to sudo to
-        get_pty (boolean): whether to get a PTY before executing the command
-        env (dict): envrionment variables to set
-        timeout (int): timeout for this command to complete before erroring
-
-    Returns:
-        tuple: (exit_code, stdout, stderr)
-        stdout and stderr are both lists of strings from each buffer.
-    '''
-
-    command = make_command(command, **command_kwargs)
-
-    logger.debug('Running command on {0}: (pty={1}) {2}'.format(
-        host.name, get_pty, command,
-    ))
-
-    if print_output:
-        print('{0}>>> {1}'.format(host.print_prefix, command))
-
-    # Run it! Get stdout, stderr & the underlying channel
-    _, stdout_buffer, stderr_buffer = host.connection.exec_command(
-        command,
-        get_pty=get_pty,
-    )
-
-    combined_output = read_buffers_into_queue(
-        host,
-        stdout_buffer,
-        stderr_buffer,
-        timeout=timeout,
-        print_output=print_output,
-    )
-
-    logger.debug('Waiting for exit status...')
-    exit_status = stdout_buffer.channel.recv_exit_status()
-    logger.debug('Command exit status: {0}'.format(exit_status))
-
-    status = exit_status == 0
-
-    if return_combined_output:
-        return status, combined_output
-
-    stdout, stderr = split_combined_output(combined_output)
-    return status, stdout, stderr
-
-
-@memoize
-def _get_sftp_connection(host):
-    transport = host.connection.get_transport()
-    return SFTPClient.from_transport(transport)
-
-
-def _get_file(host, remote_filename, filename_or_io):
-    with get_file_io(filename_or_io) as file_io:
-        sftp = _get_sftp_connection(host)
-        sftp.getfo(remote_filename, file_io)
-
-
-def get_file(
-    state, host, remote_filename, filename_or_io,
-    sudo=False, sudo_user=None, su_user=None, print_output=False,
-    **command_kwargs
-):
-    '''
-    Download a file from the remote host using SFTP. Supports download files
-    with sudo by copying to a temporary directory with read permissions,
-    downloading and then removing the copy.
-    '''
-
-    if sudo or su_user:
-        # Get temp file location
-        temp_file = state.get_temp_filename(remote_filename)
-
-        # Copy the file to the tempfile location and add read permissions
-        command = 'cp {0} {1} && chmod +r {0}'.format(remote_filename, temp_file)
-
-        copy_status, _, stderr = run_shell_command(
-            state, host, command,
-            sudo=sudo, sudo_user=sudo_user, su_user=su_user,
-            print_output=print_output,
-            **command_kwargs
-        )
-
-        if copy_status is False:
-            logger.error('File download copy temp error: {0}'.format('\n'.join(stderr)))
-            return False
-
-        try:
-            _get_file(host, temp_file, filename_or_io)
-
-        # Ensure that, even if we encounter an error, we (attempt to) remove the
-        # temporary copy of the file.
-        finally:
-            remove_status, _, stderr = run_shell_command(
-                state, host, 'rm -f {0}'.format(temp_file),
-                sudo=sudo, sudo_user=sudo_user, su_user=su_user,
-                print_output=print_output,
-                **command_kwargs
-            )
-
-        if remove_status is False:
-            logger.error('File download remove temp error: {0}'.format('\n'.join(stderr)))
-            return False
-
-    else:
-        _get_file(host, remote_filename, filename_or_io)
-
-    if print_output:
-        print('{0}file downloaded: {1}'.format(host.print_prefix, remote_filename))
-
-    return True
-
-
-def _put_file(host, filename_or_io, remote_location):
-    with get_file_io(filename_or_io) as file_io:
-        # Upload it via SFTP
-        sftp = _get_sftp_connection(host)
-
-        try:
-            sftp.putfo(file_io, remote_location)
-        except IOError as e:
-            # IO mismatch errors might indicate full disks
-            message = getattr(e, 'message', None)
-            if message and message.startswith('size mismatch in put!  0 !='):
-                raise IOError('{0} (disk may be full)'.format(e.message))
-
-            raise
-
-
-def put_file(
-    state, host, filename_or_io, remote_filename,
-    sudo=False, sudo_user=None, su_user=None, print_output=False,
-    **command_kwargs
-):
-    '''
-    Upload file-ios to the specified host using SFTP. Supports uploading files
-    with sudo by uploading to a temporary directory then moving & chowning.
-    '''
-
-    # sudo/su are a little more complicated, as you can only sftp with the SSH
-    # user connected, so upload to tmp and copy/chown w/sudo and/or su_user
-    if sudo or su_user:
-        # Get temp file location
-        temp_file = state.get_temp_filename(remote_filename)
-        _put_file(host, filename_or_io, temp_file)
-
-        # Execute run_shell_command w/sudo and/or su_user
-        command = 'mv {0} {1}'.format(temp_file, remote_filename)
-
-        # Move it to the su_user if present
-        if su_user:
-            command = '{0} && chown {1} {2}'.format(command, su_user, remote_filename)
-
-        # Otherwise any sudo_user
-        elif sudo_user:
-            command = '{0} && chown {1} {2}'.format(command, sudo_user, remote_filename)
-
-        status, _, stderr = run_shell_command(
-            state, host, command,
-            sudo=sudo, sudo_user=sudo_user, su_user=su_user,
-            print_output=print_output,
-            **command_kwargs
-        )
-
-        if status is False:
-            logger.error('File upload error: {0}'.format('\n'.join(stderr)))
-            return False
-
-    # No sudo and no su_user, so just upload it!
-    else:
-        _put_file(host, filename_or_io, remote_filename)
-
-    if print_output:
-        print('{0}file uploaded: {1}'.format(host.print_prefix, remote_filename))
-
-    return True

pyinfra/api/connectors/sshuserclient/client.py

@@ -1,105 +0,0 @@
-'''
-This file as originally part of the "sshuserclient" pypi package. The GitHub
-source has now vanished (https://github.com/tobald/sshuserclient).
-'''
-
-import os
-
-from paramiko import (
-    AutoAddPolicy,
-    ProxyCommand,
-    SSHClient as ParamikoClient,
-)
-
-from pyinfra.api.util import memoize
-
-from .config import SSHConfig
-
-
-@memoize
-def get_ssh_config():
-    user_config_file = os.path.expanduser('~/.ssh/config')
-    if os.path.exists(user_config_file):
-        with open(user_config_file) as f:
-            ssh_config = SSHConfig()
-            ssh_config.parse(f)
-            return ssh_config
-
-
-class SSHClient(ParamikoClient):
-    '''
-    An SSHClient which honors ssh_config and supports proxyjumping
-    original idea at http://bitprophet.org/blog/2012/11/05/gateway-solutions/
-    '''
-
-    def connect(self, hostname, **kwargs):
-        self.hostname, self.config = self.parse_config(hostname)
-        self.config.update(kwargs)
-        super(SSHClient, self).connect(hostname, **self.config)
-
-    def gateway(self, target, target_port):
-        transport = self.get_transport()
-        return transport.open_channel(
-            'direct-tcpip',
-            (target, target_port),
-            (self.hostname, self.config['port']))
-
-    def parse_config(self, hostname):
-        cfg = {'port': 22}
-
-        ssh_config = get_ssh_config()
-        host_config = ssh_config.lookup(hostname)
-
-        if 'hostname' in host_config:
-            hostname = host_config['hostname']
-        if 'user' in host_config:
-            cfg['username'] = host_config['user']
-        if 'identityfile' in host_config:
-            cfg['key_filename'] = host_config['identityfile']
-        if 'port' in host_config:
-            cfg['port'] = int(host_config['port'])
-        if 'proxycommand' in host_config:
-            cfg['sock'] = ProxyCommand(host_config['proxycommand'])
-        elif 'proxyjump' in host_config:
-            hops = host_config['proxyjump'].split(',')
-            sock = None
-            for i, hop in enumerate(hops):
-                hop_hostname, hop_config = self.derive_shorthand(hop)
-                c = SSHClient()
-                c.set_missing_host_key_policy(AutoAddPolicy())
-                c.connect(hop_hostname, sock=sock, **hop_config)
-                if i == len(hops) - 1:
-                    target = hostname
-                    target_config = {'port': cfg['port']}
-                else:
-                    target, target_config = self.derive_shorthand(
-                        hops[i + 1])
-                sock = c.gateway(target, target_config['port'])
-            cfg['sock'] = sock
-
-        return hostname, cfg
-
-    def derive_shorthand(self, host_string):
-        config = {}
-        user_hostport = host_string.rsplit('@', 1)
-        hostport = user_hostport.pop()
-        user = user_hostport[0] if user_hostport and user_hostport[0] else None
-        if user:
-            config['username'] = user
-
-        # IPv6: can't reliably tell where addr ends and port begins, so don't
-        # try (and don't bother adding special syntax either, user should avoid
-        # this situation by using port=).
-        if hostport.count(':') > 1:
-            host = hostport
-            config['port'] = 22
-        # IPv4: can split on ':' reliably.
-        else:
-            host_port = hostport.rsplit(':', 1)
-            host = host_port.pop(0) or None
-            if host_port and host_port[0]:
-                config['port'] = int(host_port[0])
-            else:
-                config['port'] = 22
-
-        return host, config

pyinfra/api/connectors/sshuserclient/config.py

@@ -1,90 +0,0 @@
-'''
-This file as originally part of the "sshuserclient" pypi package. The GitHub
-source has now vanished (https://github.com/tobald/sshuserclient).
-'''
-
-import glob
-import os
-import re
-
-from paramiko import SSHConfig as ParamkoSSHConfig
-
-
-class SSHConfig(ParamkoSSHConfig):
-    '''
-    an SSHConfig that supports includes directives
-    https://github.com/paramiko/paramiko/pull/1194
-    '''
-
-    SETTINGS_REGEX = re.compile(r'(\w+)(?:\s*=\s*|\s+)(.+)')
-
-    def parse(self, file_obj, parsed_files=None):
-        '''
-        Read an OpenSSH config from the given file object.
-
-        :param file_obj: a file-like object to read the config file from
-        '''
-        host = {'host': ['*'], 'config': {}}
-        for line in file_obj:
-            # Strip any leading or trailing whitespace from the line.
-            # Refer to https://github.com/paramiko/paramiko/issues/499
-            line = line.strip()
-            if not line or line.startswith('#'):
-                continue
-
-            match = re.match(self.SETTINGS_REGEX, line)
-            if not match:
-                raise Exception('Unparsable line {}'.format(line))
-            key = match.group(1).lower()
-            value = match.group(2)
-
-            if key == 'host':
-                self._config.append(host)
-                host = {
-                    'host': self._get_hosts(value),
-                    'config': {},
-                }
-            elif key == 'proxycommand' and value.lower() == 'none':
-                # Store 'none' as None; prior to 3.x, it will get stripped out
-                # at the end (for compatibility with issue #415). After 3.x, it
-                # will simply not get stripped, leaving a nice explicit marker.
-                host['config'][key] = None
-            elif key == 'include':
-                # support for Include directive in ssh_config
-                path = value
-                # the path can be relative to its parent configuration file
-                if os.path.isabs(path) is False and path[0] != '~':
-                    folder = os.path.dirname(file_obj.name)
-                    path = os.path.join(folder, path)
-
-                # expand the user home path
-                path = os.path.expanduser(path)
-                if parsed_files is None:
-                    parsed_files = []
-
-                # parse every included file
-                for filename in glob.iglob(path):
-                    if os.path.isfile(filename):
-                        if filename in parsed_files:
-                            raise Exception(
-                                'Include loop detected in ssh config file: %s' % filename,
-                            )
-                        with open(filename) as fd:
-                            parsed_files.append(filename)
-                            self.parse(fd, parsed_files)
-
-            else:
-                if value.startswith('"') and value.endswith('"'):
-                    value = value[1:-1]
-
-                # identityfile, localforward, remoteforward keys are special
-                # cases, since they are allowed to be specified multiple times
-                # and they should be tried in order of specification.
-                if key in ['identityfile', 'localforward', 'remoteforward']:
-                    if key in host['config']:
-                        host['config'][key].append(value)
-                    else:
-                        host['config'][key] = [value]
-                elif key not in host['config']:
-                    host['config'][key] = value
-        self._config.append(host)

pyinfra/api/connectors/util.py

@@ -1,63 +0,0 @@
-from socket import timeout as timeout_error
-
-import click
-import gevent
-
-from gevent.queue import Queue
-
-from pyinfra.api.util import read_buffer
-
-
-def read_buffers_into_queue(host, stdout_buffer, stderr_buffer, timeout, print_output):
-    output_queue = Queue()
-
-    # Iterate through outputs to get an exit status and generate desired list
-    # output, done in two greenlets so stdout isn't printed before stderr. Not
-    # attached to state.pool to avoid blocking it with 2x n-hosts greenlets.
-    stdout_reader = gevent.spawn(
-        read_buffer,
-        'stdout',
-        stdout_buffer,
-        output_queue,
-        print_output=print_output,
-        print_func=lambda line: '{0}{1}'.format(host.print_prefix, line),
-    )
-    stderr_reader = gevent.spawn(
-        read_buffer,
-        'stderr',
-        stderr_buffer,
-        output_queue,
-        print_output=print_output,
-        print_func=lambda line: '{0}{1}'.format(
-            host.print_prefix, click.style(line, 'red'),
-        ),
-    )
-
-    # Wait on output, with our timeout (or None)
-    greenlets = gevent.wait((stdout_reader, stderr_reader), timeout=timeout)
-
-    # Timeout doesn't raise an exception, but gevent.wait returns the greenlets
-    # which did complete. So if both haven't completed, we kill them and fail
-    # with a timeout.
-    if len(greenlets) != 2:
-        stdout_reader.kill()
-        stderr_reader.kill()
-
-        raise timeout_error()
-
-    return list(output_queue.queue)
-
-
-def split_combined_output(combined_output):
-    stdout = []
-    stderr = []
-
-    for type_, line in combined_output:
-        if type_ == 'stdout':
-            stdout.append(line)
-        elif type_ == 'stderr':
-            stderr.append(line)
-        else:  # pragma: no cover
-            raise ValueError('Incorrect output line type: {0}'.format(type_))
-
-    return stdout, stderr