pyinfra 0.11.dev3__py3-none-any.whl → 3.5.1__py3-none-any.whl

pyinfra/connectors/ssh_util.py

@@ -0,0 +1,114 @@
+from getpass import getpass
+from os import path
+from typing import TYPE_CHECKING, Type, Union
+
+from paramiko import (
+    DSSKey,
+    ECDSAKey,
+    Ed25519Key,
+    PasswordRequiredException,
+    PKey,
+    RSAKey,
+    SSHException,
+)
+
+import pyinfra
+from pyinfra.api.exceptions import ConnectError, PyinfraError
+
+if TYPE_CHECKING:
+    from pyinfra.api.host import Host
+    from pyinfra.api.state import State
+
+
+def raise_connect_error(host: "Host", message, data):
+    message = "{0} ({1})".format(message, data)
+    raise ConnectError(message)
+
+
+def _load_private_key_file(filename: str, key_filename: str, key_password: str):
+    exception: Union[PyinfraError, SSHException] = PyinfraError("Invalid key: {0}".format(filename))
+
+    key_cls: Union[Type[RSAKey], Type[DSSKey], Type[ECDSAKey], Type[Ed25519Key]]
+
+    for key_cls in (RSAKey, DSSKey, ECDSAKey, Ed25519Key):
+        try:
+            return key_cls.from_private_key_file(
+                filename=filename,
+            )
+
+        except PasswordRequiredException:
+            if not key_password:
+                # If password is not provided, but we're in CLI mode, ask for it. I'm not a
+                # huge fan of having CLI specific code in here, but it doesn't really fit
+                # anywhere else without duplicating lots of key related code into cli.py.
+                if pyinfra.is_cli:
+                    key_password = getpass(
+                        "Enter password for private key: {0}: ".format(
+                            key_filename,
+                        ),
+                    )
+
+                # API mode and no password? We can't continue!
+                else:
+                    raise PyinfraError(
+                        "Private key file ({0}) is encrypted, set ssh_key_password to "
+                        "use this key".format(key_filename),
+                    )
+
+            try:
+                return key_cls.from_private_key_file(
+                    filename=filename,
+                    password=key_password,
+                )
+            except SSHException as e:  # key does not match key_cls type
+                exception = e
+        except SSHException as e:  # key does not match key_cls type
+            exception = e
+    raise exception
+
+
+def get_private_key(state: "State", key_filename: str, key_password: str) -> PKey:
+    if key_filename in state.private_keys:
+        return state.private_keys[key_filename]
+
+    ssh_key_filenames = [
+        # Global from executed directory
+        path.expanduser(key_filename),
+    ]
+
+    if state.cwd:
+        # Relative to the CWD
+        path.join(state.cwd, key_filename)
+
+    key = None
+    key_file_exists = False
+
+    for filename in ssh_key_filenames:
+        if not path.isfile(filename):
+            continue
+
+        key_file_exists = True
+
+        try:
+            key = _load_private_key_file(filename, key_filename, key_password)
+            break
+        except SSHException:
+            pass
+
+    # No break, so no key found
+    if not key:
+        if not key_file_exists:
+            raise PyinfraError("No such private key file: {0}".format(key_filename))
+        raise PyinfraError("Invalid private key file: {0}".format(key_filename))
+
+    # Load any certificate, names from OpenSSH:
+    # https://github.com/openssh/openssh-portable/blob/049297de975b92adcc2db77e3fb7046c0e3c695d/ssh-keygen.c#L2453  # noqa: E501
+    for certificate_filename in (
+        "{0}-cert.pub".format(key_filename),
+        "{0}.pub".format(key_filename),
+    ):
+        if path.isfile(certificate_filename):
+            key.load_certificate(certificate_filename)
+
+    state.private_keys[key_filename] = key
+    return key

pyinfra/connectors/sshuserclient/client.py

@@ -0,0 +1,309 @@
+"""
+This file as originally part of the "sshuserclient" pypi package. The GitHub
+source has now vanished (https://github.com/tobald/sshuserclient).
+"""
+
+from os import path
+
+from gevent.lock import BoundedSemaphore
+from paramiko import (
+    HostKeys,
+    MissingHostKeyPolicy,
+    ProxyCommand,
+    SSHClient as ParamikoClient,
+    SSHException,
+)
+from paramiko.agent import AgentRequestHandler
+from paramiko.hostkeys import HostKeyEntry
+from typing_extensions import override
+
+from pyinfra import logger
+from pyinfra.api.util import memoize
+
+from .config import SSHConfig
+
+HOST_KEYS_LOCK = BoundedSemaphore()
+
+
+class StrictPolicy(MissingHostKeyPolicy):
+    @override
+    def missing_host_key(self, client, hostname, key):
+        logger.error("No host key for {0} found in known_hosts".format(hostname))
+        raise SSHException(
+            "StrictPolicy: No host key for {0} found in known_hosts".format(hostname),
+        )
+
+
+def append_hostkey(client, hostname, key):
+    """Append hostname to the clients host_keys_file"""
+
+    with HOST_KEYS_LOCK:
+        # The paramiko client saves host keys incorrectly whereas the host keys object does
+        # this correctly, so use that with the client filename variable.
+        # See: https://github.com/paramiko/paramiko/pull/1989
+        host_key_entry = HostKeyEntry([hostname], key)
+        if host_key_entry is None:
+            raise SSHException(
+                "Append Hostkey: Failed to parse host {0}, could not append to hostfile".format(
+                    hostname
+                ),
+            )
+        with open(client._host_keys_filename, "a") as host_keys_file:
+            hk_entry = host_key_entry.to_line()
+            if hk_entry is None:
+                raise SSHException(f"Append Hostkey: Failed to append hostkey ({host_key_entry})")
+
+            host_keys_file.write(hk_entry)
+
+
+class AcceptNewPolicy(MissingHostKeyPolicy):
+    @override
+    def missing_host_key(self, client, hostname, key):
+        logger.warning(
+            (
+                f"No host key for {hostname} found in known_hosts, "
+                "accepting & adding to host keys file"
+            ),
+        )
+
+        append_hostkey(client, hostname, key)
+        logger.warning("Added host key for {0} to known_hosts".format(hostname))
+
+
+class AskPolicy(MissingHostKeyPolicy):
+    @override
+    def missing_host_key(self, client, hostname, key):
+        should_continue = input(
+            "No host key for {0} found in known_hosts, do you want to continue [y/n] ".format(
+                hostname,
+            ),
+        )
+        if should_continue.lower() != "y":
+            raise SSHException(
+                "AskPolicy: No host key for {0} found in known_hosts".format(hostname),
+            )
+        append_hostkey(client, hostname, key)
+        logger.warning("Added host key for {0} to known_hosts".format(hostname))
+        return
+
+
+class WarningPolicy(MissingHostKeyPolicy):
+    @override
+    def missing_host_key(self, client, hostname, key):
+        logger.warning("No host key for {0} found in known_hosts".format(hostname))
+
+
+def get_missing_host_key_policy(policy):
+    if policy is None or policy == "ask":
+        return AskPolicy()
+    if policy == "no" or policy == "off":
+        return WarningPolicy()
+    if policy == "yes":
+        return StrictPolicy()
+    if policy == "accept-new":
+        return AcceptNewPolicy()
+    raise SSHException("Invalid value StrictHostKeyChecking={}".format(policy))
+
+
+@memoize
+def get_ssh_config(user_config_file=None):
+    logger.debug("Loading SSH config: %s", user_config_file)
+
+    if user_config_file is None:
+        user_config_file = path.expanduser("~/.ssh/config")
+
+    if path.exists(user_config_file):
+        with open(user_config_file, encoding="utf-8") as f:
+            ssh_config = SSHConfig()
+            ssh_config.parse(f)
+            return ssh_config
+
+
+@memoize
+def get_host_keys(filename):
+    with HOST_KEYS_LOCK:
+        host_keys = HostKeys()
+
+        try:
+            host_keys.load(filename)
+        # When paramiko encounters a bad host keys line it sometimes bails the
+        # entire load incorrectly.
+        # See: https://github.com/paramiko/paramiko/pull/1990
+        except Exception as e:
+            logger.warning("Failed to load host keys from {0}: {1}".format(filename, e))
+
+        return host_keys
+
+
+class SSHClient(ParamikoClient):
+    """
+    An SSHClient which honors ssh_config and supports proxyjumping
+    original idea at http://bitprophet.org/blog/2012/11/05/gateway-solutions/.
+    """
+
+    @override
+    def connect(  # type: ignore[override]
+        self,
+        hostname,
+        _pyinfra_ssh_forward_agent=None,
+        _pyinfra_ssh_config_file=None,
+        _pyinfra_ssh_known_hosts_file=None,
+        _pyinfra_ssh_strict_host_key_checking=None,
+        _pyinfra_ssh_paramiko_connect_kwargs=None,
+        **kwargs,
+    ):
+        (
+            hostname,
+            config,
+            forward_agent,
+            missing_host_key_policy,
+            host_keys_file,
+            keep_alive,
+        ) = self.parse_config(
+            hostname,
+            kwargs,
+            ssh_config_file=_pyinfra_ssh_config_file,
+            strict_host_key_checking=_pyinfra_ssh_strict_host_key_checking,
+        )
+        self.set_missing_host_key_policy(missing_host_key_policy)
+        config.update(kwargs)
+
+        if _pyinfra_ssh_known_hosts_file:
+            host_keys_file = _pyinfra_ssh_known_hosts_file
+
+        # Overwrite paramiko empty defaults with @memoize-d host keys object
+        self._host_keys = get_host_keys(host_keys_file)
+        self._host_keys_filename = host_keys_file
+
+        if _pyinfra_ssh_paramiko_connect_kwargs:
+            config.update(_pyinfra_ssh_paramiko_connect_kwargs)
+
+        self._ssh_config = config
+        super().connect(hostname, **config)
+
+        if _pyinfra_ssh_forward_agent is not None:
+            forward_agent = _pyinfra_ssh_forward_agent
+
+        if keep_alive:
+            transport = self.get_transport()
+            assert transport is not None, "No transport"
+            transport.set_keepalive(keep_alive)
+
+        if forward_agent:
+            transport = self.get_transport()
+            assert transport is not None, "No transport"
+            session = transport.open_session()
+            AgentRequestHandler(session)
+
+    def gateway(self, hostname, host_port, target, target_port):
+        transport = self.get_transport()
+        assert transport is not None, "No transport"
+        return transport.open_channel(
+            "direct-tcpip",
+            (target, target_port),
+            (hostname, host_port),
+        )
+
+    def parse_config(
+        self,
+        hostname,
+        initial_cfg=None,
+        ssh_config_file=None,
+        strict_host_key_checking=None,
+    ):
+        cfg: dict = {"port": 22}
+        cfg.update(initial_cfg or {})
+
+        keep_alive = 0
+        forward_agent = False
+        missing_host_key_policy = get_missing_host_key_policy(strict_host_key_checking)
+        host_keys_file = path.expanduser("~/.ssh/known_hosts")  # OpenSSH default
+
+        ssh_config = get_ssh_config(ssh_config_file)
+        if not ssh_config:
+            return hostname, cfg, forward_agent, missing_host_key_policy, host_keys_file, keep_alive
+
+        host_config = ssh_config.lookup(hostname)
+        forward_agent = host_config.get("forwardagent") == "yes"
+
+        # If not overridden, apply any StrictHostKeyChecking
+        if strict_host_key_checking is None and "stricthostkeychecking" in host_config:
+            missing_host_key_policy = get_missing_host_key_policy(
+                host_config["stricthostkeychecking"],
+            )
+
+        if "userknownhostsfile" in host_config:
+            host_keys_file = path.expanduser(host_config["userknownhostsfile"])
+
+        if "hostname" in host_config:
+            hostname = host_config["hostname"]
+
+        if "user" in host_config:
+            cfg["username"] = host_config["user"]
+
+        if "identityfile" in host_config:
+            cfg["key_filename"] = host_config["identityfile"]
+
+        if "port" in host_config:
+            cfg["port"] = int(host_config["port"])
+
+        if "serveraliveinterval" in host_config:
+            keep_alive = int(host_config["serveraliveinterval"])
+
+        if "proxycommand" in host_config:
+            cfg["sock"] = ProxyCommand(host_config["proxycommand"])
+
+        elif "proxyjump" in host_config:
+            hops = host_config["proxyjump"].split(",")
+            sock = None
+
+            for i, hop in enumerate(hops):
+                hop_hostname, hop_config = self.derive_shorthand(ssh_config, hop)
+                logger.debug("SSH ProxyJump through %s:%s", hop_hostname, hop_config["port"])
+
+                c = SSHClient()
+                c.connect(
+                    hop_hostname, _pyinfra_ssh_config_file=ssh_config_file, sock=sock, **hop_config
+                )
+
+                if i == len(hops) - 1:
+                    target = hostname
+                    target_config = {"port": cfg["port"]}
+                else:
+                    target, target_config = self.derive_shorthand(ssh_config, hops[i + 1])
+
+                sock = c.gateway(hostname, cfg["port"], target, target_config["port"])
+            cfg["sock"] = sock
+
+        return hostname, cfg, forward_agent, missing_host_key_policy, host_keys_file, keep_alive
+
+    @staticmethod
+    def derive_shorthand(ssh_config, host_string):
+        shorthand_config = {}
+        user_hostport = host_string.rsplit("@", 1)
+        hostport = user_hostport.pop()
+        user = user_hostport[0] if user_hostport and user_hostport[0] else None
+        if user:
+            shorthand_config["username"] = user
+
+        # IPv6: can't reliably tell where addr ends and port begins, so don't
+        # try (and don't bother adding special syntax either, user should avoid
+        # this situation by using port=).
+        if hostport.count(":") > 1:
+            hostname = hostport
+        # IPv4: can split on ':' reliably.
+        else:
+            host_port = hostport.rsplit(":", 1)
+            hostname = host_port.pop(0) or None
+            if host_port and host_port[0]:
+                shorthand_config["port"] = int(host_port[0])
+
+        base_config = ssh_config.lookup(hostname)
+
+        config = {
+            "port": base_config.get("port", 22),
+            "username": base_config.get("user"),
+        }
+        config.update(shorthand_config)
+
+        return hostname, config

pyinfra/connectors/sshuserclient/config.py

@@ -0,0 +1,102 @@
+"""
+This file as originally part of the "sshuserclient" pypi package. The GitHub
+source has now vanished (https://github.com/tobald/sshuserclient).
+"""
+
+import glob
+import re
+from os import environ, path
+
+import paramiko.config
+from gevent.subprocess import CalledProcessError, check_call
+from paramiko import SSHConfig as ParamikoSSHConfig
+from typing_extensions import override
+
+from pyinfra import logger
+
+SETTINGS_REGEX = re.compile(r"(\w+)(?:\s*=\s*|\s+)(.+)")
+
+
+class FakeInvokeResult:
+    ok = False
+
+
+class FakeInvoke:
+    @staticmethod
+    def run(cmd, *args, **kwargs):
+        result = FakeInvokeResult()
+
+        try:
+            cmd = [environ["SHELL"], cmd]
+            try:
+                code = check_call(cmd)
+            except CalledProcessError as e:
+                code = e.returncode
+            result.ok = code == 0
+        except Exception as e:
+            logger.warning(
+                ("pyinfra encountered an error loading SSH config match exec {0}: {1}").format(
+                    cmd,
+                    e,
+                ),
+            )
+
+        return result
+
+
+paramiko.config.invoke = FakeInvoke  # type: ignore
+
+
+def _expand_include_statements(file_obj, parsed_files=None):
+    parsed_lines = []
+
+    for line in file_obj.readlines():
+        line = line.strip()
+        if not line or line.startswith("#"):
+            continue
+
+        match = re.match(SETTINGS_REGEX, line)
+        if not match:
+            parsed_lines.append(line)
+            continue
+
+        key = match.group(1).lower()
+        value = match.group(2)
+
+        if key != "include":
+            parsed_lines.append(line)
+            continue
+
+        if parsed_files is None:
+            parsed_files = []
+
+        # The path can be relative to its parent configuration file
+        if path.isabs(value) is False and value[0] != "~":
+            folder = path.dirname(file_obj.name)
+            value = path.join(folder, value)
+
+        value = path.expanduser(value)
+
+        for filename in glob.iglob(value):
+            if path.isfile(filename):
+                if filename in parsed_files:
+                    raise Exception(
+                        "Include loop detected in ssh config file: %s" % filename,
+                    )
+                with open(filename, encoding="utf-8") as fd:
+                    parsed_files.append(filename)
+                    parsed_lines.extend(_expand_include_statements(fd, parsed_files))
+
+    return parsed_lines
+
+
+class SSHConfig(ParamikoSSHConfig):
+    """
+    an SSHConfig that supports includes directives
+    https://github.com/paramiko/paramiko/pull/1194
+    """
+
+    @override
+    def parse(self, file_obj):
+        file_obj = _expand_include_statements(file_obj)
+        return super().parse(file_obj)

pyinfra/connectors/terraform.py

@@ -0,0 +1,135 @@
+import json
+
+from typing_extensions import override
+
+from pyinfra import local, logger
+from pyinfra.api.exceptions import InventoryError
+from pyinfra.api.util import memoize
+from pyinfra.progress import progress_spinner
+
+from .base import BaseConnector
+
+
+@memoize
+def show_warning() -> None:
+    logger.warning("The @terraform connector is in beta!")
+
+
+def _flatten_dict_gen(d, parent_key, sep):
+    for k, v in d.items():
+        new_key = parent_key + sep + k if parent_key else k
+        yield new_key, v
+        if isinstance(v, dict):
+            yield from _flatten_dict(v, new_key, sep=sep).items()
+
+
+def _flatten_dict(d: dict, parent_key: str = "", sep: str = "."):
+    return dict(_flatten_dict_gen(d, parent_key, sep))
+
+
+class TerraformInventoryConnector(BaseConnector):
+    """
+    Generate one or more SSH hosts from a Terraform output variable. The variable
+    must be a list of hostnames or dictionaries.
+
+    Output is fetched from a flattened JSON dictionary output from ``terraform output
+    -json``. For example the following object:
+
+    .. code:: json
+
+        {
+          "server_group": {
+            "value": {
+              "server_group_node_ips": [
+                "1.2.3.4",
+                "1.2.3.5",
+                "1.2.3.6"
+              ]
+            }
+          }
+        }
+
+    The IP list ``server_group_node_ips`` would be used like so:
+
+    .. code:: sh
+
+        pyinfra @terraform/server_group.value.server_group_node_ips ...
+
+    You can also specify dictionaries to include extra data with hosts:
+
+    .. code:: json
+
+        {
+          "server_group": {
+            "value": {
+              "server_group_node_ips": [
+                {
+                    "ssh_hostname": "1.2.3.4",
+                    "ssh_user": "ssh-user"
+                },
+                {
+                    "ssh_hostname": "1.2.3.5",
+                    "ssh_user": "ssh-user"
+                }
+              ]
+            }
+          }
+        }
+
+    """
+
+    @override
+    @staticmethod
+    def make_names_data(name=None):
+        show_warning()
+
+        if not name:
+            # This is the default which allows one to create a Terraform output
+            # "pyinfra" and directly call: pyinfra @terraform ...
+            name = "pyinfra_inventory.value"
+
+        with progress_spinner({"fetch terraform output"}):
+            tf_output_raw = local.shell("terraform output -json")
+
+        assert isinstance(tf_output_raw, str)
+        tf_output = json.loads(tf_output_raw)
+        tf_output = _flatten_dict(tf_output)
+
+        tf_output_value = tf_output.get(name)
+        if tf_output_value is None:
+            keys = "\n".join(f"   - {k}" for k in tf_output.keys())
+            raise InventoryError(f"No Terraform output with key: `{name}`, valid keys:\n{keys}")
+
+        if not isinstance(tf_output_value, (list, dict)):
+            raise InventoryError(
+                "Invalid Terraform output type, should be `list`, got "
+                f"`{type(tf_output_value).__name__}`",
+            )
+
+        if isinstance(tf_output_value, list):
+            tf_output_value = {
+                "all": tf_output_value,
+            }
+
+        for group_name, hosts in tf_output_value.items():
+            if not isinstance(hosts, list):
+                raise InventoryError(
+                    "Invalid Terraform map value type, all values should be `list`, got "
+                    f"`{type(hosts).__name__}`",
+                )
+            for host in hosts:
+                if isinstance(host, dict):
+                    name = host.pop("name", host.get("ssh_hostname"))
+                    if name is None:
+                        raise InventoryError(
+                            "Invalid Terraform list item, missing `name` or `ssh_hostname` keys",
+                        )
+                    yield f"@terraform/{name}", host, ["@terraform", group_name]
+                elif isinstance(host, str):
+                    data = {"ssh_hostname": host}
+                    yield f"@terraform/{host}", data, ["@terraform", group_name]
+                else:
+                    raise InventoryError(
+                        "Invalid Terraform list item, should be `dict` or `str` got "
+                        f"`{type(host).__name__}`",
+                    )