pyinfra 3.0.dev0__py2.py3-none-any.whl → 3.0.2__py2.py3-none-any.whl

pyinfra/connectors/ansible.py

@@ -1,175 +0,0 @@
-"""
-**Note**: this connector is a work in progress! While it parses the list of
-hosts OK, it doesn't handle nested groups properly yet.
-
-The `@ansible` connector can be used to parse Ansible inventory files.
-
-.. code:: python
-
-    # Load an Ansible inventory relative to the current directory
-    pyinfra @ansible/path/to/inventory
-
-    # Load using an absolute path
-    pyinfra @ansible//absolute/path/to/inventory
-"""
-import json
-import re
-from collections import defaultdict
-from configparser import ConfigParser
-from os import path
-from typing import TYPE_CHECKING, Optional
-
-from pyinfra import logger
-from pyinfra.api.exceptions import InventoryError
-from pyinfra.api.util import memoize
-
-from .base import BaseConnector
-
-if TYPE_CHECKING:
-    from pyinfra.api.host import Host
-
-try:
-    import yaml
-except ImportError:
-    yaml = None  # type: ignore
-
-
-@memoize
-def show_warning():
-    logger.warning("The @ansible connector is in alpha!")
-
-
-class AnsibleInventoryConnector(BaseConnector):
-    @staticmethod
-    def make_names_data(inventory_filename: Optional[str] = None):
-        show_warning()
-
-        if not inventory_filename:
-            raise InventoryError("No Ansible inventory filename provided!")
-
-        if not path.exists(inventory_filename):
-            raise InventoryError(
-                ("Could not find Ansible inventory file: {0}").format(inventory_filename),
-            )
-
-        return parse_inventory(inventory_filename)
-
-
-def parse_inventory(inventory_filename: str):
-    # fallback to INI if no extension
-    extension = inventory_filename.split(".")[-1] if "." in inventory_filename else "ini"
-
-    # host:set(groups) mapping
-    host_to_groups = {}
-
-    if extension in ["ini"]:
-        host_to_groups = parse_inventory_ini(inventory_filename)
-    elif extension in ["json"]:
-        with open(inventory_filename, encoding="utf-8") as inventory_file:
-            inventory_tree = json.load(inventory_file)
-            # close file early
-        host_to_groups = parse_inventory_tree(inventory_tree)
-    elif extension in ["yaml", "yml"]:
-        if yaml is None:
-            raise Exception(
-                (
-                    "To parse YAML Ansible inventories requires `pyyaml`. "
-                    "Install it with `pip install pyyaml`."
-                ),
-            )
-        with open(inventory_filename, encoding="utf-8") as inventory_file:
-            inventory_tree = yaml.safe_load(inventory_file)
-            # close file early
-        host_to_groups = parse_inventory_tree(inventory_tree)
-    else:
-        raise InventoryError(("Ansible inventory file format not supported: {0}").format(extension))
-
-    return [(host, {}, sorted(list(host_to_groups.get(host, [])))) for host in host_to_groups]
-
-
-def parse_inventory_ini(inventory_filename: str):
-    config = ConfigParser(
-        delimiters=(" "),  # we only handle the hostnames for now
-        allow_no_value=True,  # we don't by default have = values
-        interpolation=None,  # remove any Python interpolation
-    )
-    config.read(inventory_filename)
-
-    host_to_groups = defaultdict(set)
-    group_to_hosts = defaultdict(set)
-    hosts = []
-
-    # First pass - load hosts/groups of hosts
-    for section in config.sections():
-        if ":" in section:  # ignore :children and :vars sections this time
-            continue
-
-        options = config.options(section)
-        for host in _parse_ansible_hosts(options):
-            hosts.append(host)
-            host_to_groups[host].add(section)
-            group_to_hosts[section].add(host)
-
-    # Second pass - load any children groups
-    for section in config.sections():
-        if not section.endswith(":children"):  # we only support :children for now
-            continue
-
-        group_name = section.replace(":children", "")
-
-        options = config.options(section)
-        for sub_group_name in options:
-            sub_group_hosts = group_to_hosts[sub_group_name]
-            for host in sub_group_hosts:
-                host_to_groups[host].add(group_name)
-
-    return host_to_groups
-
-
-def _parse_ansible_hosts(hosts):
-    for host in hosts:
-        expand_match = re.search(r"\[[0-9:]+\]", host)
-        if expand_match:
-            expand_string = host[expand_match.start() : expand_match.end()]
-            bits = expand_string[1:-1].split(":")  # remove the [] either side
-
-            zfill = 0
-            if bits[0].startswith("0"):
-                zfill = len(bits[0])
-
-            start, end = int(bits[0]), int(bits[1])
-            step = int(bits[2]) if len(bits) > 2 else 1
-
-            for n in range(start, end + 1, step):
-                number_as_string = "{0}".format(n)
-                if zfill:
-                    number_as_string = number_as_string.zfill(zfill)
-
-                hostname = host.replace(expand_string, number_as_string)
-                yield hostname
-        else:
-            yield host
-
-
-def parse_inventory_tree(inventory_tree, host_to_groups=dict(), group_stack=set()):
-    for group in inventory_tree:
-        # set logic adds tolerance for duplicate group names
-        groups = group_stack.union({group})
-
-        if "hosts" in inventory_tree[group]:
-            for host in inventory_tree[group]["hosts"]:
-                append_groups_to_host(host, groups, host_to_groups)
-
-        if "children" in inventory_tree[group]:
-            # recursively parse inventory tree
-            parse_inventory_tree(inventory_tree[group]["children"], host_to_groups, groups)
-
-    return host_to_groups
-
-
-def append_groups_to_host(host: "Host", groups, host_to_groups):
-    if host in host_to_groups:
-        # set logic handles de-duplication
-        host_to_groups[host] = host_to_groups[host].union(groups)
-    else:
-        host_to_groups[host] = groups

pyinfra/connectors/mech.py

@@ -1,189 +0,0 @@
-"""
-The ``@mech`` connector reads the current mech status and generates an inventory
-for any running VMs.
-
-.. code:: python
-
-    # Run on all hosts
-    pyinfra @mech ...
-
-    # Run on a specific VM
-    pyinfra @mech/my-vm-name ...
-
-    # Run on multiple named VMs
-    pyinfra @mech/my-vm-name,@mech/another-vm-name ...
-"""
-
-import json
-from os import path
-from queue import Queue
-from threading import Thread
-
-from pyinfra import local, logger
-from pyinfra.api.exceptions import InventoryError
-from pyinfra.api.util import memoize
-from pyinfra.progress import progress_spinner
-
-from .base import BaseConnector
-
-
-def _get_mech_ssh_config(queue, progress, target):
-    logger.debug("Loading SSH config for %s", target)
-
-    # Note: We have to work-around the fact that "mech ssh-config somehost"
-    # does not return the correct "Host" value. When "mech" fixes this
-    # issue we can simply this code.
-    lines = local.shell(
-        "mech ssh-config {0}".format(target),
-        splitlines=True,
-    )
-
-    newlines = []
-    for line in lines:
-        if line.startswith("Host "):
-            newlines.append("Host " + target)
-        else:
-            newlines.append(line)
-
-    queue.put(newlines)
-
-    progress(target)
-
-
-@memoize
-def get_mech_config(limit=None):
-    logger.info("Getting Mech config...")
-
-    if limit and not isinstance(limit, (list, tuple)):
-        limit = [limit]
-
-    # Note: There is no "--machine-readable" option to 'mech status'
-    with progress_spinner({"mech ls"}) as progress:
-        output = local.shell(
-            "mech ls",
-            splitlines=True,
-        )
-        progress("mech ls")
-
-    targets = []
-
-    for line in output:
-        address = ""
-
-        data = line.split()
-        target = data[0]
-
-        if len(data) == 5:
-            address = data[1]
-
-        # Skip anything not in the limit
-        if limit is not None and target not in limit:
-            continue
-
-        # For each vm that has an address, fetch it's SSH config in a thread
-        if address != "" and address[0].isdigit():
-            targets.append(target)
-
-    threads = []
-    config_queue = Queue()  # type: ignore
-
-    with progress_spinner(targets) as progress:
-        for target in targets:
-            thread = Thread(
-                target=_get_mech_ssh_config,
-                args=(config_queue, progress, target),
-            )
-            threads.append(thread)
-            thread.start()
-
-    for thread in threads:
-        thread.join()
-
-    queue_items = list(config_queue.queue)
-
-    lines = []
-    for output in queue_items:
-        lines.extend(output)
-
-    return lines
-
-
-@memoize
-def get_mech_options():
-    if path.exists("@mech.json"):
-        with open("@mech.json", "r", encoding="utf-8") as f:
-            return json.loads(f.read())
-    return {}
-
-
-def _make_name_data(host):
-    mech_options = get_mech_options()
-    mech_host = host["Host"]
-
-    data = {
-        "ssh_hostname": host["HostName"],
-    }
-
-    for config_key, data_key in (
-        ("Port", "ssh_port"),
-        ("User", "ssh_user"),
-        ("IdentityFile", "ssh_key"),
-    ):
-        if config_key in host:
-            data[data_key] = host[config_key]
-
-    # Update any configured JSON data
-    if mech_host in mech_options.get("data", {}):
-        data.update(mech_options["data"][mech_host])
-
-    # Work out groups
-    groups = mech_options.get("groups", {}).get(mech_host, [])
-
-    if "@mech" not in groups:
-        groups.append("@mech")
-
-    return "@mech/{0}".format(host["Host"]), data, groups
-
-
-class MechInventoryConnector(BaseConnector):
-    @staticmethod
-    def make_names_data(limit=None):
-        mech_ssh_info = get_mech_config(limit)
-
-        logger.debug("Got Mech SSH info: \n%s", mech_ssh_info)
-
-        hosts = []
-        current_host = None
-
-        for line in mech_ssh_info:
-            if not line:
-                if current_host:
-                    hosts.append(_make_name_data(current_host))
-
-                current_host = None
-                continue
-
-            key, value = line.strip().split(" ", 1)
-
-            if key == "Host":
-                if current_host:
-                    hosts.append(_make_name_data(current_host))
-
-                # Set the new host
-                current_host = {
-                    key: value,
-                }
-
-            elif current_host:
-                current_host[key] = value
-
-            else:
-                logger.debug("Extra Mech SSH key/value (%s=%s)", key, value)
-
-        if current_host:
-            hosts.append(_make_name_data(current_host))
-
-        if not hosts:
-            raise InventoryError("No running Mech instances found!")
-
-        return hosts

pyinfra/connectors/pyinfrawinrmsession/__init__.py

@@ -1,28 +0,0 @@
-import base64
-
-import winrm
-
-
-class PyinfraWinrmSession(winrm.Session):
-    """This is our subclassed Session that allows for env setting"""
-
-    def run_cmd(self, command, args=(), env=None):
-        shell_id = self.protocol.open_shell(env_vars=env)
-        command_id = self.protocol.run_command(shell_id, command, args)
-        rs = winrm.Response(self.protocol.get_command_output(shell_id, command_id))
-        self.protocol.cleanup_command(shell_id, command_id)
-        self.protocol.close_shell(shell_id)
-        return rs
-
-    def run_ps(self, script, env=None):
-        """base64 encodes a Powershell script and executes the powershell
-        encoded script command
-        """
-        # must use utf16 little endian on windows
-        encoded_ps = base64.b64encode(script.encode("utf_16_le")).decode("ascii")
-        rs = self.run_cmd("powershell -encodedcommand {0}".format(encoded_ps), env=env)
-        if len(rs.std_err):
-            # if there was an error message, clean it it up and make it human
-            # readable
-            rs.std_err = self._clean_error_msg(rs.std_err)
-        return rs

pyinfra/connectors/winrm.py

@@ -1,312 +0,0 @@
-"""
-.. warning::
-    This connector is in alpha and may change in future releases.
-
-Some Windows facts and Windows operations work but this is to be considered
-experimental. For now, only ``winrm_username`` and ``winrm_password`` is
-being used. There are other methods for authentication, but they have not yet
-been added/experimented with.
-
-The ``@winrm`` connector can be used to communicate with Windows instances that have WinRM enabled.
-
-Examples using ``@winrm``:
-
-.. code:: python
-
-    # Get the windows_home fact
-    pyinfra @winrm/192.168.3.232 --winrm-username vagrant \\
-        --winrm-password vagrant --winrm-port 5985 -vv --debug fact windows_home
-
-    # Create a directory
-    pyinfra @winrm/192.168.3.232 --winrm-username vagrant \\
-        --winrm-password vagrant --winrm-port 5985 windows_files.windows_directory 'c:\temp'
-
-    # Run a powershell command ('ps' is the default shell-executable for the winrm connector)
-    pyinfra @winrm/192.168.3.232 --winrm-username vagrant \\
-        --winrm-password vagrant --winrm-port 5985 exec -- write-host hello
-
-    # Run a command using the command prompt:
-    pyinfra @winrm/192.168.3.232 --winrm-username vagrant \\
-        --winrm-password vagrant --winrm-port 5985 --shell-executable cmd exec -- date /T
-
-    # Run a command using the winrm ntlm transport
-    pyinfra @winrm/192.168.3.232 --winrm-username vagrant \\
-        --winrm-password vagrant --winrm-port 5985 --winrm-transport ntlm exec -- hostname
-"""
-
-import base64
-import ntpath
-
-import click
-
-from pyinfra import logger
-from pyinfra.api.exceptions import ConnectError, PyinfraError
-from pyinfra.api.util import get_file_io, memoize, sha1_hash
-
-from .base import BaseConnector, make_keys
-from .pyinfrawinrmsession import PyinfraWinrmSession
-from .util import make_win_command
-
-
-class DataKeys:
-    hostname = "WinRM hostname to connect to"
-    port = "WinRM port to connect to"
-    user = "WinRM username"
-    password = "WinRM password"
-    transport = "WinRM transport (default: ``plaintext``)"
-    read_timeout_sec = "Read timeout in seconds (default: ``30``)"
-    operation_timeout_sec = "Operation timeout in seconds (default: ``20``)"
-
-
-DATA_KEYS = make_keys("winrm", DataKeys)
-
-
-def _raise_connect_error(host, message, data):
-    message = "{0} ({1})".format(message, data)
-    raise ConnectError(message)
-
-
-@memoize
-def show_warning():
-    logger.warning("The @winrm connector is alpha!")
-
-
-def _make_winrm_kwargs(state, host):
-    kwargs = {}
-
-    for key, value in (
-        ("username", host.data.get(DATA_KEYS.user)),
-        ("password", host.data.get(DATA_KEYS.password)),
-        ("winrm_port", int(host.data.get(DATA_KEYS.port, 0))),
-        ("winrm_transport", host.data.get(DATA_KEYS.transport, "plaintext")),
-        (
-            "winrm_read_timeout_sec",
-            host.data.get(DATA_KEYS.read_timeout_sec, 30),
-        ),
-        (
-            "winrm_operation_timeout_sec",
-            host.data.get(DATA_KEYS.operation_timeout_sec, 20),
-        ),
-    ):
-        if value:
-            kwargs[key] = value
-
-    # FUTURE: add more auth
-    # pywinrm supports: basic, certificate, ntlm, kerberos, plaintext, ssl, credssp
-    # see https://github.com/diyan/pywinrm/blob/master/winrm/__init__.py#L12
-
-    return kwargs
-
-
-class WinRMConnector(BaseConnector):
-    @staticmethod
-    def make_names_data(hostname):
-        show_warning()
-
-        yield "@winrm/{0}".format(hostname), {"winrm_hostname": hostname}, []
-
-    def connect(self):
-        """
-        Connect to a single host. Returns the winrm Session if successful.
-        """
-
-        kwargs = _make_winrm_kwargs(self.state, self.host)
-        logger.debug("Connecting to: %s (%s)", self.host.name, kwargs)
-
-        # Hostname can be provided via winrm config (alias), data, or the hosts name
-        hostname = kwargs.pop(
-            "hostname",
-            self.host.data.get(DATA_KEYS.hostname, self.host.name),
-        )
-
-        try:
-            # Create new session
-            host_and_port = "{}:{}".format(hostname, self.host.data.get(DATA_KEYS.port))
-            logger.debug("host_and_port: %s", host_and_port)
-
-            session = PyinfraWinrmSession(
-                host_and_port,
-                auth=(
-                    kwargs["username"],
-                    kwargs["password"],
-                ),
-                transport=kwargs["winrm_transport"],
-                read_timeout_sec=kwargs["winrm_read_timeout_sec"],
-                operation_timeout_sec=kwargs["winrm_operation_timeout_sec"],
-            )
-
-            return session
-
-        # TODO: add exceptions here
-        except Exception as e:
-            auth_kwargs = {}
-
-            for key, value in kwargs.items():
-                if key in ("username", "password"):
-                    auth_kwargs[key] = value
-
-            auth_args = ", ".join(
-                "{0}={1}".format(key, value) for key, value in auth_kwargs.items()
-            )
-            logger.debug("%s", e)
-            _raise_connect_error(self.host, "Authentication error", auth_args)
-
-    def run_shell_command(
-        self,
-        command,
-        env=None,
-        success_exit_codes=None,
-        print_output=False,
-        print_input=False,
-        return_combined_output=False,
-        shell_executable=None,
-        **ignored_command_kwargs,
-    ):
-        """
-        Execute a command on the specified host.
-
-        Args:
-            state (``pyinfra.api.State`` obj): state object for this command
-            hostname (string): hostname of the target
-            command (string): actual command to execute
-            success_exit_codes (list): all values in the list that will return success
-            print_output (boolean): print the output
-            print_intput (boolean): print the input
-            return_combined_output (boolean): combine the stdout and stderr lists
-            shell_executable (string): shell to use - 'cmd'=cmd, 'ps'=powershell(default)
-            env (dict): environment variables to set
-
-        Returns:
-            tuple: (exit_code, stdout, stderr)
-            stdout and stderr are both lists of strings from each buffer.
-        """
-
-        command = make_win_command(command)
-
-        logger.debug("Running command on %s: %s", self.host.name, command)
-
-        if print_input:
-            click.echo("{0}>>> {1}".format(self.host.print_prefix, command), err=True)
-
-        # get rid of leading/trailing quote
-        tmp_command = command.strip("'")
-
-        if print_output:
-            click.echo(
-                "{0}>>> {1}".format(self.host.print_prefix, command),
-                err=True,
-            )
-
-        if not shell_executable:
-            shell_executable = "ps"
-        logger.debug("shell_executable:%s", shell_executable)
-
-        # we use our own subclassed session that allows for env setting from open_shell.
-        if shell_executable in ["cmd"]:
-            response = self.host.connection.run_cmd(tmp_command, env=env)  # type: ignore
-        else:
-            response = self.host.connection.run_ps(tmp_command, env=env)  # type: ignore
-
-        return_code = response.status_code
-        logger.debug("response:%s", response)
-
-        std_out_str = response.std_out.decode("utf-8")
-        std_err_str = response.std_err.decode("utf-8")
-
-        # split on '\r\n' (windows newlines)
-        std_out = std_out_str.split("\r\n")
-        std_err = std_err_str.split("\r\n")
-
-        logger.debug("std_out:%s", std_out)
-        logger.debug("std_err:%s", std_err)
-
-        if print_output:
-            click.echo(
-                "{0}>>> {1}".format(self.host.print_prefix, "\n".join(std_out)),
-                err=True,
-            )
-
-        if success_exit_codes:
-            status = return_code in success_exit_codes
-        else:
-            status = return_code == 0
-
-        logger.debug("Command exit status: %s", status)
-
-        if return_combined_output:
-            std_out = [("stdout", line) for line in std_out]
-            std_err = [("stderr", line) for line in std_err]
-            return status, std_out + std_err
-
-        return status, std_out, std_err
-
-    def get_file(
-        state, host, remote_filename, filename_or_io, remote_temp_filename=None, **command_kwargs
-    ):
-        raise PyinfraError("Not implemented")
-
-    def _put_file(self, filename_or_io, remote_location, chunk_size=2048):
-        # this should work fine on smallish files, but there will be perf issues
-        # on larger files both due to the full read, the base64 encoding, and
-        # the latency when sending chunks
-        with get_file_io(filename_or_io) as file_io:
-            data = file_io.read()
-            for i in range(0, len(data), chunk_size):
-                chunk = data[i : i + chunk_size]
-                ps = (
-                    '$data = [System.Convert]::FromBase64String("{0}"); '
-                    '{1} -Value $data -Encoding byte -Path "{2}"'
-                ).format(
-                    base64.b64encode(chunk).decode("utf-8"),
-                    "Set-Content" if i == 0 else "Add-Content",
-                    remote_location,
-                )
-                status, _stdout, stderr = self.run_shell_command(ps)
-                if status is False:
-                    logger.error("File upload error: {0}".format("\n".join(stderr)))
-                    return False
-
-        return True
-
-    def put_file(
-        self,
-        filename_or_io,
-        remote_filename,
-        print_output=False,
-        print_input=False,
-        remote_temp_filename=None,  # ignored
-        **command_kwargs,
-    ):
-        """
-        Upload file by chunking and sending base64 encoded via winrm
-        """
-
-        # TODO: fix this? Workaround for circular import
-        from pyinfra.facts.windows_files import TempDir
-
-        # Always use temp file here in case of failure
-        temp_file = ntpath.join(
-            self.host.get_fact(TempDir),
-            "pyinfra-{0}".format(sha1_hash(remote_filename)),
-        )
-
-        if not self._put_file(filename_or_io, temp_file):
-            return False
-
-        # Execute run_shell_command w/sudo and/or su_user
-        command = "Move-Item -Path {0} -Destination {1} -Force".format(temp_file, remote_filename)
-        status, _, stderr = self.run_shell_command(
-            command, print_output=print_output, print_input=print_input, **command_kwargs
-        )
-
-        if status is False:
-            logger.error("File upload error: {0}".format("\n".join(stderr)))
-            return False
-
-        if print_output:
-            click.echo(
-                "{0}file uploaded: {1}".format(self.host.print_prefix, remote_filename),
-                err=True,
-            )
-
-        return True