pyinfra 3.0.dev0__py2.py3-none-any.whl → 3.0.2__py2.py3-none-any.whl

pyinfra_cli/inventory.py

@@ -1,4 +1,3 @@
-import re
 import socket
 from collections import defaultdict
 from os import listdir, path
@@ -37,32 +36,33 @@ def _is_inventory_group(key: str, value: Any):
     return all(isinstance(item, ALLOWED_HOST_TYPES) for item in value)
 
 
-def _get_group_data(dirname: str):
+def _get_group_data(dirname_or_filename: str):
     group_data = {}
-    group_data_directory = path.join(dirname, "group_data")
 
-    logger.debug("Checking possible group_data directory: %s", dirname)
+    logger.debug("Checking possible group_data at: %s", dirname_or_filename)
 
-    if path.exists(group_data_directory):
-        files = listdir(group_data_directory)
+    if path.exists(dirname_or_filename):
+        if path.isfile(dirname_or_filename):
+            files = [dirname_or_filename]
+        else:
+            files = [path.join(dirname_or_filename, file) for file in listdir(dirname_or_filename)]
 
         for file in files:
             if not file.endswith(".py"):
                 continue
 
-            group_data_file = path.join(group_data_directory, file)
             group_name = path.basename(file)[:-3]
 
-            logger.debug("Looking for group data in: %s", group_data_file)
+            logger.debug("Looking for group data in: %s", file)
 
             # Read the files locals into a dict
-            attrs = exec_file(group_data_file, return_locals=True)
+            attrs = exec_file(file, return_locals=True)
             keys = attrs.get("__all__", attrs.keys())
 
             group_data[group_name] = {
                 key: value
                 for key, value in attrs.items()
-                if key in keys and not key.startswith("_")
+                if key in keys and not key.startswith("__")
             }
 
     return group_data
@@ -81,32 +81,52 @@ def _get_any_tuple_first(item: Union[T, Tuple[T, Any]]) -> T:
     return item[0] if isinstance(item, tuple) else item
 
 
+def _resolves_to_host(maybe_host: str) -> bool:
+    """Check if a string resolves to a valid IP address."""
+    try:
+        # Use getaddrinfo to support IPv6 hosts
+        socket.getaddrinfo(maybe_host, port=None)
+        return True
+    except socket.gaierror:
+        return False
+
+
 def make_inventory(
     inventory: str,
     override_data=None,
     cwd: Optional[str] = None,
     group_data_directories=None,
 ):
-    inventory_func = None
-
-    # (Un)fortunately the CLI is pretty flexible for inventory inputs; we support a single hostname,
-    # a Python module.function import or a Python file path. All of these are kind of similar, and
-    # we want error handling to be a good user experience.
-    # Thus, we'll check for everything but also drop a warning to the console if the inventory looks
-    # like either an import or hostname but neither works.
-    if re.match("[a-zA-Z0-9\\.]+[\\.:][a-zA-Z0-9]+", inventory):
-        # First, try loading the inventory as if it's a Python import function
-        inventory_func = try_import_module_attribute(inventory, raise_for_no_module=False)
-        if inventory_func is None:
-            try:
-                socket.gethostbyname(inventory)
-            except socket.gaierror:
-                logger.warning(f"{inventory} is neither a valid Python import or hostname.")
-
-    if inventory_func is None:
-        # If not an import, load as if from the filesystem *or* comma separated list, which also
-        # loads any all.py group data files (imported functions do not load group data).
+    # (Un)fortunately the CLI is pretty flexible for inventory inputs; we support inventory files, a
+    # single hostname, list of hosts, connectors, and python module.function or module:function
+    # imports.
+    #
+    # We check first for an inventory file, a list of hosts or anything with a connector, because
+    # (1) an inventory file is a common use case and (2) no other option can have a comma or an @
+    # symbol in them.
+    is_path_or_host_list_or_connector = (
+        path.exists(inventory) or "," in inventory or "@" in inventory
+    )
+    if not is_path_or_host_list_or_connector:
+        # Next, try loading the inventory from a python function. This happens before checking for a
+        # single-host inventory, so that your command does not stop working because somebody
+        # registered the domain `my.module.name`.
+        inventory_func = try_import_module_attribute(inventory, raise_for_none=False)
+
+        # If the inventory does not refer to a module, we finally check if it refers to a reachable
+        # host
+        if inventory_func is None and _resolves_to_host(inventory):
+            is_path_or_host_list_or_connector = True
+
+    if is_path_or_host_list_or_connector:
+        # The inventory is either an inventory file or a (list of) hosts
         return make_inventory_from_files(inventory, override_data, cwd, group_data_directories)
+    elif inventory_func is None:
+        logger.warn(
+            f"{inventory} is neither an inventory file, a (list of) hosts or connectors "
+            "nor refers to a python module"
+        )
+        return Inventory.empty()
     else:
         return make_inventory_from_func(inventory_func, override_data)
 
@@ -236,10 +256,10 @@ def make_inventory_from_files(
 
     possible_group_data_folders = []
     if cwd:
-        possible_group_data_folders.append(cwd)
+        possible_group_data_folders.append(path.join(cwd, "group_data"))
     inventory_dirname = path.abspath(path.dirname(inventory_filename))
     if inventory_dirname != cwd:
-        possible_group_data_folders.append(inventory_dirname)
+        possible_group_data_folders.append(path.join(inventory_dirname, "group_data"))
 
     if group_data_directories:
         possible_group_data_folders.extend(group_data_directories)
@@ -250,6 +270,7 @@ def make_inventory_from_files(
         for folder in possible_group_data_folders:
             for group_name, data in _get_group_data(folder).items():
                 group_data[group_name].update(data)
+                logger.debug("Adding data to group %s: %r", group_name, data)
 
     # For each group load up any data
     for name, hosts in groups.items():

pyinfra_cli/log.py

@@ -2,7 +2,8 @@ import logging
 
 import click
 
-from pyinfra import logger
+from pyinfra import logger, state
+from pyinfra.context import ctx_state
 
 
 class LogHandler(logging.Handler):
@@ -41,6 +42,9 @@ class LogFormatter(logging.Formatter):
 
         # We only handle strings here
         if isinstance(message, str):
+            if ctx_state.isset() and record.levelno is logging.WARNING:
+                state.increment_warning_counter()
+
             if "-->" in message:
                 if not self.previous_was_header:
                     click.echo(err=True)
@@ -57,9 +61,13 @@ class LogFormatter(logging.Formatter):
         return super().format(record)
 
 
-def setup_logging(log_level):
+def setup_logging(log_level, other_log_level=None):
+    if other_log_level:
+        logging.basicConfig(level=other_log_level)
+
     logger.setLevel(log_level)
     handler = LogHandler()
     formatter = LogFormatter()
     handler.setFormatter(formatter)
     logger.addHandler(handler)
+    logger.propagate = False

pyinfra_cli/main.py

@@ -13,12 +13,13 @@ from pyinfra.api.connect import connect_all, disconnect_all
 from pyinfra.api.exceptions import NoGroupError, PyinfraError
 from pyinfra.api.facts import get_facts
 from pyinfra.api.operations import run_ops
+from pyinfra.api.state import StateStage
 from pyinfra.api.util import get_kwargs_str
 from pyinfra.context import ctx_config, ctx_inventory, ctx_state
 from pyinfra.operations import server
 
 from .commands import get_facts_and_args, get_func_and_args
-from .exceptions import CliError, UnexpectedExternalError, UnexpectedInternalError
+from .exceptions import CliError, UnexpectedExternalError, UnexpectedInternalError, WrappedError
 from .inventory import make_inventory
 from .log import setup_logging
 from .prints import (
@@ -69,6 +70,8 @@ def _print_support(ctx, param, value):
     is_flag=True,
     default=False,
     help="Execute operations immediately on hosts without prompt or checking for changes.",
+    envvar="PYINFRA_YES",
+    show_envvar=True,
 )
 @click.option(
     "--limit",
@@ -138,11 +141,6 @@ def _print_support(ctx, param, value):
     help="SSH Private key password.",
 )
 @click.option("--ssh-password", "--password", "ssh_password", help="SSH password.")
-# WinRM connector args
-@click.option("--winrm-username", help="WINRM user to connect as.")
-@click.option("--winrm-password", help="WINRM password.")
-@click.option("--winrm-port", help="WINRM port to connect to.")
-@click.option("--winrm-transport", help="WINRM transport for use.")
 # Eager commands (pyinfra --support)
 @click.option(
     "--support",
@@ -153,16 +151,16 @@ def _print_support(ctx, param, value):
 )
 # Debug args
 @click.option(
-    "--quiet",
+    "--debug",
     is_flag=True,
     default=False,
-    help="Hide most pyinfra output.",
+    help="Print debug logs from pyinfra.",
 )
 @click.option(
-    "--debug",
+    "--debug-all",
     is_flag=True,
     default=False,
-    help="Print debug info.",
+    help="Print debug logs from all packages including pyinfra.",
 )
 @click.option(
     "--debug-facts",
@@ -222,24 +220,14 @@ def cli(*args, **kwargs):
 
     try:
         _main(*args, **kwargs)
-
-    except PyinfraError as e:
-        # Re-raise any internal exceptions that aren't handled by click as
-        # CliErrors which are.
-        if not isinstance(e, click.ClickException):
-            message = getattr(e, "message", e.args[0])
-            raise CliError(message)
-
-        raise
-
-    except UnexpectedExternalError:
-        # Pass unexpected external exceptions through as-is
+    except (CliError, UnexpectedExternalError):
         raise
-
+    except PyinfraError as e:
+        # Re-raise "expected" pyinfra exceptions with our click exception wrapper
+        raise WrappedError(e)
     except Exception as e:
         # Re-raise any unexpected internal exceptions as UnexpectedInternalError
         raise UnexpectedInternalError(e)
-
     finally:
         if ctx_state.isset() and state.initialised:
             logger.info("--> Disconnecting from hosts...")
@@ -265,10 +253,6 @@ def _main(
     ssh_key,
     ssh_key_password: str,
     ssh_password: str,
-    winrm_username: str,
-    winrm_password: str,
-    winrm_port,
-    winrm_transport,
     shell_executable,
     sudo: bool,
     sudo_user: str,
@@ -284,8 +268,8 @@ def _main(
     limit: Iterable,
     no_wait: bool,
     serial: bool,
-    quiet: bool,
     debug: bool,
+    debug_all: bool,
     debug_facts: bool,
     debug_operations: bool,
     support: bool = False,
@@ -297,7 +281,7 @@ def _main(
 
     # Setup logging & Bootstrap/Venv
     #
-    _setup_log_level(debug)
+    _setup_log_level(debug, debug_all)
     init_virtualenv()
 
     # Check operations are valid and setup commands
@@ -306,7 +290,7 @@ def _main(
 
     # Setup state, config & inventory
     #
-    state = _setup_state(verbosity, quiet, check_for_changes=not yes)
+    state = _setup_state(verbosity, yes)
     config = Config()
     ctx_config.set(config)
 
@@ -322,7 +306,7 @@ def _main(
         parallel,
         shell_executable,
         fail_percent,
-        quiet,
+        yes,
     )
     override_data = _set_override_data(
         data,
@@ -331,12 +315,11 @@ def _main(
         ssh_key_password,
         ssh_port,
         ssh_password,
-        winrm_username,
-        winrm_password,
-        winrm_port,
-        winrm_transport,
     )
 
+    if yes is False:
+        _set_fail_prompts(state, config)
+
     # Load up the inventory from the filesystem
     #
     logger.info("--> Loading inventory...")
@@ -361,7 +344,11 @@ def _main(
     # Connect to the hosts & start handling the user commands
     #
     logger.info("--> Connecting to hosts...")
+    state.set_stage(StateStage.Connect)
     connect_all(state)
+
+    logger.info("--> Preparing operations...")
+    state.set_stage(StateStage.Prepare)
     can_diff, state, config = _handle_commands(
         state, config, command, original_operations, operations
     )
@@ -374,6 +361,14 @@ def _main(
         else:
             logger.info("--> Detected changes:")
             print_meta(state)
+            click.echo(
+                """
+    Detected changes may not include every change pyinfra will execute.
+    Hidden side effects of operations may alter behaviour of future operations,
+    this will be shown in the results. The remote state will always be updated
+    to reflect the state defined by the input operations.""",
+                err=True,
+            )
 
     # If --debug-facts or --debug-operations, print and exit
     if debug_facts or debug_operations:
@@ -385,41 +380,64 @@ def _main(
     if dry:
         _exit()
 
-    if can_diff and not yes:
-        click.echo(err=True)
-        confirm_msg = "    Operations ready, press enter to execute..."
-        click.echo(confirm_msg, err=True, nl=False)
-        v = input()
-        if v:
-            click.echo(f"Unexpected user input: {v}", err=True)
-            _exit()
-        # Go up, clear the line, go up again - as if the confirmation statement was never here!
-        click.echo(
-            "\033[1A{0}\033[1A".format("".join(" " for _ in range(len(confirm_msg)))),
-            err=True,
-            nl=False,
-        )
+    if (
+        can_diff
+        and not yes
+        and not _do_confirm("Detected changes displayed above, skip this step with -y")
+    ):
+        _exit()
 
     logger.info("--> Beginning operation run...")
-
+    state.set_stage(StateStage.Execute)
     run_ops(state, serial=serial, no_wait=no_wait)
 
     logger.info("--> Results:")
+    state.set_stage(StateStage.Disconnect)
     print_results(state)
     _exit()
 
 
+def _do_confirm(msg: str) -> bool:
+    click.echo(err=True)
+    click.echo(f"    {msg}", err=True)
+    warning_count = state.get_warning_counter()
+    if warning_count > 0:
+        click.secho(
+            f"    {warning_count} warnings shown during change detection, see above",
+            fg="yellow",
+            err=True,
+        )
+    confirm_msg = "    Press enter to execute..."
+    click.echo(confirm_msg, err=True, nl=False)
+    v = input()
+    if v:
+        click.echo(f"    Unexpected user input: {v}", err=True)
+        return False
+    # Go up, clear the line, go up again - as if the confirmation statement was never here!
+    click.echo(
+        "\033[1A{0}\033[1A".format("".join(" " for _ in range(len(confirm_msg)))),
+        err=True,
+        nl=False,
+    )
+    click.echo(err=True)
+    return True
+
+
 # Setup
 #
-def _setup_log_level(debug):
+def _setup_log_level(debug, debug_all):
     if not debug and not sys.warnoptions:
         warnings.simplefilter("ignore")
 
     log_level = logging.INFO
-    if debug:
+    if debug or debug_all:
         log_level = logging.DEBUG
 
-    setup_logging(log_level)
+    other_log_level = None
+    if debug_all:
+        other_log_level = logging.DEBUG
+
+    setup_logging(log_level, other_log_level)
 
 
 def _validate_operations(operations, chdir):
@@ -503,12 +521,12 @@ def _set_verbosity(state, verbosity):
     return state
 
 
-def _setup_state(verbosity, quiet, check_for_changes):
+def _setup_state(verbosity, yes):
     cwd = getcwd()
     if cwd not in sys.path:  # ensure cwd is present in sys.path
         sys.path.append(cwd)
 
-    state = State(check_for_changes=check_for_changes)
+    state = State(check_for_changes=not yes)
     state.cwd = cwd
     ctx_state.set(state)
 
@@ -526,7 +544,7 @@ def _set_config(
     parallel,
     shell_executable,
     fail_percent,
-    quiet,
+    yes,
 ):
     logger.info("--> Loading config...")
 
@@ -571,10 +589,6 @@ def _set_override_data(
     ssh_key_password,
     ssh_port,
     ssh_password,
-    winrm_username,
-    winrm_password,
-    winrm_port,
-    winrm_transport,
 ):
     override_data = {}
 
@@ -590,10 +604,6 @@ def _set_override_data(
         ("ssh_key_password", ssh_key_password),
         ("ssh_port", ssh_port),
         ("ssh_password", ssh_password),
-        ("winrm_username", winrm_username),
-        ("winrm_password", winrm_password),
-        ("winrm_port", winrm_port),
-        ("winrm_transport", winrm_transport),
     ):
         if value:
             override_data[key] = value
@@ -601,6 +611,19 @@ def _set_override_data(
     return override_data
 
 
+def _set_fail_prompts(state: State, config: Config) -> None:
+    # Set fail percent to zero, meaning we'll raise an exception for any fail,
+    # and we can capture + prompt the user to continue/exit.
+    config.FAIL_PERCENT = 0
+
+    def should_raise_failed_hosts(state: State) -> bool:
+        if state.current_stage == StateStage.Connect:
+            return not _do_confirm("One of more hosts failed to connect, continue?")
+        return not _do_confirm("One of more hosts failed, continue?")
+
+    state.should_raise_failed_hosts = should_raise_failed_hosts
+
+
 def _apply_inventory_limit(inventory, limit):
     initial_limit = None
     if limit:

pyinfra_cli/prints.py

@@ -217,28 +217,42 @@ def pretty_op_name(op_meta):
 
 def print_meta(state: "State"):
     rows: List[Tuple[Callable, Union[List[str], str]]] = [
-        (logger.info, ["Operation", "Hosts"]),
+        (logger.info, ["Operation", "Change", "Conditional Change"]),
     ]
 
     for op_hash in state.get_op_order():
-        hosts_in_op = list()
+        hosts_in_op = []
+        hosts_maybe_in_op = []
         for host in state.inventory.iter_activated_hosts():
             if op_hash in state.ops[host]:
-                if state.ops[host][op_hash].operation_meta.changed:
-                    hosts_in_op.append(host.name)
+                op_data = state.get_op_data_for_host(host, op_hash)
+                if op_data.operation_meta._maybe_is_change:
+                    if op_data.global_arguments["_if"]:
+                        hosts_maybe_in_op.append(host.name)
+                    else:
+                        hosts_in_op.append(host.name)
 
         rows.append(
             (
                 logger.info,
                 [
                     pretty_op_name(state.op_meta[op_hash]),
-                    "{0}/{1} ({2})".format(
-                        len(hosts_in_op),
-                        len(state.inventory),
-                        truncate(", ".join(sorted(hosts_in_op)), 48),
-                    )
-                    if hosts_in_op
-                    else "No hosts with changes at this time",
+                    (
+                        "-"
+                        if len(hosts_in_op) == 0
+                        else "{0} ({1})".format(
+                            len(hosts_in_op),
+                            truncate(", ".join(sorted(hosts_in_op)), 48),
+                        )
+                    ),
+                    (
+                        "-"
+                        if len(hosts_maybe_in_op) == 0
+                        else "{0} ({1})".format(
+                            len(hosts_maybe_in_op),
+                            truncate(", ".join(sorted(hosts_maybe_in_op)), 48),
+                        )
+                    ),
                 ],
             )
         )
@@ -253,25 +267,23 @@ def print_results(state: "State"):
 
     for op_hash in state.get_op_order():
         hosts_in_op = 0
-        hosts_in_op_success = list()
-        hosts_in_op_error = list()
-        hosts_in_op_no_attempt = list()
+        hosts_in_op_success: list[str] = []
+        hosts_in_op_error: list[str] = []
+        hosts_in_op_no_change: list[str] = []
         for host in state.inventory.iter_activated_hosts():
             if op_hash not in state.ops[host]:
                 continue
 
             hosts_in_op += 1
 
-            result = state.ops[host][op_hash].operation_meta.success
-            if result is True:
-                hosts_in_op_success.append(host.name)
-            elif result is False:
-                hosts_in_op_error.append(host.name)
+            op_meta = state.ops[host][op_hash].operation_meta
+            if op_meta.did_succeed(_raise_if_not_complete=False):
+                if op_meta.did_change():
+                    hosts_in_op_success.append(host.name)
+                else:
+                    hosts_in_op_no_change.append(host.name)
             else:
-                hosts_in_op_no_attempt.append(host.name)
-
-        # if not hosts_in_op:
-        #     continue
+                hosts_in_op_error.append(host.name)
 
         row = [
             pretty_op_name(state.op_meta[op_hash]),
@@ -286,95 +298,11 @@ def print_results(state: "State"):
             row.append(f"{len(hosts_in_op_error)}")
         else:
             row.append("-")
-        if hosts_in_op_no_attempt:
-            row.append(f"{len(hosts_in_op_no_attempt)}")
+        if hosts_in_op_no_change:
+            row.append(f"{len(hosts_in_op_no_change)}")
         else:
             row.append("-")
 
         rows.append((logger.info, row))
 
     print_rows(rows)
-
-
-def get_fucked(state: "State"):
-    group_combinations = _get_group_combinations(state.inventory.iter_activated_hosts())
-    rows: List[Tuple[Callable, Union[List[str], str]]] = []
-
-    for i, (groups, hosts) in enumerate(group_combinations.items(), 1):
-        if not hosts:
-            continue
-
-        if groups:
-            rows.append(
-                (
-                    logger.info,
-                    "Groups: {0}".format(
-                        click.style(" / ".join(groups), bold=True),
-                    ),
-                ),
-            )
-        else:
-            rows.append((logger.info, "Ungrouped:"))
-
-        for host in hosts:
-            # Didn't connect to this host?
-            if host not in state.activated_hosts:
-                rows.append(
-                    (
-                        logger.info,
-                        [
-                            host.style_print_prefix("red", bold=True),
-                            click.style("No connection", "red"),
-                        ],
-                    ),
-                )
-                continue
-
-            results = state.results[host]
-
-            meta = state.meta[host]
-            success_ops = results.success_ops
-            partial_ops = results.partial_ops
-            # TODO: type meta object
-            changed_ops = success_ops - meta.ops_no_change  # type: ignore
-            error_ops = results.error_ops
-            ignored_error_ops = results.ignored_error_ops
-
-            host_args = ("green",)
-            host_kwargs = {}
-
-            # If all ops got complete
-            if results.ops == meta.ops:
-                # We had some errors - but we ignored them - so "warning" color
-                if error_ops != 0:
-                    host_args = ("yellow",)
-
-            # Ops did not complete!
-            else:
-                host_args = ("red",)
-                host_kwargs["bold"] = True
-
-            changed_str = "Changed: {0}".format(click.style(f"{changed_ops}", bold=True))
-            if partial_ops:
-                changed_str = f"{changed_str} ({partial_ops} partial)"
-
-            error_str = "Errors: {0}".format(click.style(f"{error_ops}", bold=True))
-            if ignored_error_ops:
-                error_str = f"{error_str} ({ignored_error_ops} ignored)"
-
-            rows.append(
-                (
-                    logger.info,
-                    [
-                        host.style_print_prefix(*host_args, **host_kwargs),
-                        changed_str,
-                        "No change: {0}".format(click.style(f"{meta.ops_no_change}", bold=True)),
-                        error_str,
-                    ],
-                ),
-            )
-
-        if i != len(group_combinations):
-            rows.append((lambda m: click.echo(m, err=True), []))
-
-    print_rows(rows)