ansible-core 2.18.7rc1__py3-none-any.whl → 2.19.0__py3-none-any.whl

ansible/inventory/group.py

@@ -16,6 +16,8 @@
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 from __future__ import annotations
 
+import typing as t
+
 from collections.abc import Mapping, MutableMapping
 from enum import Enum
 from itertools import chain
@@ -24,10 +26,15 @@ from ansible import constants as C
 from ansible.errors import AnsibleError
 from ansible.module_utils.common.text.converters import to_native, to_text
 from ansible.utils.display import Display
-from ansible.utils.vars import combine_vars
+from ansible.utils.vars import combine_vars, validate_variable_name
+
+from . import helpers  # this is left as a module import to facilitate easier unit test patching
 
 display = Display()
 
+if t.TYPE_CHECKING:
+    from .host import Host
+
 
 def to_safe_group_name(name, replacer="_", force=False, silent=False):
     # Converts 'bad' characters in a string to underscores (or provided replacer) so they can be used as Ansible hosts or groups
@@ -59,22 +66,23 @@ class InventoryObjectType(Enum):
 
 
 class Group:
-    ''' a group of ansible hosts '''
+    """A group of ansible hosts."""
     base_type = InventoryObjectType.GROUP
 
     # __slots__ = [ 'name', 'hosts', 'vars', 'child_groups', 'parent_groups', 'depth', '_hosts_cache' ]
 
-    def __init__(self, name=None):
+    def __init__(self, name: str) -> None:
+        name = helpers.remove_trust(name)
 
-        self.depth = 0
-        self.name = to_safe_group_name(name)
-        self.hosts = []
-        self._hosts = None
-        self.vars = {}
-        self.child_groups = []
-        self.parent_groups = []
-        self._hosts_cache = None
-        self.priority = 1
+        self.depth: int = 0
+        self.name: str = to_safe_group_name(name)
+        self.hosts: list[Host] = []
+        self._hosts: set[str] | None = None
+        self.vars: dict[str, t.Any] = {}
+        self.child_groups: list[Group] = []
+        self.parent_groups: list[Group] = []
+        self._hosts_cache: list[Host] | None = None
+        self.priority: int = 1
 
     def __repr__(self):
         return self.get_name()
@@ -82,45 +90,8 @@ class Group:
     def __str__(self):
         return self.get_name()
 
-    def __getstate__(self):
-        return self.serialize()
-
-    def __setstate__(self, data):
-        return self.deserialize(data)
-
-    def serialize(self):
-        parent_groups = []
-        for parent in self.parent_groups:
-            parent_groups.append(parent.serialize())
-
-        self._hosts = None
-
-        result = dict(
-            name=self.name,
-            vars=self.vars.copy(),
-            parent_groups=parent_groups,
-            depth=self.depth,
-            hosts=self.hosts,
-        )
-
-        return result
-
-    def deserialize(self, data):
-        self.__init__()  # used by __setstate__ to deserialize in place  # pylint: disable=unnecessary-dunder-call
-        self.name = data.get('name')
-        self.vars = data.get('vars', dict())
-        self.depth = data.get('depth', 0)
-        self.hosts = data.get('hosts', [])
-        self._hosts = None
-
-        parent_groups = data.get('parent_groups', [])
-        for parent_data in parent_groups:
-            g = Group()
-            g.deserialize(parent_data)
-            self.parent_groups.append(g)
-
-    def _walk_relationship(self, rel, include_self=False, preserve_ordering=False):
-        '''
+    def _walk_relationship(self, rel, include_self=False, preserve_ordering=False) -> set[Group] | list[Group]:
+        """
         Given `rel` that is an iterable property of Group,
         consitituting a directed acyclic graph among all groups,
         Returns a set of all groups in full tree
@@ -132,13 +103,13 @@ class Group:
         | /     are directed upward
         F
         Called on F, returns set of (A, B, C, D, E)
-        '''
-        seen = set([])
+        """
+        seen: set[Group] = set([])
         unprocessed = set(getattr(self, rel))
         if include_self:
             unprocessed.add(self)
         if preserve_ordering:
-            ordered = [self] if include_self else []
+            ordered: list[Group] = [self] if include_self else []
             ordered.extend(getattr(self, rel))
 
         while unprocessed:
@@ -158,22 +129,22 @@ class Group:
             return ordered
         return seen
 
-    def get_ancestors(self):
-        return self._walk_relationship('parent_groups')
+    def get_ancestors(self) -> set[Group]:
+        return t.cast(set, self._walk_relationship('parent_groups'))
 
-    def get_descendants(self, **kwargs):
+    def get_descendants(self, **kwargs) -> set[Group] | list[Group]:
         return self._walk_relationship('child_groups', **kwargs)
 
     @property
-    def host_names(self):
+    def host_names(self) -> set[str]:
         if self._hosts is None:
-            self._hosts = set(self.hosts)
+            self._hosts = {h.name for h in self.hosts}
         return self._hosts
 
-    def get_name(self):
+    def get_name(self) -> str:
         return self.name
 
-    def add_child_group(self, group):
+    def add_child_group(self, group: Group) -> bool:
         added = False
         if self == group:
             raise Exception("can't add group to itself")
@@ -208,7 +179,7 @@ class Group:
             self.clear_hosts_cache()
         return added
 
-    def _check_children_depth(self):
+    def _check_children_depth(self) -> None:
 
         depth = self.depth
         start_depth = self.depth  # self.depth could change over loop
@@ -227,7 +198,7 @@ class Group:
             if depth - start_depth > len(seen):
                 raise AnsibleError("The group named '%s' has a recursive dependency loop." % to_native(self.name))
 
-    def add_host(self, host):
+    def add_host(self, host: Host) -> bool:
         added = False
         if host.name not in self.host_names:
             self.hosts.append(host)
@@ -237,7 +208,7 @@ class Group:
             added = True
         return added
 
-    def remove_host(self, host):
+    def remove_host(self, host: Host) -> bool:
         removed = False
         if host.name in self.host_names:
             self.hosts.remove(host)
@@ -247,7 +218,13 @@ class Group:
             removed = True
         return removed
 
-    def set_variable(self, key, value):
+    def set_variable(self, key: str, value: t.Any) -> None:
+        key = helpers.remove_trust(key)
+
+        try:
+            validate_variable_name(key)
+        except AnsibleError as ex:
+            Display().deprecated(msg=f'Accepting inventory variable with invalid name {key!r}.', version='2.23', help_text=ex._help_text, obj=ex.obj)
 
         if key == 'ansible_group_priority':
             self.set_priority(int(value))
@@ -257,36 +234,36 @@ class Group:
             else:
                 self.vars[key] = value
 
-    def clear_hosts_cache(self):
+    def clear_hosts_cache(self) -> None:
 
         self._hosts_cache = None
         for g in self.get_ancestors():
             g._hosts_cache = None
 
-    def get_hosts(self):
+    def get_hosts(self) -> list[Host]:
 
         if self._hosts_cache is None:
             self._hosts_cache = self._get_hosts()
         return self._hosts_cache
 
-    def _get_hosts(self):
+    def _get_hosts(self) -> list[Host]:
 
-        hosts = []
-        seen = {}
+        hosts: list[Host] = []
+        seen: set[Host] = set()
         for kid in self.get_descendants(include_self=True, preserve_ordering=True):
             kid_hosts = kid.hosts
             for kk in kid_hosts:
                 if kk not in seen:
-                    seen[kk] = 1
+                    seen.add(kk)
                     if self.name == 'all' and kk.implicit:
                         continue
                     hosts.append(kk)
         return hosts
 
-    def get_vars(self):
+    def get_vars(self) -> dict[str, t.Any]:
         return self.vars.copy()
 
-    def set_priority(self, priority):
+    def set_priority(self, priority: int | str) -> None:
         try:
             self.priority = int(priority)
         except TypeError:

ansible/inventory/helpers.py

@@ -18,6 +18,7 @@
 #############################################
 from __future__ import annotations
 
+from ansible._internal._datatag._tags import TrustedAsTemplate
 from ansible.utils.vars import combine_vars
 
 
@@ -37,3 +38,11 @@ def get_group_vars(groups):
         results = combine_vars(results, group.get_vars())
 
     return results
+
+
+def remove_trust(value: str) -> str:
+    """
+    Remove trust from strings which should not be trusted.
+    This exists to centralize the untagging call which facilitate patching it out in unit tests.
+    """
+    return TrustedAsTemplate.untag(value)

ansible/inventory/host.py

@@ -17,28 +17,28 @@
 
 from __future__ import annotations
 
+import collections.abc as c
+import typing as t
+
 from collections.abc import Mapping, MutableMapping
 
+from ansible.errors import AnsibleError
 from ansible.inventory.group import Group, InventoryObjectType
 from ansible.parsing.utils.addresses import patterns
-from ansible.utils.vars import combine_vars, get_unique_id
+from ansible.utils.display import Display
+from ansible.utils.vars import combine_vars, get_unique_id, validate_variable_name
 
+from . import helpers  # this is left as a module import to facilitate easier unit test patching
 
 __all__ = ['Host']
 
 
 class Host:
-    ''' a single ansible host '''
+    """A single ansible host."""
     base_type = InventoryObjectType.HOST
 
     # __slots__ = [ 'name', 'vars', 'groups' ]
 
-    def __getstate__(self):
-        return self.serialize()
-
-    def __setstate__(self, data):
-        return self.deserialize(data)
-
     def __eq__(self, other):
         if not isinstance(other, Host):
             return False
@@ -56,55 +56,28 @@ class Host:
     def __repr__(self):
         return self.get_name()
 
-    def serialize(self):
-        groups = []
-        for group in self.groups:
-            groups.append(group.serialize())
-
-        return dict(
-            name=self.name,
-            vars=self.vars.copy(),
-            address=self.address,
-            uuid=self._uuid,
-            groups=groups,
-            implicit=self.implicit,
-        )
-
-    def deserialize(self, data):
-        self.__init__(gen_uuid=False)  # used by __setstate__ to deserialize in place  # pylint: disable=unnecessary-dunder-call
-
-        self.name = data.get('name')
-        self.vars = data.get('vars', dict())
-        self.address = data.get('address', '')
-        self._uuid = data.get('uuid', None)
-        self.implicit = data.get('implicit', False)
-
-        groups = data.get('groups', [])
-        for group_data in groups:
-            g = Group()
-            g.deserialize(group_data)
-            self.groups.append(g)
+    def __init__(self, name: str, port: int | str | None = None, gen_uuid: bool = True) -> None:
+        name = helpers.remove_trust(name)
 
-    def __init__(self, name=None, port=None, gen_uuid=True):
+        self.vars: dict[str, t.Any] = {}
+        self.groups: list[Group] = []
+        self._uuid: str | None = None
 
-        self.vars = {}
-        self.groups = []
-        self._uuid = None
-
-        self.name = name
-        self.address = name
+        self.name: str = name
+        self.address: str = name
 
         if port:
             self.set_variable('ansible_port', int(port))
 
         if gen_uuid:
             self._uuid = get_unique_id()
-        self.implicit = False
 
-    def get_name(self):
+        self.implicit: bool = False
+
+    def get_name(self) -> str:
         return self.name
 
-    def populate_ancestors(self, additions=None):
+    def populate_ancestors(self, additions: c.Iterable[Group] | None = None) -> None:
         # populate ancestors
         if additions is None:
             for group in self.groups:
@@ -114,7 +87,7 @@ class Host:
                 if group not in self.groups:
                     self.groups.append(group)
 
-    def add_group(self, group):
+    def add_group(self, group: Group) -> bool:
         added = False
         # populate ancestors first
         for oldg in group.get_ancestors():
@@ -127,7 +100,7 @@ class Host:
             added = True
         return added
 
-    def remove_group(self, group):
+    def remove_group(self, group: Group) -> bool:
         removed = False
         if group in self.groups:
             self.groups.remove(group)
@@ -143,18 +116,28 @@ class Host:
                         self.remove_group(oldg)
         return removed
 
-    def set_variable(self, key, value):
+    def set_variable(self, key: str, value: t.Any) -> None:
+        key = helpers.remove_trust(key)
+
+        try:
+            validate_variable_name(key)
+        except AnsibleError as ex:
+            Display().deprecated(msg=f'Accepting inventory variable with invalid name {key!r}.', version='2.23', help_text=ex._help_text, obj=ex.obj)
+
         if key in self.vars and isinstance(self.vars[key], MutableMapping) and isinstance(value, Mapping):
             self.vars = combine_vars(self.vars, {key: value})
         else:
             self.vars[key] = value
 
-    def get_groups(self):
+    def get_groups(self) -> list[Group]:
         return self.groups
 
-    def get_magic_vars(self):
-        results = {}
-        results['inventory_hostname'] = self.name
+    def get_magic_vars(self) -> dict[str, t.Any]:
+        results: dict[str, t.Any] = dict(
+            inventory_hostname=self.name,
+        )
+
+        # FUTURE: these values should be dynamically calculated on access ala the rest of magic vars
         if patterns['ipv4'].match(self.name) or patterns['ipv6'].match(self.name):
             results['inventory_hostname_short'] = self.name
         else:
@@ -164,5 +147,5 @@ class Host:
 
         return results
 
-    def get_vars(self):
+    def get_vars(self) -> dict[str, t.Any]:
         return combine_vars(self.vars, self.get_magic_vars())

ansible/inventory/manager.py

@@ -19,28 +19,34 @@
 from __future__ import annotations
 
 import fnmatch
+import functools
 import os
-import sys
 import re
 import itertools
-import traceback
+import typing as t
 
 from operator import attrgetter
 from random import shuffle
 
 from ansible import constants as C
-from ansible.errors import AnsibleError, AnsibleOptionsError, AnsibleParserError
+from ansible._internal import _json, _wrapt
+from ansible._internal._json import EncryptedStringBehavior
+from ansible.errors import AnsibleError, AnsibleOptionsError
 from ansible.inventory.data import InventoryData
 from ansible.module_utils.six import string_types
 from ansible.module_utils.common.text.converters import to_bytes, to_text
 from ansible.parsing.utils.addresses import parse_address
 from ansible.plugins.loader import inventory_loader
+from ansible._internal._datatag._tags import Origin
 from ansible.utils.helpers import deduplicate_list
 from ansible.utils.path import unfrackpath
 from ansible.utils.display import Display
 from ansible.utils.vars import combine_vars
 from ansible.vars.plugins import get_vars_from_inventory_sources
 
+if t.TYPE_CHECKING:
+    from ansible.plugins.inventory import BaseInventoryPlugin
+
 display = Display()
 
 IGNORED_ALWAYS = [br"^\.", b"^host_vars$", b"^group_vars$", b"^vars_plugins$"]
@@ -50,7 +56,7 @@ IGNORED_EXTS = [b'%s$' % to_bytes(re.escape(x)) for x in C.INVENTORY_IGNORE_EXTS
 IGNORED = re.compile(b'|'.join(IGNORED_ALWAYS + IGNORED_PATTERNS + IGNORED_EXTS))
 
 PATTERN_WITH_SUBSCRIPT = re.compile(
-    r'''^
+    r"""^
         (.+)                    # A pattern expression ending with...
         \[(?:                   # A [subscript] expression comprising:
             (-?[0-9]+)|         # A single positive or negative number
@@ -58,12 +64,12 @@ PATTERN_WITH_SUBSCRIPT = re.compile(
             ([0-9]*)
         )\]
         $
-    ''', re.X
+    """, re.X
 )
 
 
 def order_patterns(patterns):
-    ''' takes a list of patterns and reorders them by modifier to apply them consistently '''
+    """ takes a list of patterns and reorders them by modifier to apply them consistently """
 
     # FIXME: this goes away if we apply patterns incrementally or by groups
     pattern_regular = []
@@ -125,19 +131,19 @@ def split_host_pattern(pattern):
             # This mishandles IPv6 addresses, and is retained only for backwards
             # compatibility.
             patterns = re.findall(
-                to_text(r'''(?:     # We want to match something comprising:
+                to_text(r"""(?:     # We want to match something comprising:
                         [^\s:\[\]]  # (anything other than whitespace or ':[]'
                         |           # ...or...
                         \[[^\]]*\]  # a single complete bracketed expression)
                     )+              # occurring once or more
-                '''), pattern, re.X
+                """), pattern, re.X
             )
 
     return [p.strip() for p in patterns if p.strip()]
 
 
 class InventoryManager(object):
-    ''' Creates and manages inventory '''
+    """ Creates and manages inventory """
 
     def __init__(self, loader, sources=None, parse=True, cache=True):
 
@@ -196,12 +202,12 @@ class InventoryManager(object):
     def get_host(self, hostname):
         return self._inventory.get_host(hostname)
 
-    def _fetch_inventory_plugins(self):
-        ''' sets up loaded inventory plugins for usage '''
+    def _fetch_inventory_plugins(self) -> list[BaseInventoryPlugin]:
+        """ sets up loaded inventory plugins for usage """
 
         display.vvvv('setting up inventory plugins')
 
-        plugins = []
+        plugins: list[BaseInventoryPlugin] = []
         for name in C.INVENTORY_ENABLED:
             plugin = inventory_loader.get(name)
             if plugin:
@@ -215,7 +221,7 @@ class InventoryManager(object):
         return plugins
 
     def parse_sources(self, cache=False):
-        ''' iterate over inventory sources and parse each one to populate it'''
+        """ iterate over inventory sources and parse each one to populate it"""
 
         parsed = False
         # allow for multiple inventory parsing
@@ -243,7 +249,7 @@ class InventoryManager(object):
             host.vars = combine_vars(host.vars, get_vars_from_inventory_sources(self._loader, self._sources, [host], 'inventory'))
 
     def parse_source(self, source, cache=False):
-        ''' Generate or update inventory for the source provided '''
+        """ Generate or update inventory for the source provided """
 
         parsed = False
         failures = []
@@ -276,7 +282,6 @@ class InventoryManager(object):
 
             # try source with each plugin
             for plugin in self._fetch_inventory_plugins():
-
                 plugin_name = to_text(getattr(plugin, '_load_name', getattr(plugin, '_original_path', '')))
                 display.debug(u'Attempting to use plugin %s (%s)' % (plugin_name, plugin._original_path))
 
@@ -287,9 +292,14 @@ class InventoryManager(object):
                     plugin_wants = False
 
                 if plugin_wants:
+                    # have this tag ready to apply to errors or output; str-ify source since it is often tagged by the CLI
+                    origin = Origin(description=f'<inventory plugin {plugin_name!r} with source {str(source)!r}>')
                     try:
-                        # FIXME in case plugin fails 1/2 way we have partial inventory
-                        plugin.parse(self._inventory, self._loader, source, cache=cache)
+                        inventory_wrapper = _InventoryDataWrapper(self._inventory, target_plugin=plugin, origin=origin)
+
+                        # FUTURE: now that we have a wrapper around inventory, we can have it use ChainMaps to preview the in-progress inventory,
+                        #  but be able to roll back partial inventory failures by discarding the outermost layer
+                        plugin.parse(inventory_wrapper, self._loader, source, cache=cache)
                         try:
                             plugin.update_cache_if_changed()
                         except AttributeError:
@@ -298,14 +308,17 @@ class InventoryManager(object):
                         parsed = True
                         display.vvv('Parsed %s inventory source with %s plugin' % (source, plugin_name))
                         break
-                    except AnsibleParserError as e:
-                        display.debug('%s was not parsable by %s' % (source, plugin_name))
-                        tb = ''.join(traceback.format_tb(sys.exc_info()[2]))
-                        failures.append({'src': source, 'plugin': plugin_name, 'exc': e, 'tb': tb})
-                    except Exception as e:
-                        display.debug('%s failed while attempting to parse %s' % (plugin_name, source))
-                        tb = ''.join(traceback.format_tb(sys.exc_info()[2]))
-                        failures.append({'src': source, 'plugin': plugin_name, 'exc': AnsibleError(e), 'tb': tb})
+                    except AnsibleError as ex:
+                        if not ex.obj:
+                            ex.obj = origin
+                        failures.append({'src': source, 'plugin': plugin_name, 'exc': ex})
+                    except Exception as ex:
+                        # DTFIX-FUTURE: fix this error handling to correctly deal with messaging
+                        try:
+                            # omit line number to prevent contextual display of script or possibly sensitive info
+                            raise AnsibleError(str(ex), obj=origin) from ex
+                        except AnsibleError as ex:
+                            failures.append({'src': source, 'plugin': plugin_name, 'exc': ex})
                 else:
                     display.vvv("%s declined parsing %s as it did not pass its verify_file() method" % (plugin_name, source))
 
@@ -319,9 +332,8 @@ class InventoryManager(object):
                 if failures:
                     # only if no plugin processed files should we show errors.
                     for fail in failures:
-                        display.warning(u'\n* Failed to parse %s with %s plugin: %s' % (to_text(fail['src']), fail['plugin'], to_text(fail['exc'])))
-                        if 'tb' in fail:
-                            display.vvv(to_text(fail['tb']))
+                        # `obj` should always be set
+                        display.error_as_warning(msg=f'Failed to parse inventory with {fail["plugin"]!r} plugin.', exception=fail['exc'])
 
                 # final error/warning on inventory source failure
                 if C.INVENTORY_ANY_UNPARSED_IS_FAILED:
@@ -335,12 +347,12 @@ class InventoryManager(object):
         return parsed
 
     def clear_caches(self):
-        ''' clear all caches '''
+        """ clear all caches """
         self._hosts_patterns_cache = {}
         self._pattern_cache = {}
 
     def refresh_inventory(self):
-        ''' recalculate inventory '''
+        """ recalculate inventory """
 
         self.clear_caches()
         self._inventory = InventoryData()
@@ -657,9 +669,9 @@ class InventoryManager(object):
         self._pattern_cache = {}
 
     def add_dynamic_host(self, host_info, result_item):
-        '''
+        """
         Helper function to add a new host to inventory based on a task result.
-        '''
+        """
 
         changed = False
         if not result_item.get('refresh'):
@@ -697,10 +709,10 @@ class InventoryManager(object):
             result_item['changed'] = changed
 
     def add_dynamic_group(self, host, result_item):
-        '''
+        """
         Helper function to add a group (if it does not exist), and to assign the
         specified host to that group.
-        '''
+        """
 
         changed = False
 
@@ -749,3 +761,36 @@ class InventoryManager(object):
             self.reconcile_inventory()
 
         result_item['changed'] = changed
+
+
+class _InventoryDataWrapper(_wrapt.ObjectProxy):
+    """
+    Proxy wrapper around InventoryData.
+    Allows `set_variable` calls to automatically apply template trust for plugins that don't know how.
+    """
+
+    # declared as class attrs to signal to ObjectProxy that we want them stored on the proxy, not the wrapped value
+    _target_plugin = None
+    _default_origin = None
+
+    def __init__(self, referent: InventoryData, target_plugin: BaseInventoryPlugin, origin: Origin) -> None:
+        super().__init__(referent)
+        self._target_plugin = target_plugin
+        # fallback origin to ensure that vars are tagged with at least the file they came from
+        self._default_origin = origin
+
+    @functools.cached_property
+    def _inspector(self) -> _json.AnsibleVariableVisitor:
+        """
+        Inventory plugins can delegate to other plugins (e.g. `auto`).
+        This hack defers sampling the target plugin's `trusted_by_default` attr until `set_variable` is called, typically inside `parse`.
+        Trust is then optionally applied based on the plugin's declared intent via `trusted_by_default`.
+        """
+        return _json.AnsibleVariableVisitor(
+            trusted_as_template=self._target_plugin.trusted_by_default,
+            origin=self._default_origin,
+            encrypted_string_behavior=EncryptedStringBehavior.PRESERVE,
+        )
+
+    def set_variable(self, entity: str, varname: str, value: t.Any) -> None:
+        self.__wrapped__.set_variable(entity, varname, self._inspector.visit(value))