ansible-core 2.18.5rc1__py3-none-any.whl → 2.19.0b2__py3-none-any.whl

ansible/plugins/inventory/script.py

@@ -29,6 +29,8 @@ DOCUMENTATION = '''
         - The plugin does not cache results because external inventory scripts are responsible for their own caching.
         - To write your own inventory script see (R(Developing dynamic inventory,developing_inventory) from the documentation site.
         - To find the scripts that used to be part of the code release, go to U(https://github.com/ansible-community/contrib-scripts/).
+        - Since 2.19 using a directory as an inventory source will no longer ignore .ini files by default,
+          but you can still update the configuration to do so.
 '''
 
 EXAMPLES = r'''# fmt: code
@@ -151,148 +153,136 @@ EXAMPLES = r'''# fmt: code
 '''
 
 
+import json
 import os
+import shlex
 import subprocess
+import typing as t
 
-from collections.abc import Mapping
-
-from ansible.errors import AnsibleError, AnsibleParserError
-from ansible.module_utils.basic import json_dict_bytes_to_unicode
-from ansible.module_utils.common.text.converters import to_native, to_text
+from ansible.errors import AnsibleError, AnsibleJSONParserError
+from ansible.inventory.data import InventoryData
+from ansible.module_utils.datatag import native_type_name
+from ansible.module_utils.common.json import get_decoder
+from ansible.parsing.dataloader import DataLoader
 from ansible.plugins.inventory import BaseInventoryPlugin
+from ansible._internal._datatag._tags import TrustedAsTemplate, Origin
 from ansible.utils.display import Display
+from ansible._internal._json._profiles import _legacy, _inventory_legacy
 
 display = Display()
 
 
 class InventoryModule(BaseInventoryPlugin):
-    ''' Host inventory parser for ansible using external inventory scripts. '''
+    """Host inventory parser for ansible using external inventory scripts."""
 
     NAME = 'script'
 
-    def __init__(self):
-
+    def __init__(self) -> None:
         super(InventoryModule, self).__init__()
 
-        self._hosts = set()
-
-    def verify_file(self, path):
-        ''' Verify if file is usable by this plugin, base does minimal accessibility check '''
-
-        valid = super(InventoryModule, self).verify_file(path)
-
-        if valid:
-            # not only accessible, file must be executable and/or have shebang
-            shebang_present = False
-            try:
-                with open(path, 'rb') as inv_file:
-                    initial_chars = inv_file.read(2)
-                    if initial_chars.startswith(b'#!'):
-                        shebang_present = True
-            except Exception:
-                pass
+        self._hosts: set[str] = set()
 
-            if not os.access(path, os.X_OK) and not shebang_present:
-                valid = False
-
-        return valid
-
-    def parse(self, inventory, loader, path, cache=None):
+    def verify_file(self, path: str) -> bool:
+        return super(InventoryModule, self).verify_file(path) and os.access(path, os.X_OK)
 
+    def parse(self, inventory: InventoryData, loader: DataLoader, path: str, cache: bool = False) -> None:
         super(InventoryModule, self).parse(inventory, loader, path)
-        self.set_options()
 
-        # Support inventory scripts that are not prefixed with some
-        # path information but happen to be in the current working
-        # directory when '.' is not in PATH.
-        cmd = [path, "--list"]
-
-        try:
-            try:
-                sp = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-            except OSError as e:
-                raise AnsibleParserError("problem running %s (%s)" % (' '.join(cmd), to_native(e)))
-            (stdout, stderr) = sp.communicate()
+        self.set_options()
 
-            path = to_native(path)
-            err = to_native(stderr or "")
+        origin = Origin(description=f'<inventory script output from {path!r}>')
 
-            if err and not err.endswith('\n'):
-                err += '\n'
+        data, stderr, stderr_help_text = run_command(path, ['--list'], origin)
 
-            if sp.returncode != 0:
-                raise AnsibleError("Inventory script (%s) had an execution error: %s " % (path, err))
+        try:
+            profile_name = detect_profile_name(data)
+            decoder = get_decoder(profile_name)
+        except Exception as ex:
+            raise AnsibleError(
+                message="Unable to get JSON decoder for inventory script result.",
+                help_text=stderr_help_text,
+                # obj will be added by inventory manager
+            ) from ex
 
-            # make sure script output is unicode so that json loader will output unicode strings itself
+        try:
             try:
-                data = to_text(stdout, errors="strict")
-            except Exception as e:
-                raise AnsibleError("Inventory {0} contained characters that cannot be interpreted as UTF-8: {1}".format(path, to_native(e)))
+                processed = json.loads(data, cls=decoder)
+            except Exception as json_ex:
+                AnsibleJSONParserError.handle_exception(json_ex, origin)
+        except Exception as ex:
+            raise AnsibleError(
+                message="Inventory script result could not be parsed as JSON.",
+                help_text=stderr_help_text,
+                # obj will be added by inventory manager
+            ) from ex
+
+        # if no other errors happened, and you want to force displaying stderr, do so now
+        if stderr and self.get_option('always_show_stderr'):
+            self.display.error(msg=stderr)
+
+        data_from_meta: dict | None = None
+
+        # A "_meta" subelement may contain a variable "hostvars" which contains a hash for each host
+        # if this "hostvars" exists at all then do not call --host for each # host.
+        # This is for efficiency and scripts should still return data
+        # if called with --host for backwards compat with 1.2 and earlier.
+        for (group, gdata) in processed.items():
+            if group == '_meta':
+                data_from_meta = gdata.get('hostvars')
+
+                if not isinstance(data_from_meta, dict):
+                    raise TypeError(f"Value contains '_meta.hostvars' which is {native_type_name(data_from_meta)!r} instead of {native_type_name(dict)!r}.")
+            else:
+                self._parse_group(group, gdata, origin)
+
+        if data_from_meta is None:
+            display.deprecated(
+                msg="Inventory scripts should always provide 'meta.hostvars'. "
+                    "Host variables will be collected by running the inventory script with the '--host' option for each host.",
+                version='2.23',
+                obj=origin,
+            )
+
+        for host in self._hosts:
+            if data_from_meta is None:
+                got = self.get_host_variables(path, host, origin)
+            else:
+                got = data_from_meta.get(host, {})
 
-            try:
-                processed = self.loader.load(data, json_only=True)
-            except Exception as e:
-                raise AnsibleError("failed to parse executable inventory script results from {0}: {1}\n{2}".format(path, to_native(e), err))
-
-            # if no other errors happened and you want to force displaying stderr, do so now
-            if stderr and self.get_option('always_show_stderr'):
-                self.display.error(msg=to_text(err))
-
-            if not isinstance(processed, Mapping):
-                raise AnsibleError("failed to parse executable inventory script results from {0}: needs to be a json dict\n{1}".format(path, err))
-
-            group = None
-            data_from_meta = None
-
-            # A "_meta" subelement may contain a variable "hostvars" which contains a hash for each host
-            # if this "hostvars" exists at all then do not call --host for each # host.
-            # This is for efficiency and scripts should still return data
-            # if called with --host for backwards compat with 1.2 and earlier.
-            for (group, gdata) in processed.items():
-                if group == '_meta':
-                    if 'hostvars' in gdata:
-                        data_from_meta = gdata['hostvars']
-                else:
-                    self._parse_group(group, gdata)
-
-            for host in self._hosts:
-                got = {}
-                if data_from_meta is None:
-                    got = self.get_host_variables(path, host)
-                else:
-                    try:
-                        got = data_from_meta.get(host, {})
-                    except AttributeError as e:
-                        raise AnsibleError("Improperly formatted host information for %s: %s" % (host, to_native(e)), orig_exc=e)
-
-                self._populate_host_vars([host], got)
-
-        except Exception as e:
-            raise AnsibleParserError(to_native(e))
-
-    def _parse_group(self, group, data):
+            self._populate_host_vars([host], got)
 
+    def _parse_group(self, group: str, data: t.Any, origin: Origin) -> None:
+        """Normalize and ingest host/var information for the named group."""
         group = self.inventory.add_group(group)
 
         if not isinstance(data, dict):
             data = {'hosts': data}
-        # is not those subkeys, then simplified syntax, host with vars
+            display.deprecated(
+                msg=f"Group {group!r} was converted to {native_type_name(dict)!r} from {native_type_name(data)!r}.",
+                version='2.23',
+                obj=origin,
+            )
         elif not any(k in data for k in ('hosts', 'vars', 'children')):
             data = {'hosts': [group], 'vars': data}
+            display.deprecated(
+                msg=f"Treating malformed group {group!r} as the sole host of that group. Variables provided in this manner cannot be templated.",
+                version='2.23',
+                obj=origin,
+            )
 
-        if 'hosts' in data:
-            if not isinstance(data['hosts'], list):
-                raise AnsibleError("You defined a group '%s' with bad data for the host list:\n %s" % (group, data))
+        if (data_hosts := data.get('hosts', ...)) is not ...:
+            if not isinstance(data_hosts, list):
+                raise TypeError(f"Value contains '{group}.hosts' which is {native_type_name(data_hosts)!r} instead of {native_type_name(list)!r}.")
 
-            for hostname in data['hosts']:
+            for hostname in data_hosts:
                 self._hosts.add(hostname)
                 self.inventory.add_host(hostname, group)
 
-        if 'vars' in data:
-            if not isinstance(data['vars'], dict):
-                raise AnsibleError("You defined a group '%s' with bad data for variables:\n %s" % (group, data))
+        if (data_vars := data.get('vars', ...)) is not ...:
+            if not isinstance(data_vars, dict):
+                raise TypeError(f"Value contains '{group}.vars' which is {native_type_name(data_vars)!r} instead of {native_type_name(dict)!r}.")
 
-            for k, v in data['vars'].items():
+            for k, v in data_vars.items():
                 self.inventory.set_variable(group, k, v)
 
         if group != '_meta' and isinstance(data, dict) and 'children' in data:
@@ -300,22 +290,102 @@ class InventoryModule(BaseInventoryPlugin):
                 child_name = self.inventory.add_group(child_name)
                 self.inventory.add_child(group, child_name)
 
-    def get_host_variables(self, path, host):
-        """ Runs <script> --host <hostname>, to determine additional host variables """
+    @staticmethod
+    def get_host_variables(path: str, host: str, origin: Origin) -> dict:
+        """Runs <script> --host <hostname>, to determine additional host variables."""
+        origin = origin.replace(description=f'{origin.description} for host {host!r}')
 
-        cmd = [path, "--host", host]
-        try:
-            sp = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        except OSError as e:
-            raise AnsibleError("problem running %s (%s)" % (' '.join(cmd), e))
-        (out, stderr) = sp.communicate()
+        data, stderr, stderr_help_text = run_command(path, ['--host', host], origin)
 
-        if sp.returncode != 0:
-            raise AnsibleError("Inventory script (%s) had an execution error: %s" % (path, to_native(stderr)))
-
-        if out.strip() == '':
+        if not data.strip():
             return {}
+
         try:
-            return json_dict_bytes_to_unicode(self.loader.load(out, file_name=path))
-        except ValueError:
-            raise AnsibleError("could not parse post variable response: %s, %s" % (cmd, out))
+            try:
+                # Use standard legacy trust inversion here.
+                # Unlike the normal inventory output, everything here is considered a variable and thus supports trust (and trust inversion).
+                processed = json.loads(data, cls=_legacy.Decoder)
+            except Exception as json_ex:
+                AnsibleJSONParserError.handle_exception(json_ex, origin)
+        except Exception as ex:
+            raise AnsibleError(
+                message=f"Inventory script result for host {host!r} could not be parsed as JSON.",
+                help_text=stderr_help_text,
+                # obj will be added by inventory manager
+            ) from ex
+
+        return processed
+
+
+def detect_profile_name(value: str) -> str:
+    """
+    Detect (optional) JSON profile name from an inventory JSON document.
+    Defaults to `inventory_legacy`.
+    """
+    try:
+        data = json.loads(value)
+    except Exception as ex:
+        raise ValueError('Value could not be parsed as JSON.') from ex
+
+    if not isinstance(data, dict):
+        raise TypeError(f'Value is {native_type_name(data)!r} instead of {native_type_name(dict)!r}.')
+
+    if (meta := data.get('_meta', ...)) is ...:
+        return _inventory_legacy.Decoder.profile_name
+
+    if not isinstance(meta, dict):
+        raise TypeError(f"Value contains '_meta' which is {native_type_name(meta)!r} instead of {native_type_name(dict)!r}.")
+
+    if (profile := meta.get('profile', ...)) is ...:
+        return _inventory_legacy.Decoder.profile_name
+
+    if not isinstance(profile, str):
+        raise TypeError(f"Value contains '_meta.profile' which is {native_type_name(profile)!r} instead of {native_type_name(str)!r}.")
+
+    if not profile.startswith('inventory_'):
+        raise ValueError(f"Non-inventory profile {profile!r} is not allowed.")
+
+    return profile
+
+
+def run_command(path: str, options: list[str], origin: Origin) -> tuple[str, str, str]:
+    """Run an inventory script, normalize and validate output."""
+    cmd = [path] + options
+
+    try:
+        sp = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    except OSError as ex:
+        raise AnsibleError(
+            message=f"Failed to execute inventory script command {shlex.join(cmd)!r}.",
+            # obj will be added by inventory manager
+        ) from ex
+
+    stdout_bytes, stderr_bytes = sp.communicate()
+
+    stderr = stderr_bytes.decode(errors='surrogateescape')
+
+    if stderr and not stderr.endswith('\n'):
+        stderr += '\n'
+
+    # DTFIX-RELEASE: another use case for the "not quite help text, definitely not message" diagnostic output on errors
+    stderr_help_text = f'Standard error from inventory script:\n{stderr}' if stderr.strip() else None
+
+    if sp.returncode != 0:
+        raise AnsibleError(
+            message=f"Inventory script returned non-zero exit code {sp.returncode}.",
+            help_text=stderr_help_text,
+            # obj will be added by inventory manager
+        )
+
+    try:
+        data = stdout_bytes.decode()
+    except Exception as ex:
+        raise AnsibleError(
+            "Inventory script result contained characters that cannot be interpreted as UTF-8.",
+            help_text=stderr_help_text,
+            # obj will be added by inventory manager
+        ) from ex
+    else:
+        data = TrustedAsTemplate().tag(origin.tag(data))
+
+    return data, stderr, stderr_help_text

ansible/plugins/inventory/toml.py

@@ -3,19 +3,16 @@
 
 from __future__ import annotations
 
-DOCUMENTATION = r'''
+DOCUMENTATION = r"""
     name: toml
     version_added: "2.8"
     short_description: Uses a specific TOML file as an inventory source.
     description:
         - TOML based inventory format
         - File MUST have a valid '.toml' file extension
-    notes:
-        - >
-          Requires one of the following python libraries: 'toml', 'tomli', or 'tomllib'
-'''
+"""
 
-EXAMPLES = r'''# fmt: toml
+EXAMPLES = r"""# fmt: toml
 # Example 1
 [all.vars]
 has_java = false
@@ -84,115 +81,27 @@ host4 = {}
 
 [g2.hosts]
 host4 = {}
-'''
+"""
 
 import os
-import typing as t
+import tomllib
 
 from collections.abc import MutableMapping, MutableSequence
-from functools import partial
 
-from ansible.errors import AnsibleFileNotFound, AnsibleParserError, AnsibleRuntimeError
-from ansible.module_utils.common.text.converters import to_bytes, to_native, to_text
-from ansible.module_utils.six import string_types, text_type
-from ansible.parsing.yaml.objects import AnsibleSequence, AnsibleUnicode
+from ansible.errors import AnsibleFileNotFound, AnsibleParserError
+from ansible.module_utils.common.text.converters import to_bytes, to_native
+from ansible.module_utils.six import string_types
 from ansible.plugins.inventory import BaseFileInventoryPlugin
 from ansible.utils.display import Display
-from ansible.utils.unsafe_proxy import AnsibleUnsafeBytes, AnsibleUnsafeText
-
-HAS_TOML = False
-try:
-    import toml
-    HAS_TOML = True
-except ImportError:
-    pass
-
-HAS_TOMLIW = False
-try:
-    import tomli_w  # type: ignore[import]
-    HAS_TOMLIW = True
-except ImportError:
-    pass
-
-HAS_TOMLLIB = False
-try:
-    import tomllib  # type: ignore[import]
-    HAS_TOMLLIB = True
-except ImportError:
-    try:
-        import tomli as tomllib  # type: ignore[no-redef]
-        HAS_TOMLLIB = True
-    except ImportError:
-        pass
 
 display = Display()
 
 
-# dumps
-if HAS_TOML and hasattr(toml, 'TomlEncoder'):
-    # toml>=0.10.0
-    class AnsibleTomlEncoder(toml.TomlEncoder):
-        def __init__(self, *args, **kwargs):
-            super(AnsibleTomlEncoder, self).__init__(*args, **kwargs)
-            # Map our custom YAML object types to dump_funcs from ``toml``
-            self.dump_funcs.update({
-                AnsibleSequence: self.dump_funcs.get(list),
-                AnsibleUnicode: self.dump_funcs.get(str),
-                AnsibleUnsafeBytes: self.dump_funcs.get(str),
-                AnsibleUnsafeText: self.dump_funcs.get(str),
-            })
-    toml_dumps = partial(toml.dumps, encoder=AnsibleTomlEncoder())  # type: t.Callable[[t.Any], str]
-else:
-    # toml<0.10.0
-    # tomli-w
-    def toml_dumps(data):  # type: (t.Any) -> str
-        if HAS_TOML:
-            return toml.dumps(convert_yaml_objects_to_native(data))
-        elif HAS_TOMLIW:
-            return tomli_w.dumps(convert_yaml_objects_to_native(data))
-        raise AnsibleRuntimeError(
-            'The python "toml" or "tomli-w" library is required when using the TOML output format'
-        )
-
-# loads
-if HAS_TOML:
-    # prefer toml if installed, since it supports both encoding and decoding
-    toml_loads = toml.loads  # type: ignore[assignment]
-    TOMLDecodeError = toml.TomlDecodeError  # type: t.Any
-elif HAS_TOMLLIB:
-    toml_loads = tomllib.loads  # type: ignore[assignment]
-    TOMLDecodeError = tomllib.TOMLDecodeError  # type: t.Any  # type: ignore[no-redef]
-
-
-def convert_yaml_objects_to_native(obj):
-    """Older versions of the ``toml`` python library, and tomllib, don't have
-    a pluggable way to tell the encoder about custom types, so we need to
-    ensure objects that we pass are native types.
-
-    Used with:
-      - ``toml<0.10.0`` where ``toml.TomlEncoder`` is missing
-      - ``tomli`` or ``tomllib``
-
-    This function recurses an object and ensures we cast any of the types from
-    ``ansible.parsing.yaml.objects`` into their native types, effectively cleansing
-    the data before we hand it over to the toml library.
-
-    This function doesn't directly check for the types from ``ansible.parsing.yaml.objects``
-    but instead checks for the types those objects inherit from, to offer more flexibility.
-    """
-    if isinstance(obj, dict):
-        return dict((k, convert_yaml_objects_to_native(v)) for k, v in obj.items())
-    elif isinstance(obj, list):
-        return [convert_yaml_objects_to_native(v) for v in obj]
-    elif isinstance(obj, text_type):
-        return text_type(obj)
-    else:
-        return obj
-
-
 class InventoryModule(BaseFileInventoryPlugin):
     NAME = 'toml'
 
+    trusted_by_default = True  # we need the inventory system to mark trust for us, since we're not manually traversing var assignments
+
     def _parse_group(self, group, group_data):
         if group_data is not None and not isinstance(group_data, MutableMapping):
             self.display.warning("Skipping '%s' as this is not a valid group definition" % group)
@@ -246,33 +155,14 @@ class InventoryModule(BaseFileInventoryPlugin):
             raise AnsibleFileNotFound("Unable to retrieve file contents", file_name=file_name)
 
         try:
-            (b_data, private) = self.loader._get_file_contents(file_name)
-            return toml_loads(to_text(b_data, errors='surrogate_or_strict'))
-        except TOMLDecodeError as e:
-            raise AnsibleParserError(
-                'TOML file (%s) is invalid: %s' % (file_name, to_native(e)),
-                orig_exc=e
-            )
-        except (IOError, OSError) as e:
-            raise AnsibleParserError(
-                "An error occurred while trying to read the file '%s': %s" % (file_name, to_native(e)),
-                orig_exc=e
-            )
-        except Exception as e:
-            raise AnsibleParserError(
-                "An unexpected error occurred while parsing the file '%s': %s" % (file_name, to_native(e)),
-                orig_exc=e
-            )
+            return tomllib.loads(self.loader.get_text_file_contents(file_name))
+        except tomllib.TOMLDecodeError as ex:
+            raise AnsibleParserError(f'TOML file {file_name!r} is invalid.') from ex
+        except Exception as ex:
+            raise AnsibleParserError(f'An error occurred while parsing the file {file_name!r}.') from ex
 
     def parse(self, inventory, loader, path, cache=True):
-        ''' parses the inventory file '''
-        if not HAS_TOMLLIB and not HAS_TOML:
-            # tomllib works here too, but we don't call it out in the error,
-            # since you either have it or not as part of cpython stdlib >= 3.11
-            raise AnsibleParserError(
-                'The TOML inventory plugin requires the python "toml", or "tomli" library'
-            )
-
+        """ parses the inventory file """
         super(InventoryModule, self).parse(inventory, loader, path)
         self.set_options()
 

ansible/plugins/inventory/yaml.py

@@ -3,7 +3,7 @@
 
 from __future__ import annotations
 
-DOCUMENTATION = '''
+DOCUMENTATION = """
     name: yaml
     version_added: "2.4"
     short_description: Uses a specific YAML file as an inventory source.
@@ -31,8 +31,8 @@ DOCUMENTATION = '''
           - section: inventory_plugin_yaml
             key: yaml_valid_extensions
 
-'''
-EXAMPLES = '''
+"""
+EXAMPLES = """
 all: # keys must be unique, i.e. only one 'hosts' per group
     hosts:
         test1:
@@ -63,7 +63,7 @@ all: # keys must be unique, i.e. only one 'hosts' per group
                 test1 # same host as above, additional group membership
             vars:
                 group_last_var: value
-'''
+"""
 
 import os
 
@@ -81,6 +81,8 @@ class InventoryModule(BaseFileInventoryPlugin):
 
     NAME = 'yaml'
 
+    # implicit trust behavior is already added by the YAML parser invoked by the loader
+
     def __init__(self):
 
         super(InventoryModule, self).__init__()
@@ -95,13 +97,13 @@ class InventoryModule(BaseFileInventoryPlugin):
         return valid
 
     def parse(self, inventory, loader, path, cache=True):
-        ''' parses the inventory file '''
+        """ parses the inventory file """
 
         super(InventoryModule, self).parse(inventory, loader, path)
         self.set_options()
 
         try:
-            data = self.loader.load_from_file(path, cache='none')
+            data = self.loader.load_from_file(path, cache='none', trusted_as_template=True)
         except Exception as e:
             raise AnsibleParserError(e)
 
@@ -170,9 +172,9 @@ class InventoryModule(BaseFileInventoryPlugin):
         return group
 
     def _parse_host(self, host_pattern):
-        '''
+        """
         Each host key can be a pattern, try to process it and add variables as needed
-        '''
+        """
         try:
             (hostnames, port) = self._expand_hostpattern(host_pattern)
         except TypeError:

ansible/plugins/list.py

@@ -124,7 +124,7 @@ def list_collection_plugins(ptype, collections, search_paths=None):
     try:
         ploader = getattr(loader, '{0}_loader'.format(ptype))
     except AttributeError:
-        raise AnsibleError('Cannot list plugins, incorrect plugin type supplied: {0}'.format(ptype))
+        raise AnsibleError(f"Cannot list plugins, incorrect plugin type {ptype!r} supplied.") from None
 
     # get plugins for each collection
     for collection in collections.keys():
@@ -191,8 +191,8 @@ def list_plugins(ptype, collections=None, search_paths=None):
             else:
                 try:
                     plugin_collections[collection] = to_bytes(_get_collection_path(collection))
-                except ValueError as e:
-                    raise AnsibleError("Cannot use supplied collection {0}: {1}".format(collection, to_native(e)), orig_exc=e)
+                except ValueError as ex:
+                    raise AnsibleError(f"Cannot use supplied collection {collection!r}.") from ex
 
     if plugin_collections:
         plugins.update(list_collection_plugins(ptype, plugin_collections))