ansible-core 2.18.7__py3-none-any.whl → 2.19.0__py3-none-any.whl

ansible/parsing/dataloader.py

@@ -2,23 +2,28 @@
 # Copyright: (c) 2017, Ansible Project
 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
 
+from __future__ import annotations
 from __future__ import annotations
 
 import copy
 import os
 import os.path
+import pathlib
 import re
 import tempfile
 import typing as t
 
 from ansible import constants as C
 from ansible.errors import AnsibleFileNotFound, AnsibleParserError
+from ansible._internal._errors import _error_utils
 from ansible.module_utils.basic import is_executable
+from ansible._internal._datatag._tags import Origin, TrustedAsTemplate, SourceWasEncrypted
+from ansible.module_utils._internal._datatag import AnsibleTagHelper
 from ansible.module_utils.six import binary_type, text_type
 from ansible.module_utils.common.text.converters import to_bytes, to_native, to_text
 from ansible.parsing.quoting import unquote
 from ansible.parsing.utils.yaml import from_yaml
-from ansible.parsing.vault import VaultLib, is_encrypted, is_encrypted_file, parse_vaulttext_envelope, PromptVaultSecret
+from ansible.parsing.vault import VaultLib, is_encrypted, is_encrypted_file, PromptVaultSecret
 from ansible.utils.path import unfrackpath
 from ansible.utils.display import Display
 
@@ -32,7 +37,7 @@ RE_TASKS = re.compile(u'(?:^|%s)+tasks%s?$' % (os.path.sep, os.path.sep))
 
 class DataLoader:
 
-    '''
+    """
     The DataLoader class is used to load and parse YAML or JSON content,
     either from a given file name or from a string that was previously
     read in through other means. A Vault password can be specified, and
@@ -47,7 +52,7 @@ class DataLoader:
         # optionally: dl.set_vault_secrets([('default', ansible.parsing.vault.PrompVaultSecret(...),)])
         ds = dl.load('...')
         ds = dl.load_from_file('/path/to/file')
-    '''
+    """
 
     def __init__(self):
 
@@ -73,12 +78,19 @@ class DataLoader:
     def set_vault_secrets(self, vault_secrets: list[tuple[str, PromptVaultSecret]] | None) -> None:
         self._vault.secrets = vault_secrets
 
-    def load(self, data: str, file_name: str = '<string>', show_content: bool = True, json_only: bool = False) -> t.Any:
-        '''Backwards compat for now'''
-        return from_yaml(data, file_name, show_content, self._vault.secrets, json_only=json_only)
-
-    def load_from_file(self, file_name: str, cache: str = 'all', unsafe: bool = False, json_only: bool = False) -> t.Any:
-        '''
+    def load(
+            self,
+            data: str,
+            file_name: str | None = None,  # DTFIX-FUTURE: consider deprecating this in favor of tagging Origin on data
+            show_content: bool = True,  # DTFIX-FUTURE: consider future deprecation, but would need RedactAnnotatedSourceContext public
+            json_only: bool = False,
+    ) -> t.Any:
+        """Backwards compat for now"""
+        with _error_utils.RedactAnnotatedSourceContext.when(not show_content):
+            return from_yaml(data=data, file_name=file_name, json_only=json_only)
+
+    def load_from_file(self, file_name: str, cache: str = 'all', unsafe: bool = False, json_only: bool = False, trusted_as_template: bool = False) -> t.Any:
+        """
         Loads data from a file, which can contain either JSON or YAML.
 
         :param file_name: The name of the file to load data from.
@@ -86,7 +98,7 @@ class DataLoader:
         :param unsafe: If True, returns the parsed data as-is without deep copying.
         :param json_only: If True, only loads JSON data from the file.
         :return: The loaded data, optionally deep-copied for safety.
-        '''
+        """
 
         # Resolve the file name
         file_name = self.path_dwim(file_name)
@@ -98,16 +110,22 @@ class DataLoader:
         if cache != 'none' and file_name in self._FILE_CACHE:
             parsed_data = self._FILE_CACHE[file_name]
         else:
-            # Read the file contents and load the data structure from them
-            (b_file_data, show_content) = self._get_file_contents(file_name)
+            file_data = self.get_text_file_contents(file_name)
 
-            file_data = to_text(b_file_data, errors='surrogate_or_strict')
-            parsed_data = self.load(data=file_data, file_name=file_name, show_content=show_content, json_only=json_only)
+            if trusted_as_template:
+                file_data = TrustedAsTemplate().tag(file_data)
+
+            parsed_data = self.load(data=file_data, file_name=file_name, json_only=json_only)
+
+            # only tagging the container, used by include_vars to determine if vars should be shown or not
+            # this is a temporary measure until a proper data senitivity system is in place
+            if SourceWasEncrypted.is_tagged_on(file_data):
+                parsed_data = SourceWasEncrypted().tag(parsed_data)
 
             # Cache the file contents for next time based on the cache option
             if cache == 'all':
                 self._FILE_CACHE[file_name] = parsed_data
-            elif cache == 'vaulted' and not show_content:
+            elif cache == 'vaulted' and SourceWasEncrypted.is_tagged_on(file_data):
                 self._FILE_CACHE[file_name] = parsed_data
 
         # Return the parsed data, optionally deep-copied for safety
@@ -133,25 +151,51 @@ class DataLoader:
         return os.listdir(path)
 
     def is_executable(self, path: str) -> bool:
-        '''is the given path executable?'''
+        """is the given path executable?"""
         path = self.path_dwim(path)
         return is_executable(path)
 
-    def _decrypt_if_vault_data(self, b_vault_data: bytes, b_file_name: bytes | None = None) -> tuple[bytes, bool]:
-        '''Decrypt b_vault_data if encrypted and return b_data and the show_content flag'''
+    def _decrypt_if_vault_data(self, b_data: bytes) -> tuple[bytes, bool]:
+        """Decrypt b_vault_data if encrypted and return b_data and the show_content flag"""
+
+        if encrypted_source := is_encrypted(b_data):
+            b_data = self._vault.decrypt(b_data)
+
+        return b_data, not encrypted_source
+
+    def get_text_file_contents(self, file_name: str, encoding: str | None = None) -> str:
+        """
+        Returns an `Origin` tagged string with the content of the specified (DWIM-expanded for relative) file path, decrypting if necessary.
+        Callers must only specify `encoding` when the user can configure it, as error messages in that case will imply configurability.
+        If `encoding` is not specified, UTF-8 will be used.
+        """
+        bytes_content, source_was_plaintext = self._get_file_contents(file_name)
+
+        if encoding is None:
+            encoding = 'utf-8'
+            help_text = 'This file must be UTF-8 encoded.'
+        else:
+            help_text = 'Ensure the correct encoding was specified.'
+
+        try:
+            str_content = bytes_content.decode(encoding=encoding, errors='strict')
+        except UnicodeDecodeError:
+            str_content = bytes_content.decode(encoding=encoding, errors='surrogateescape')
 
-        if not is_encrypted(b_vault_data):
-            show_content = True
-            return b_vault_data, show_content
+            display.deprecated(
+                msg=f"File {file_name!r} could not be decoded as {encoding!r}. Invalid content has been escaped.",
+                version="2.23",
+                # obj intentionally omitted since there's no value in showing its contents
+                help_text=help_text,
+            )
 
-        b_ciphertext, b_version, cipher_name, vault_id = parse_vaulttext_envelope(b_vault_data)
-        b_data = self._vault.decrypt(b_vault_data, filename=b_file_name)
+        if not source_was_plaintext:
+            str_content = SourceWasEncrypted().tag(str_content)
 
-        show_content = False
-        return b_data, show_content
+        return AnsibleTagHelper.tag_copy(bytes_content, str_content)
 
     def _get_file_contents(self, file_name: str) -> tuple[bytes, bool]:
-        '''
+        """
         Reads the file contents from the given file name
 
         If the contents are vault-encrypted, it will decrypt them and return
@@ -162,40 +206,41 @@ class DataLoader:
         :raises AnsibleFileNotFound: if the file_name does not refer to a file
         :raises AnsibleParserError: if we were unable to read the file
         :return: Returns a byte string of the file contents
-        '''
-        if not file_name or not isinstance(file_name, (binary_type, text_type)):
-            raise AnsibleParserError("Invalid filename: '%s'" % to_native(file_name))
+        """
+        if not file_name or not isinstance(file_name, str):
+            raise TypeError(f"Invalid filename {file_name!r}.")
 
-        b_file_name = to_bytes(self.path_dwim(file_name))
-        # This is what we really want but have to fix unittests to make it pass
-        # if not os.path.exists(b_file_name) or not os.path.isfile(b_file_name):
-        if not self.path_exists(b_file_name):
-            raise AnsibleFileNotFound("Unable to retrieve file contents", file_name=file_name)
+        file_name = self.path_dwim(file_name)
 
         try:
-            with open(b_file_name, 'rb') as f:
-                data = f.read()
-                return self._decrypt_if_vault_data(data, b_file_name)
-        except (IOError, OSError) as e:
-            raise AnsibleParserError("an error occurred while trying to read the file '%s': %s" % (file_name, to_native(e)), orig_exc=e)
+            data = pathlib.Path(file_name).read_bytes()
+        except FileNotFoundError as ex:
+            # DTFIX-FUTURE: why not just let the builtin one fly?
+            raise AnsibleFileNotFound("Unable to retrieve file contents.", file_name=file_name) from ex
+        except OSError as ex:
+            raise AnsibleParserError(f"An error occurred while trying to read the file {file_name!r}.") from ex
+
+        data = Origin(path=file_name).tag(data)
+
+        return self._decrypt_if_vault_data(data)
 
     def get_basedir(self) -> str:
-        ''' returns the current basedir '''
+        """ returns the current basedir """
         return self._basedir
 
     def set_basedir(self, basedir: str) -> None:
-        ''' sets the base directory, used to find files when a relative path is given '''
+        """ sets the base directory, used to find files when a relative path is given """
 
         if basedir is not None:
             self._basedir = to_text(basedir)
 
     def path_dwim(self, given: str) -> str:
-        '''
+        """
         make relative paths work like folks expect.
-        '''
+        """
 
-        given = unquote(given)
         given = to_text(given, errors='surrogate_or_strict')
+        given = unquote(given)
 
         if given.startswith(to_text(os.path.sep)) or given.startswith(u'~'):
             path = given
@@ -206,7 +251,7 @@ class DataLoader:
         return unfrackpath(path, follow=False)
 
     def _is_role(self, path: str) -> bool:
-        ''' imperfect role detection, roles are still valid w/o tasks|meta/main.yml|yaml|etc '''
+        """ imperfect role detection, roles are still valid w/o tasks|meta/main.yml|yaml|etc """
 
         b_path = to_bytes(path, errors='surrogate_or_strict')
         b_path_dirname = os.path.dirname(b_path)
@@ -240,13 +285,13 @@ class DataLoader:
         return False
 
     def path_dwim_relative(self, path: str, dirname: str, source: str, is_role: bool = False) -> str:
-        '''
+        """
         find one file in either a role or playbook dir with or without
         explicitly named dirname subdirs
 
         Used in action plugins and lookups to find supplemental files that
         could be in either place.
-        '''
+        """
 
         search = []
         source = to_text(source, errors='surrogate_or_strict')
@@ -295,7 +340,7 @@ class DataLoader:
         return candidate
 
     def path_dwim_relative_stack(self, paths: list[str], dirname: str, source: str, is_role: bool = False) -> str:
-        '''
+        """
         find one file in first path in stack taking roles into account and adding play basedir as fallback
 
         :arg paths: A list of text strings which are the paths to look for the filename in.
@@ -305,7 +350,7 @@ class DataLoader:
         :rtype: A text string
         :returns: An absolute path to the filename ``source`` if found
         :raises: An AnsibleFileNotFound Exception if the file is found to exist in the search paths
-        '''
+        """
         b_dirname = to_bytes(dirname, errors='surrogate_or_strict')
         b_source = to_bytes(source, errors='surrogate_or_strict')
 
@@ -353,7 +398,7 @@ class DataLoader:
         return result
 
     def _create_content_tempfile(self, content: str | bytes) -> str:
-        ''' Create a tempfile containing defined content '''
+        """ Create a tempfile containing defined content """
         fd, content_tempfile = tempfile.mkstemp(dir=C.DEFAULT_LOCAL_TMP)
         f = os.fdopen(fd, 'wb')
         content = to_bytes(content)
@@ -392,19 +437,19 @@ class DataLoader:
                         # if the file is encrypted and no password was specified,
                         # the decrypt call would throw an error, but we check first
                         # since the decrypt function doesn't know the file name
-                        data = f.read()
+                        data = Origin(path=real_path).tag(f.read())
                         if not self._vault.secrets:
                             raise AnsibleParserError("A vault password or secret must be specified to decrypt %s" % to_native(file_path))
 
-                        data = self._vault.decrypt(data, filename=real_path)
+                        data = self._vault.decrypt(data)
                         # Make a temp file
                         real_path = self._create_content_tempfile(data)
                         self._tempfiles.add(real_path)
 
             return real_path
 
-        except (IOError, OSError) as e:
-            raise AnsibleParserError("an error occurred while trying to read the file '%s': %s" % (to_native(real_path), to_native(e)), orig_exc=e)
+        except OSError as ex:
+            raise AnsibleParserError(f"an error occurred while trying to read the file {to_text(real_path)!r}.") from ex
 
     def cleanup_tmp_file(self, file_path: str) -> None:
         """

ansible/parsing/mod_args.py

@@ -19,16 +19,18 @@ from __future__ import annotations
 
 import ansible.constants as C
 from ansible.errors import AnsibleParserError, AnsibleError, AnsibleAssertionError
-from ansible.module_utils.six import string_types
+from ansible.module_utils._internal._datatag import AnsibleTagHelper
+from ansible.module_utils.common.sentinel import Sentinel
 from ansible.module_utils.common.text.converters import to_text
 from ansible.parsing.splitter import parse_kv, split_args
+from ansible.parsing.vault import EncryptedString
 from ansible.plugins.loader import module_loader, action_loader
-from ansible.template import Templar
+from ansible._internal._templating import _jinja_bits
+from ansible.utils.display import Display
 from ansible.utils.fqcn import add_internal_fqcns
-from ansible.utils.sentinel import Sentinel
 
 
-# modules formated for user msg
+# modules formatted for user msg
 _BUILTIN_RAW_PARAM_MODULES_SIMPLE = set([
     'include_vars',
     'include_tasks',
@@ -129,59 +131,64 @@ class ModuleArgsParser:
         self._task_attrs.update(['local_action', 'static'])
         self._task_attrs = frozenset(self._task_attrs)
 
-        self.resolved_action = None
-
-    def _split_module_string(self, module_string):
-        '''
+    def _split_module_string(self, module_string: str) -> tuple[str, str]:
+        """
         when module names are expressed like:
         action: copy src=a dest=b
         the first part of the string is the name of the module
         and the rest are strings pertaining to the arguments.
-        '''
+        """
 
         tokens = split_args(module_string)
         if len(tokens) > 1:
-            return (tokens[0].strip(), " ".join(tokens[1:]))
+            result = (tokens[0].strip(), " ".join(tokens[1:]))
         else:
-            return (tokens[0].strip(), "")
+            result = (tokens[0].strip(), "")
+
+        return AnsibleTagHelper.tag_copy(module_string, result[0]), AnsibleTagHelper.tag_copy(module_string, result[1])
 
     def _normalize_parameters(self, thing, action=None, additional_args=None):
-        '''
+        """
         arguments can be fuzzy.  Deal with all the forms.
-        '''
-
-        additional_args = {} if additional_args is None else additional_args
+        """
 
         # final args are the ones we'll eventually return, so first update
         # them with any additional args specified, which have lower priority
         # than those which may be parsed/normalized next
         final_args = dict()
-        if additional_args:
-            if isinstance(additional_args, string_types):
-                templar = Templar(loader=None)
-                if templar.is_template(additional_args):
-                    final_args['_variable_params'] = additional_args
-                else:
-                    raise AnsibleParserError("Complex args containing variables cannot use bare variables (without Jinja2 delimiters), "
-                                             "and must use the full variable style ('{{var_name}}')")
+
+        if additional_args is not Sentinel:
+            if isinstance(additional_args, str) and _jinja_bits.is_possibly_all_template(additional_args):
+                final_args['_variable_params'] = additional_args
             elif isinstance(additional_args, dict):
                 final_args.update(additional_args)
+            elif additional_args is None:
+                Display().deprecated(
+                    msg="Ignoring empty task `args` keyword.",
+                    version="2.23",
+                    help_text='A mapping or template which resolves to a mapping is required.',
+                    obj=self._task_ds,
+                )
             else:
-                raise AnsibleParserError('Complex args must be a dictionary or variable string ("{{var}}").')
+                raise AnsibleParserError(
+                    message='The value of the task `args` keyword is invalid.',
+                    help_text='A mapping or template which resolves to a mapping is required.',
+                    obj=additional_args,
+                )
 
         # how we normalize depends if we figured out what the module name is
         # yet.  If we have already figured it out, it's a 'new style' invocation.
         # otherwise, it's not
 
         if action is not None:
-            args = self._normalize_new_style_args(thing, action)
+            args = self._normalize_new_style_args(thing, action, additional_args)
         else:
             (action, args) = self._normalize_old_style_args(thing)
 
             # this can occasionally happen, simplify
             if args and 'args' in args:
                 tmp_args = args.pop('args')
-                if isinstance(tmp_args, string_types):
+                if isinstance(tmp_args, str):
                     tmp_args = parse_kv(tmp_args)
                 args.update(tmp_args)
 
@@ -204,8 +211,8 @@ class ModuleArgsParser:
 
         return (action, final_args)
 
-    def _normalize_new_style_args(self, thing, action):
-        '''
+    def _normalize_new_style_args(self, thing, action, additional_args):
+        """
         deals with fuzziness in new style module invocations
         accepting key=value pairs and dictionaries, and returns
         a dictionary of arguments
@@ -215,15 +222,29 @@ class ModuleArgsParser:
             {'region': 'xyz'}, 'ec2'
         standardized outputs like:
             { _raw_params: 'echo hi', _uses_shell: True }
-        '''
+        """
 
         if isinstance(thing, dict):
             # form is like: { xyz: { x: 2, y: 3 } }
             args = thing
-        elif isinstance(thing, string_types):
+        elif isinstance(thing, str):
             # form is like: copy: src=a dest=b
             check_raw = action in FREEFORM_ACTIONS
             args = parse_kv(thing, check_raw=check_raw)
+            args_keys = set(args) - {'_raw_params'}
+
+            if args_keys and additional_args is not Sentinel:
+                kv_args = ', '.join(repr(arg) for arg in sorted(args_keys))
+
+                Display().deprecated(
+                    msg=f"Merging legacy k=v args ({kv_args}) into task args.",
+                    help_text="Include all task args in the task `args` mapping.",
+                    version="2.23",
+                    obj=thing,
+                )
+        elif isinstance(thing, EncryptedString):
+            # k=v parsing intentionally omitted
+            args = dict(_raw_params=thing)
         elif thing is None:
             # this can happen with modules which take no params, like ping:
             args = None
@@ -232,7 +253,7 @@ class ModuleArgsParser:
         return args
 
     def _normalize_old_style_args(self, thing):
-        '''
+        """
         deals with fuzziness in old-style (action/local_action) module invocations
         returns tuple of (module_name, dictionary_args)
 
@@ -242,13 +263,14 @@ class ModuleArgsParser:
            {'module': 'ec2', 'x': 1 }
         standardized outputs like:
            ('ec2', { 'x': 1} )
-        '''
+        """
 
         action = None
         args = None
 
         if isinstance(thing, dict):
             # form is like:  action: { module: 'copy', src: 'a', dest: 'b' }
+            Display().deprecated("Using a mapping for `action` is deprecated.", version='2.23', help_text='Use a string value for `action`.', obj=thing)
             thing = thing.copy()
             if 'module' in thing:
                 action, module_args = self._split_module_string(thing['module'])
@@ -257,7 +279,7 @@ class ModuleArgsParser:
                 args.update(parse_kv(module_args, check_raw=check_raw))
                 del args['module']
 
-        elif isinstance(thing, string_types):
+        elif isinstance(thing, str):
             # form is like:  action: copy src=a dest=b
             (action, args) = self._split_module_string(thing)
             check_raw = action in FREEFORM_ACTIONS
@@ -270,13 +292,11 @@ class ModuleArgsParser:
         return (action, args)
 
     def parse(self, skip_action_validation=False):
-        '''
+        """
         Given a task in one of the supported forms, parses and returns
         returns the action, arguments, and delegate_to values for the
         task, dealing with all sorts of levels of fuzziness.
-        '''
-
-        thing = None
+        """
 
         action = None
         delegate_to = self._task_ds.get('delegate_to', Sentinel)
@@ -285,14 +305,14 @@ class ModuleArgsParser:
         # This is the standard YAML form for command-type modules. We grab
         # the args and pass them in as additional arguments, which can/will
         # be overwritten via dict updates from the other arg sources below
-        additional_args = self._task_ds.get('args', dict())
+        additional_args = self._task_ds.get('args', Sentinel)
 
         # We can have one of action, local_action, or module specified
         # action
         if 'action' in self._task_ds:
             # an old school 'action' statement
             thing = self._task_ds['action']
-            action, args = self._normalize_parameters(thing, action=action, additional_args=additional_args)
+            action, args = self._normalize_parameters(thing, additional_args=additional_args)
 
         # local_action
         if 'local_action' in self._task_ds:
@@ -301,12 +321,7 @@ class ModuleArgsParser:
                 raise AnsibleParserError("action and local_action are mutually exclusive", obj=self._task_ds)
             thing = self._task_ds.get('local_action', '')
             delegate_to = 'localhost'
-            action, args = self._normalize_parameters(thing, action=action, additional_args=additional_args)
-
-        if action is not None and not skip_action_validation:
-            context = _get_action_context(action, self._collection_list)
-            if context is not None and context.resolved:
-                self.resolved_action = context.resolved_fqcn
+            action, args = self._normalize_parameters(thing, additional_args=additional_args)
 
         # module: <stuff> is the more new-style invocation
 
@@ -315,14 +330,13 @@ class ModuleArgsParser:
 
         # walk the filtered input dictionary to see if we recognize a module name
         for item, value in non_task_ds.items():
-            context = None
-            is_action_candidate = False
             if item in BUILTIN_TASKS:
                 is_action_candidate = True
             elif skip_action_validation:
                 is_action_candidate = True
             else:
                 try:
+                    # DTFIX-FUTURE: extract to a helper method, shared with Task.post_validate_args
                     context = _get_action_context(item, self._collection_list)
                 except AnsibleError as e:
                     if e.obj is None:
@@ -336,9 +350,6 @@ class ModuleArgsParser:
                 if action is not None:
                     raise AnsibleParserError("conflicting action statements: %s, %s" % (action, item), obj=self._task_ds)
 
-                if context is not None and context.resolved:
-                    self.resolved_action = context.resolved_fqcn
-
                 action = item
                 thing = value
                 action, args = self._normalize_parameters(thing, action=action, additional_args=additional_args)
@@ -353,14 +364,5 @@ class ModuleArgsParser:
             else:
                 raise AnsibleParserError("no module/action detected in task.",
                                          obj=self._task_ds)
-        elif args.get('_raw_params', '') != '' and action not in RAW_PARAM_MODULES:
-            templar = Templar(loader=None)
-            raw_params = args.pop('_raw_params')
-            if templar.is_template(raw_params):
-                args['_variable_params'] = raw_params
-            else:
-                raise AnsibleParserError(
-                    "this task '%s' has extra params, which is only allowed in the following modules: %s" % (action, ", ".join(RAW_PARAM_MODULES_SIMPLE)),
-                    obj=self._task_ds)
 
-        return (action, args, delegate_to)
+        return action, args, delegate_to