ansible-core 2.18.7rc1__py3-none-any.whl → 2.19.0__py3-none-any.whl

ansible/plugins/lookup/ini.py

@@ -92,13 +92,13 @@ from io import StringIO
 from collections import defaultdict
 from collections.abc import MutableSequence
 
-from ansible.errors import AnsibleLookupError, AnsibleOptionsError
-from ansible.module_utils.common.text.converters import to_text, to_native
+from ansible.errors import AnsibleError
+from ansible.module_utils.common.text.converters import to_native
 from ansible.plugins.lookup import LookupBase
 
 
 def _parse_params(term, paramvals):
-    '''Safely split parameter term to preserve spaces'''
+    """Safely split parameter term to preserve spaces"""
 
     # TODO: deprecate this method
     valid_keys = paramvals.keys()
@@ -161,6 +161,7 @@ class LookupModule(LookupBase):
             if '=' in term or ' ' in term.strip():
                 self._deprecate_inline_kv()
                 params = _parse_params(term, paramvals)
+                param = None
                 try:
                     updated_key = False
                     updated_options = False
@@ -168,7 +169,7 @@ class LookupModule(LookupBase):
                         if '=' in param:
                             name, value = param.split('=')
                             if name not in paramvals:
-                                raise AnsibleLookupError('%s is not a valid option.' % name)
+                                raise AnsibleError(f"{name!r} is not a valid option.")
                             self.set_option(name, value)
                             updated_options = True
                         elif key == term:
@@ -177,11 +178,11 @@ class LookupModule(LookupBase):
                             updated_key = True
                     if updated_options:
                         paramvals = self.get_options()
-                except ValueError as e:
+                except ValueError as ex:
                     # bad params passed
-                    raise AnsibleLookupError("Could not use '%s' from '%s': %s" % (param, params, to_native(e)), orig_exc=e)
+                    raise ValueError(f"Could not use {param!r} from {params!r}.") from ex
                 if not updated_key:
-                    raise AnsibleOptionsError("No key to lookup was provided as first term with in string inline options: %s" % term)
+                    raise ValueError(f"No key to look up was provided as first term within string inline options: {term}")
                     # only passed options in inline string
 
             # TODO: look to use cache to avoid redoing this for every term if they use same file
@@ -195,25 +196,25 @@ class LookupModule(LookupBase):
                 config.write(u'[java_properties]\n')
                 paramvals['section'] = 'java_properties'
 
-            # Open file using encoding
-            contents, show_data = self._loader._get_file_contents(path)
-            contents = to_text(contents, errors='surrogate_or_strict', encoding=paramvals['encoding'])
+            contents = self._loader.get_text_file_contents(path, encoding=paramvals['encoding'])
             config.write(contents)
             config.seek(0, os.SEEK_SET)
 
             try:
                 self.cp.read_file(config)
-            except configparser.DuplicateOptionError as doe:
-                raise AnsibleLookupError("Duplicate option in '{file}': {error}".format(file=paramvals['file'], error=to_native(doe)))
+            except configparser.DuplicateOptionError as ex:
+                raise ValueError(f"Duplicate option in {paramvals['file']!r}.") from ex
 
             try:
                 var = self.get_value(key, paramvals['section'], paramvals['default'], paramvals['re'])
             except configparser.NoSectionError:
-                raise AnsibleLookupError("No section '{section}' in {file}".format(section=paramvals['section'], file=paramvals['file']))
+                raise ValueError(f"No section {paramvals['section']!r} in {paramvals['file']!r}.") from None
+
             if var is not None:
                 if isinstance(var, MutableSequence):
                     for v in var:
                         ret.append(v)
                 else:
                     ret.append(var)
+
         return ret

ansible/plugins/lookup/items.py

@@ -67,6 +67,6 @@ from ansible.plugins.lookup import LookupBase
 
 class LookupModule(LookupBase):
 
-    def run(self, terms, **kwargs):
+    def run(self, terms, variables=None, **kwargs):
 
         return self._flatten(terms)

ansible/plugins/lookup/lines.py

@@ -16,6 +16,9 @@ DOCUMENTATION = """
         description: command(s) to run
         required: True
     notes:
+      - The given commands are passed to a shell for execution, therefore all variables that are part of the commands and
+        come from a remote/untrusted source MUST be sanitized using the P(ansible.builtin.quote#filter) filter to avoid
+        shell injection vulnerabilities. See the example section.
       - Like all lookups, this runs on the Ansible controller and is unaffected by other keywords such as 'become'.
         If you need to use different permissions, you must change the command or run Ansible as another user.
       - Alternatively, you can use a shell/command task that runs against localhost and registers the result.
@@ -27,6 +30,10 @@ EXAMPLES = """
   ansible.builtin.debug: msg="{{ item }} is an output line from running cat on /etc/motd"
   with_lines: cat /etc/motd
 
+- name: Always use quote filter to make sure your variables are safe to use with shell
+  ansible.builtin.debug: msg="{{ item }} is an output line from running given command"
+  with_lines: "cat {{ file_name | quote }}"
+
 - name: More useful example of looping over a command result
   ansible.builtin.shell: "/usr/bin/frobnicate {{ item }}"
   with_lines:
@@ -49,7 +56,7 @@ from ansible.module_utils.common.text.converters import to_text
 
 class LookupModule(LookupBase):
 
-    def run(self, terms, variables, **kwargs):
+    def run(self, terms, variables=None, **kwargs):
 
         ret = []
         for term in terms:

ansible/plugins/lookup/list.py

@@ -37,7 +37,7 @@ from ansible.errors import AnsibleError
 
 class LookupModule(LookupBase):
 
-    def run(self, terms, **kwargs):
+    def run(self, terms, variables=None, **kwargs):
         if not isinstance(terms, Sequence):
             raise AnsibleError("with_list expects a list")
         return terms

ansible/plugins/lookup/nested.py

@@ -46,32 +46,16 @@ RETURN = """
     type: list
 """
 
-from jinja2.exceptions import UndefinedError
 
-from ansible.errors import AnsibleError, AnsibleUndefinedVariable
+from ansible.errors import AnsibleError
 from ansible.plugins.lookup import LookupBase
-from ansible.utils.listify import listify_lookup_plugin_terms
 
 
 class LookupModule(LookupBase):
-
-    def _lookup_variables(self, terms, variables):
-        results = []
-        for x in terms:
-            try:
-                intermediate = listify_lookup_plugin_terms(x, templar=self._templar, fail_on_undefined=True)
-            except UndefinedError as e:
-                raise AnsibleUndefinedVariable("One of the nested variables was undefined. The error was: %s" % e)
-            results.append(intermediate)
-        return results
-
     def run(self, terms, variables=None, **kwargs):
-
-        terms = self._lookup_variables(terms, variables)
-
         my_list = terms[:]
         my_list.reverse()
-        result = []
+
         if len(my_list) == 0:
             raise AnsibleError("with_nested requires at least one element in the nested list")
         result = my_list.pop()

ansible/plugins/lookup/password.py

@@ -161,7 +161,7 @@ def _read_password_file(b_path):
 
 
 def _gen_candidate_chars(characters):
-    '''Generate a string containing all valid chars as defined by ``characters``
+    """Generate a string containing all valid chars as defined by ``characters``
 
     :arg characters: A list of character specs. The character specs are
         shorthand names for sets of characters like 'digits', 'ascii_letters',
@@ -182,7 +182,7 @@ def _gen_candidate_chars(characters):
     the question mark and pipe characters directly. Return will be the string::
 
         u'0123456789?|'
-    '''
+    """
     chars = []
     for chars_spec in characters:
         # getattr from string expands things like "ascii_letters" and "digits"
@@ -193,11 +193,11 @@ def _gen_candidate_chars(characters):
 
 
 def _parse_content(content):
-    '''parse our password data format into password and salt
+    """parse our password data format into password and salt
 
     :arg content: The data read from the file
     :returns: password and salt
-    '''
+    """
     password = content
     salt = None
     ident = None
@@ -348,7 +348,7 @@ class LookupModule(LookupBase):
 
         return relpath, params
 
-    def run(self, terms, variables, **kwargs):
+    def run(self, terms, variables=None, **kwargs):
         ret = []
 
         for term in terms:

ansible/plugins/lookup/pipe.py

@@ -15,14 +15,12 @@ DOCUMENTATION = r"""
         description: command(s) to run.
         required: True
     notes:
+      - The given command is passed to a shell for execution, therefore all variables that are part of the command and
+        come from a remote/untrusted source MUST be sanitized using the P(ansible.builtin.quote#filter) filter to avoid
+        shell injection vulnerabilities. See the example section.
       - Like all lookups this runs on the Ansible controller and is unaffected by other keywords, such as become,
         so if you need to different permissions you must change the command or run Ansible as another user.
       - Alternatively you can use a shell/command task that runs against localhost and registers the result.
-      - Pipe lookup internally invokes Popen with shell=True (this is required and intentional).
-        This type of invocation is considered a security issue if appropriate care is not taken to sanitize any user provided or variable input.
-        It is strongly recommended to pass user input or variable input via quote filter before using with pipe lookup.
-        See example section for this.
-        Read more about this L(Bandit B602 docs,https://bandit.readthedocs.io/en/latest/plugins/b602_subprocess_popen_with_shell_equals_true.html)
       - The directory of the play is used as the current working directory.
 """
 
@@ -52,7 +50,7 @@ from ansible.plugins.lookup import LookupBase
 
 class LookupModule(LookupBase):
 
-    def run(self, terms, variables, **kwargs):
+    def run(self, terms, variables=None, **kwargs):
 
         ret = []
         for term in terms:
@@ -65,7 +63,7 @@ class LookupModule(LookupBase):
             # https://github.com/ansible/ansible/issues/6550
             term = str(term)
 
-            p = subprocess.Popen(term, cwd=self._loader.get_basedir(), shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+            p = subprocess.Popen(term, cwd=self._templar.basedir, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
             (stdout, stderr) = p.communicate()
             if p.returncode == 0:
                 ret.append(stdout.decode("utf-8").rstrip())

ansible/plugins/lookup/sequence.py

@@ -171,6 +171,12 @@ class LookupModule(LookupBase):
             setattr(self, f, self.get_option(f))
 
     def sanity_check(self):
+        """
+        Returns True if options comprise a valid sequence expression
+        Raises AnsibleError if options are an invalid expression
+        Returns false if options are valid but result in an empty sequence - these cases do not raise exceptions
+        in order to maintain historic behavior
+        """
         if self.count is None and self.end is None:
             raise AnsibleError("must specify count or end in with_sequence")
         elif self.count is not None and self.end is not None:
@@ -180,17 +186,18 @@ class LookupModule(LookupBase):
             if self.count != 0:
                 self.end = self.start + self.count * self.stride - 1
             else:
-                self.start = 0
-                self.end = 0
-                self.stride = 0
-            del self.count
+                return False
         if self.stride > 0 and self.end < self.start:
             raise AnsibleError("to count backwards make stride negative")
         if self.stride < 0 and self.end > self.start:
             raise AnsibleError("to count forward don't make stride negative")
+        if self.stride == 0:
+            return False
         if self.format.count('%') != 1:
             raise AnsibleError("bad formatting string: %s" % self.format)
 
+        return True
+
     def generate_sequence(self):
         if self.stride >= 0:
             adjust = 1
@@ -207,9 +214,13 @@ class LookupModule(LookupBase):
                     "problem formatting %r with %r" % (i, self.format)
                 )
 
-    def run(self, terms, variables, **kwargs):
+    def run(self, terms, variables=None, **kwargs):
         results = []
 
+        if kwargs and not terms:
+            # All of the necessary arguments can be provided as keywords, but we still need something to loop over
+            terms = ['']
+
         for term in terms:
             try:
                 # set defaults/global
@@ -223,10 +234,9 @@ class LookupModule(LookupBase):
                     raise AnsibleError("unknown error parsing with_sequence arguments: %r. Error was: %s" % (term, e))
 
                 self.set_fields()
-                self.sanity_check()
-
-                if self.stride != 0:
+                if self.sanity_check():
                     results.extend(self.generate_sequence())
+
             except AnsibleError:
                 raise
             except Exception as e:

ansible/plugins/lookup/subelements.py

@@ -86,7 +86,6 @@ from ansible.errors import AnsibleError
 from ansible.module_utils.six import string_types
 from ansible.module_utils.parsing.convert_bool import boolean
 from ansible.plugins.lookup import LookupBase
-from ansible.utils.listify import listify_lookup_plugin_terms
 
 
 FLAGS = ('skip_missing',)
@@ -94,14 +93,12 @@ FLAGS = ('skip_missing',)
 
 class LookupModule(LookupBase):
 
-    def run(self, terms, variables, **kwargs):
+    def run(self, terms, variables=None, **kwargs):
 
         def _raise_terms_error(msg=""):
             raise AnsibleError(
                 "subelements lookup expects a list of two or three items, " + msg)
 
-        terms[0] = listify_lookup_plugin_terms(terms[0], templar=self._templar)
-
         # check lookup terms - check number of terms
         if not isinstance(terms, list) or not 2 <= len(terms) <= 3:
             _raise_terms_error()

ansible/plugins/lookup/template.py

@@ -17,9 +17,13 @@ DOCUMENTATION = """
       convert_data:
         type: bool
         description:
-            - Whether to convert YAML into data. If False, strings that are YAML will be left untouched.
-            - Mutually exclusive with the jinja2_native option.
+            - Whether to convert YAML into data. If V(False), strings that are YAML will be left untouched.
+            - Mutually exclusive with the O(jinja2_native) option.
         default: true
+        deprecated:
+          why: This option is no longer used in the Ansible Core code base.
+          version: "2.23"
+          alternatives: Jinja2 native mode is now the default and only option, which is mutually exclusive with this option.
       variable_start_string:
         description: The string marking the beginning of a print statement.
         default: '{{'
@@ -33,13 +37,16 @@ DOCUMENTATION = """
       jinja2_native:
         description:
             - Controls whether to use Jinja2 native types.
-            - It is off by default even if global jinja2_native is True.
-            - Has no effect if global jinja2_native is False.
+            - It is off by default even if global O(jinja2_native) is V(True).
+            - Has no effect if global O(jinja2_native) is V(False).
             - This offers more flexibility than the template module which does not use Jinja2 native types at all.
-            - Mutually exclusive with the convert_data option.
-        default: False
+        default: True
         version_added: '2.11'
         type: bool
+        deprecated:
+          why: This option is no longer used in the Ansible Core code base.
+          version: "2.23"
+          alternatives: Jinja2 native mode is now the default and only option.
       template_vars:
         description: A dictionary, the keys become additional variables available for templating.
         default: {}
@@ -55,6 +62,13 @@ DOCUMENTATION = """
         version_added: '2.12'
         type: str
         default: '#}'
+      trim_blocks:
+        description:
+        - Determine when newlines should be removed from blocks.
+        - When set to V(yes) the first newline after a block is removed (block, not variable tag!).
+        type: bool
+        default: yes
+        version_added: '2.19'
     seealso:
       - ref: playbook_task_paths
         description: Search paths used for relative templates.
@@ -72,6 +86,11 @@ EXAMPLES = """
 - name: show templating results with different comment start and end string
   ansible.builtin.debug:
     msg: "{{ lookup('ansible.builtin.template', './some_template.j2', comment_start_string='[#', comment_end_string='#]') }}"
+
+- name: show templating results with trim_blocks
+  ansible.builtin.debug:
+    msg: "{{ lookup('ansible.builtin.template', './some_template.j2', trim_blocks=True) }}"
+
 """
 
 RETURN = """
@@ -84,14 +103,11 @@ _raw:
 from copy import deepcopy
 import os
 
-import ansible.constants as C
-
 from ansible.errors import AnsibleError
 from ansible.plugins.lookup import LookupBase
-from ansible.module_utils.common.text.converters import to_text
-from ansible.template import generate_ansible_template_vars, AnsibleEnvironment
+from ansible.template import trust_as_template
+from ansible._internal._templating import _template_vars
 from ansible.utils.display import Display
-from ansible.utils.native_jinja import NativeJinjaText
 
 
 display = Display()
@@ -99,25 +115,21 @@ display = Display()
 
 class LookupModule(LookupBase):
 
-    def run(self, terms, variables, **kwargs):
+    def run(self, terms, variables=None, **kwargs):
 
         ret = []
 
         self.set_options(var_options=variables, direct=kwargs)
 
         # capture options
-        convert_data_p = self.get_option('convert_data')
         lookup_template_vars = self.get_option('template_vars')
-        jinja2_native = self.get_option('jinja2_native') and C.DEFAULT_JINJA2_NATIVE
         variable_start_string = self.get_option('variable_start_string')
         variable_end_string = self.get_option('variable_end_string')
         comment_start_string = self.get_option('comment_start_string')
         comment_end_string = self.get_option('comment_end_string')
+        trim_blocks = self.get_option('trim_blocks')
 
-        if jinja2_native:
-            templar = self._templar
-        else:
-            templar = self._templar.copy_with_new_env(environment_class=AnsibleEnvironment)
+        templar = self._templar
 
         for term in terms:
             display.debug("File lookup term: %s" % term)
@@ -125,8 +137,7 @@ class LookupModule(LookupBase):
             lookupfile = self.find_file_in_search_path(variables, 'templates', term)
             display.vvvv("File lookup using %s as file" % lookupfile)
             if lookupfile:
-                b_template_data, show_data = self._loader._get_file_contents(lookupfile)
-                template_data = to_text(b_template_data, errors='surrogate_or_strict')
+                template_data = trust_as_template(self._loader.get_text_file_contents(lookupfile))
 
                 # set jinja2 internal search path for includes
                 searchpath = variables.get('ansible_search_path', [])
@@ -145,25 +156,25 @@ class LookupModule(LookupBase):
                 # plus some added by ansible (e.g., template_{path,mtime}),
                 # plus anything passed to the lookup with the template_vars=
                 # argument.
+                # FIXME: why isn't this a chainmap with a sacrificial bottom layer?
                 vars = deepcopy(variables)
-                vars.update(generate_ansible_template_vars(term, lookupfile))
+                vars.update(_template_vars.generate_ansible_template_vars(
+                    path=term,
+                    fullpath=lookupfile,
+                    include_ansible_managed='ansible_managed' not in vars,  # do not clobber ansible_managed when set by the user
+                ))
                 vars.update(lookup_template_vars)
 
-                with templar.set_temporary_context(available_variables=vars, searchpath=searchpath):
-                    overrides = dict(
-                        variable_start_string=variable_start_string,
-                        variable_end_string=variable_end_string,
-                        comment_start_string=comment_start_string,
-                        comment_end_string=comment_end_string
-                    )
-                    res = templar.template(template_data, preserve_trailing_newlines=True,
-                                           convert_data=convert_data_p, escape_backslashes=False,
-                                           overrides=overrides)
-
-                if (C.DEFAULT_JINJA2_NATIVE and not jinja2_native) or not convert_data_p:
-                    # jinja2_native is true globally but off for the lookup, we need this text
-                    # not to be processed by literal_eval anywhere in Ansible
-                    res = NativeJinjaText(res)
+                overrides = dict(
+                    variable_start_string=variable_start_string,
+                    variable_end_string=variable_end_string,
+                    comment_start_string=comment_start_string,
+                    comment_end_string=comment_end_string,
+                    trim_blocks=trim_blocks,
+                )
+
+                data_templar = templar.copy_with_new_env(available_variables=vars, searchpath=searchpath)
+                res = data_templar.template(template_data, escape_backslashes=False, overrides=overrides)
 
                 ret.append(res)
             else:

ansible/plugins/lookup/together.py

@@ -38,7 +38,6 @@ import itertools
 
 from ansible.errors import AnsibleError
 from ansible.plugins.lookup import LookupBase
-from ansible.utils.listify import listify_lookup_plugin_terms
 
 
 class LookupModule(LookupBase):
@@ -48,18 +47,7 @@ class LookupModule(LookupBase):
     Replace any empty spots in 2nd array with None:
     [1, 2], [3] -> [1, 3], [2, None]
     """
-
-    def _lookup_variables(self, terms):
-        results = []
-        for x in terms:
-            intermediate = listify_lookup_plugin_terms(x, templar=self._templar)
-            results.append(intermediate)
-        return results
-
     def run(self, terms, variables=None, **kwargs):
-
-        terms = self._lookup_variables(terms)
-
         my_list = terms[:]
         if len(my_list) == 0:
             raise AnsibleError("with_together requires at least one element in each list")

ansible/plugins/lookup/unvault.py

@@ -34,7 +34,6 @@ RETURN = """
 
 from ansible.errors import AnsibleParserError
 from ansible.plugins.lookup import LookupBase
-from ansible.module_utils.common.text.converters import to_text
 from ansible.utils.display import Display
 
 display = Display()
@@ -55,10 +54,7 @@ class LookupModule(LookupBase):
             lookupfile = self.find_file_in_search_path(variables, 'files', term)
             display.vvvv(u"Unvault lookup found %s" % lookupfile)
             if lookupfile:
-                actual_file = self._loader.get_real_file(lookupfile, decrypt=True)
-                with open(actual_file, 'rb') as f:
-                    b_contents = f.read()
-                ret.append(to_text(b_contents))
+                ret.append(self._loader.get_text_file_contents(lookupfile))
             else:
                 raise AnsibleParserError('Unable to find file matching "%s" ' % term)
 

ansible/plugins/lookup/url.py

@@ -87,7 +87,6 @@ options:
         - section: url_lookup
           key: force_basic_auth
   follow_redirects:
-    description: String of urllib2, all/yes, safe, none to determine how redirects are followed
     type: string
     version_added: "2.10"
     default: 'urllib2'
@@ -98,13 +97,6 @@ options:
     ini:
         - section: url_lookup
           key: follow_redirects
-    choices:
-      all: Will follow all redirects.
-      none: Will not follow any redirects.
-      safe: Only redirects doing GET or HEAD requests will be followed.
-      urllib2: Defer to urllib2 behavior (As of writing this follows HTTP redirects).
-      'no': (DEPRECATED, removed in 2.22) alias of V(none).
-      'yes': (DEPRECATED, removed in 2.22) alias of V(all).
   use_gssapi:
     description:
     - Use GSSAPI handler of requests
@@ -185,6 +177,8 @@ options:
     ini:
         - section: url_lookup
           key: ciphers
+extends_documentation_fragment:
+  - url.url_redirect
 """
 
 EXAMPLES = """
@@ -236,8 +230,8 @@ class LookupModule(LookupBase):
             display.vvvv("url lookup connecting to %s" % term)
             if self.get_option('follow_redirects') in ('yes', 'no'):
                 display.deprecated(
-                    "Using 'yes' or 'no' for 'follow_redirects' parameter is deprecated.",
-                    version='2.22'
+                    msg="Using 'yes' or 'no' for 'follow_redirects' parameter is deprecated.",
+                    version='2.22',
                 )
             try:
                 response = open_url(

ansible/plugins/lookup/vars.py

@@ -73,40 +73,32 @@ _value:
   elements: raw
 """
 
-from ansible.errors import AnsibleError, AnsibleUndefinedVariable
-from ansible.module_utils.six import string_types
+from ansible.errors import AnsibleTypeError
 from ansible.plugins.lookup import LookupBase
+from ansible.module_utils.datatag import native_type_name
+from ansible._internal._templating import _jinja_bits
 
 
 class LookupModule(LookupBase):
-
-    def run(self, terms, variables=None, **kwargs):
-        if variables is not None:
-            self._templar.available_variables = variables
-        myvars = getattr(self._templar, '_available_variables', {})
-
+    def run(self, terms, variables, **kwargs):
         self.set_options(var_options=variables, direct=kwargs)
+
         default = self.get_option('default')
 
         ret = []
+
         for term in terms:
-            if not isinstance(term, string_types):
-                raise AnsibleError('Invalid setting identifier, "%s" is not a string, its a %s' % (term, type(term)))
+            if not isinstance(term, str):
+                raise AnsibleTypeError(f'Variable name must be {native_type_name(str)!r} not {native_type_name(term)!r}.', obj=term)
 
             try:
-                try:
-                    value = myvars[term]
-                except KeyError:
-                    try:
-                        value = myvars['hostvars'][myvars['inventory_hostname']][term]
-                    except KeyError:
-                        raise AnsibleUndefinedVariable('No variable found with this name: %s' % term)
-
-                ret.append(self._templar.template(value, fail_on_undefined=True))
-            except AnsibleUndefinedVariable:
-                if default is not None:
-                    ret.append(default)
+                value = variables[term]
+            except KeyError:
+                if default is None:
+                    value = _jinja_bits._undef(f'No variable named {term!r} was found.')
                 else:
-                    raise
+                    value = default
+
+            ret.append(value)
 
-        return ret
+        return self._templar._engine.template(ret)