ansible-core 2.18.7rc1__py3-none-any.whl → 2.19.0__py3-none-any.whl

ansible/plugins/lookup/__init__.py

@@ -23,14 +23,31 @@ from ansible.errors import AnsibleFileNotFound
 from ansible.plugins import AnsiblePlugin
 from ansible.utils.display import Display
 
+import typing as t
+
+if t.TYPE_CHECKING:
+    from ansible.parsing import dataloader as _dataloader
+    from ansible import template as _template
+
 display = Display()
 
 __all__ = ['LookupBase']
 
 
 class LookupBase(AnsiblePlugin):
+    accept_args_markers: t.ClassVar[bool] = False
+    """
+    When `False`, plugin invocation is skipped when a top-level argument is a `Marker`, with the first such value substituted as the plugin result.
+    This ensures that only plugins which understand `Marker` instances for top-level arguments will encounter them.
+    """
+
+    accept_lazy_markers: t.ClassVar[bool] = False
+    """
+    When `False`, plugins will trigger a `MarkerError` exception when attempting to retrieve a `Marker` from a lazy container.
+    This ensures that only plugins which understand lazy retrieval of `Marker` instances will encounter them.
+    """
 
-    def __init__(self, loader=None, templar=None, **kwargs):
+    def __init__(self, loader: _dataloader.DataLoader | None = None, templar: _template.Templar | None = None, **kwargs) -> None:
 
         super(LookupBase, self).__init__()
 
@@ -72,7 +89,7 @@ class LookupBase(AnsiblePlugin):
         return ret
 
     @abstractmethod
-    def run(self, terms, variables=None, **kwargs):
+    def run(self, terms, variables, **kwargs):
         """
         When the playbook specifies a lookup, this method is run.  The
         arguments to the lookup become the arguments to this method.  One
@@ -104,9 +121,9 @@ class LookupBase(AnsiblePlugin):
         pass
 
     def find_file_in_search_path(self, myvars, subdir, needle, ignore_missing=False):
-        '''
+        """
         Return a file (needle) in the task's expected search path.
-        '''
+        """
 
         if 'ansible_search_path' in myvars:
             paths = myvars['ansible_search_path']

ansible/plugins/lookup/config.py

@@ -83,32 +83,20 @@ _raw:
 import ansible.plugins.loader as plugin_loader
 
 from ansible import constants as C
-from ansible.errors import AnsibleError, AnsibleLookupError, AnsibleOptionsError
-from ansible.module_utils.common.text.converters import to_native
-from ansible.module_utils.six import string_types
+from ansible.module_utils.common.sentinel import Sentinel
+from ansible.errors import AnsibleError, AnsibleUndefinedConfigEntry
 from ansible.plugins.lookup import LookupBase
-from ansible.utils.sentinel import Sentinel
 
 
-class MissingSetting(AnsibleOptionsError):
-    pass
+def _get_plugin_config(pname, ptype, config, variables):
+    # plugin creates settings on load, this is cached so not too expensive to redo
+    loader = getattr(plugin_loader, '%s_loader' % ptype)
+    p = loader.get(pname, class_only=True)
 
+    if p is None:
+        raise AnsibleError(f"Unable to load {ptype} plugin {pname!r}.")
 
-def _get_plugin_config(pname, ptype, config, variables):
-    try:
-        # plugin creates settings on load, this is cached so not too expensive to redo
-        loader = getattr(plugin_loader, '%s_loader' % ptype)
-        p = loader.get(pname, class_only=True)
-        if p is None:
-            raise AnsibleLookupError('Unable to load %s plugin "%s"' % (ptype, pname))
-        result, origin = C.config.get_config_value_and_origin(config, plugin_type=ptype, plugin_name=p._load_name, variables=variables)
-    except AnsibleLookupError:
-        raise
-    except AnsibleError as e:
-        msg = to_native(e)
-        if 'was not defined' in msg:
-            raise MissingSetting(msg, orig_exc=e)
-        raise e
+    result, origin = C.config.get_config_value_and_origin(config, plugin_type=ptype, plugin_name=p._load_name, variables=variables)
 
     return result, origin
 
@@ -116,10 +104,11 @@ def _get_plugin_config(pname, ptype, config, variables):
 def _get_global_config(config):
     try:
         result = getattr(C, config)
-        if callable(result):
-            raise AnsibleLookupError('Invalid setting "%s" attempted' % config)
-    except AttributeError as e:
-        raise MissingSetting(to_native(e), orig_exc=e)
+    except AttributeError:
+        raise AnsibleUndefinedConfigEntry(f"Setting {config!r} does not exist.") from None
+
+    if callable(result):
+        raise ValueError(f"Invalid setting {config!r} attempted.")
 
     return result
 
@@ -136,16 +125,13 @@ class LookupModule(LookupBase):
         show_origin = self.get_option('show_origin')
 
         if (ptype or pname) and not (ptype and pname):
-            raise AnsibleOptionsError('Both plugin_type and plugin_name are required, cannot use one without the other')
-
-        if not isinstance(missing, string_types) or missing not in ['error', 'warn', 'skip']:
-            raise AnsibleOptionsError('"on_missing" must be a string and one of "error", "warn" or "skip", not %s' % missing)
+            raise AnsibleError('Both plugin_type and plugin_name are required, cannot use one without the other.')
 
         ret = []
 
         for term in terms:
-            if not isinstance(term, string_types):
-                raise AnsibleOptionsError('Invalid setting identifier, "%s" is not a string, its a %s' % (term, type(term)))
+            if not isinstance(term, str):
+                raise AnsibleError(f'Invalid setting identifier, {term!r} is not a {str}, its a {type(term)}.')
 
             result = Sentinel
             origin = None
@@ -154,17 +140,17 @@ class LookupModule(LookupBase):
                     result, origin = _get_plugin_config(pname, ptype, term, variables)
                 else:
                     result = _get_global_config(term)
-            except MissingSetting as e:
+            except AnsibleUndefinedConfigEntry:
                 if missing == 'error':
-                    raise AnsibleLookupError('Unable to find setting %s' % term, orig_exc=e)
+                    raise
                 elif missing == 'warn':
-                    self._display.warning('Skipping, did not find setting %s' % term)
+                    self._display.warning(f"Skipping, did not find setting {term!r}.")
                 elif missing == 'skip':
                     pass  # this is not needed, but added to have all 3 options stated
 
             if result is not Sentinel:
                 if show_origin:
-                    ret.append((result, origin))
+                    ret.append([result, origin])
                 else:
                     ret.append(result)
         return ret

ansible/plugins/lookup/csvfile.py

@@ -16,7 +16,8 @@ DOCUMENTATION = r"""
     options:
       col:
         description:  column to return (0 indexed).
-        default: "1"
+        default: 1
+        type: int
       keycol:
         description:  column to search in (0 indexed).
         default: 0
@@ -102,76 +103,26 @@ RETURN = """
     elements: str
 """
 
-import codecs
 import csv
 
 from collections.abc import MutableSequence
 
-from ansible.errors import AnsibleError, AnsibleAssertionError
+from ansible.errors import AnsibleError
 from ansible.parsing.splitter import parse_kv
 from ansible.plugins.lookup import LookupBase
-from ansible.module_utils.six import PY2
-from ansible.module_utils.common.text.converters import to_bytes, to_native, to_text
-
-
-class CSVRecoder:
-    """
-    Iterator that reads an encoded stream and encodes the input to UTF-8
-    """
-    def __init__(self, f, encoding='utf-8'):
-        self.reader = codecs.getreader(encoding)(f)
-
-    def __iter__(self):
-        return self
-
-    def __next__(self):
-        return next(self.reader).encode("utf-8")
-
-    next = __next__   # For Python 2
-
-
-class CSVReader:
-    """
-    A CSV reader which will iterate over lines in the CSV file "f",
-    which is encoded in the given encoding.
-    """
-
-    def __init__(self, f, dialect=csv.excel, encoding='utf-8', **kwds):
-        if PY2:
-            f = CSVRecoder(f, encoding)
-        else:
-            f = codecs.getreader(encoding)(f)
-
-        self.reader = csv.reader(f, dialect=dialect, **kwds)
-
-    def __next__(self):
-        row = next(self.reader)
-        return [to_text(s) for s in row]
-
-    next = __next__  # For Python 2
-
-    def __iter__(self):
-        return self
 
 
 class LookupModule(LookupBase):
 
     def read_csv(self, filename, key, delimiter, encoding='utf-8', dflt=None, col=1, keycol=0):
-
-        try:
-            f = open(to_bytes(filename), 'rb')
-            creader = CSVReader(f, delimiter=to_native(delimiter), encoding=encoding)
-
-            for row in creader:
-                if len(row) and row[keycol] == key:
-                    return row[int(col)]
-        except Exception as e:
-            raise AnsibleError("csvfile: %s" % to_native(e))
+        with open(filename, encoding=encoding) as f:
+            for row in csv.reader(f, dialect=csv.excel, delimiter=delimiter):
+                if row and row[keycol] == key:
+                    return row[col]
 
         return dflt
 
     def run(self, terms, variables=None, **kwargs):
-
         ret = []
 
         self.set_options(var_options=variables, direct=kwargs)
@@ -191,23 +142,19 @@ class LookupModule(LookupBase):
             key = kv['_raw_params']
 
             # parameters override per term using k/v
-            try:
-                reset_params = False
-                for name, value in kv.items():
-                    if name == '_raw_params':
-                        continue
-                    if name not in paramvals:
-                        raise AnsibleAssertionError('%s is not a valid option' % name)
-
-                    self._deprecate_inline_kv()
-                    self.set_option(name, value)
-                    reset_params = True
+            reset_params = False
+            for name, value in kv.items():
+                if name == '_raw_params':
+                    continue
+                if name not in paramvals:
+                    raise ValueError(f'{name!r} is not a valid option')
 
-                if reset_params:
-                    paramvals = self.get_options()
+                self._deprecate_inline_kv()
+                self.set_option(name, value)
+                reset_params = True
 
-            except (ValueError, AssertionError) as e:
-                raise AnsibleError(e)
+            if reset_params:
+                paramvals = self.get_options()
 
             # default is just placeholder for real tab
             if paramvals['delimiter'] == 'TAB':
@@ -217,8 +164,7 @@ class LookupModule(LookupBase):
             var = self.read_csv(lookupfile, key, paramvals['delimiter'], paramvals['encoding'], paramvals['default'], paramvals['col'], paramvals['keycol'])
             if var is not None:
                 if isinstance(var, MutableSequence):
-                    for v in var:
-                        ret.append(v)
+                    ret.extend(var)
                 else:
                     ret.append(var)
 

ansible/plugins/lookup/dict.py

@@ -61,16 +61,11 @@ from ansible.plugins.lookup import LookupBase
 class LookupModule(LookupBase):
 
     def run(self, terms, variables=None, **kwargs):
-
-        # NOTE: can remove if with_ is removed
-        if not isinstance(terms, list):
-            terms = [terms]
-
         results = []
         for term in terms:
             # Expect any type of Mapping, notably hostvars
             if not isinstance(term, Mapping):
-                raise AnsibleError("with_dict expects a dict")
+                raise AnsibleError(f"the 'dict' lookup plugin expects a dictionary, got '{term}' of type {type(term)})")
 
             results.extend(self._flatten_hash_to_list(term))
         return results

ansible/plugins/lookup/env.py

@@ -38,10 +38,6 @@ EXAMPLES = """
   ansible.builtin.debug:
     msg: "Hello {{ lookup('ansible.builtin.env', 'UNDEFINED_VARIABLE', default='World') }}"
 
-- name: Fail if the variable is not defined by setting default value to 'Undefined'
-  ansible.builtin.debug:
-    msg: "Hello {{ lookup('ansible.builtin.env', 'UNDEFINED_VARIABLE', default=Undefined) }}"
-
 - name: Fail if the variable is not defined by setting default value to 'undef()'
   ansible.builtin.debug:
     msg: "Hello {{ lookup('ansible.builtin.env', 'UNDEFINED_VARIABLE', default=undef()) }}"
@@ -56,23 +52,30 @@ RETURN = """
 
 import os
 
-from jinja2.runtime import Undefined
 
-from ansible.errors import AnsibleUndefinedVariable
 from ansible.plugins.lookup import LookupBase
+from ansible._internal._templating._jinja_bits import _undef, _DEFAULT_UNDEF
+from ansible._internal._datatag._tags import Origin
 
 
 class LookupModule(LookupBase):
-    def run(self, terms, variables, **kwargs):
+    accept_args_markers = True  # the `default` arg can accept undefined values
 
+    def run(self, terms, variables=None, **kwargs):
         self.set_options(var_options=variables, direct=kwargs)
 
         ret = []
         d = self.get_option('default')
+
         for term in terms:
             var = term.split()[0]
             val = os.environ.get(var, d)
-            if isinstance(val, Undefined):
-                raise AnsibleUndefinedVariable('The "env" lookup, found an undefined variable: %s' % var)
+
+            if val is _DEFAULT_UNDEF:
+                val = _undef(f'The environment variable {var!r} is not set.')
+            else:
+                val = Origin(description=f"<environment variable {var!r}>").try_tag(val)
+
             ret.append(val)
+
         return ret

ansible/plugins/lookup/file.py

@@ -25,7 +25,6 @@ DOCUMENTATION = """
         required: False
         default: False
     notes:
-      - if read in variable context, the file can be interpreted as YAML if the content is valid to the parser.
       - this lookup does not understand 'globbing', use the fileglob lookup instead.
     seealso:
       - ref: playbook_task_paths
@@ -52,9 +51,8 @@ RETURN = """
     elements: str
 """
 
-from ansible.errors import AnsibleError, AnsibleOptionsError, AnsibleLookupError
+from ansible.errors import AnsibleError
 from ansible.plugins.lookup import LookupBase
-from ansible.module_utils.common.text.converters import to_text
 from ansible.utils.display import Display
 
 display = Display()
@@ -74,8 +72,7 @@ class LookupModule(LookupBase):
                 lookupfile = self.find_file_in_search_path(variables, 'files', term, ignore_missing=True)
                 display.vvvv(u"File lookup using %s as file" % lookupfile)
                 if lookupfile:
-                    b_contents, show_data = self._loader._get_file_contents(lookupfile)
-                    contents = to_text(b_contents, errors='surrogate_or_strict')
+                    contents = self._loader.get_text_file_contents(lookupfile)
                     if self.get_option('lstrip'):
                         contents = contents.lstrip()
                     if self.get_option('rstrip'):
@@ -83,8 +80,8 @@ class LookupModule(LookupBase):
                     ret.append(contents)
                 else:
                     # TODO: only add search info if abs path?
-                    raise AnsibleOptionsError("file not found, use -vvvvv to see paths searched")
-            except AnsibleError as e:
-                raise AnsibleLookupError("The 'file' lookup had an issue accessing the file '%s'" % term, orig_exc=e)
+                    raise AnsibleError("File not found. Use -vvvvv to see paths searched.")
+            except AnsibleError as ex:
+                raise AnsibleError(f"Unable to access the file {term!r}.") from ex
 
         return ret

ansible/plugins/lookup/first_found.py

@@ -40,15 +40,16 @@ DOCUMENTATION = """
           - When used as a template via C(lookup) or C(query), setting O(skip=True) will *not* cause the task to skip.
             Tasks must handle the empty list return from the template.
           - When V(False) and C(lookup) or C(query) specifies O(ignore:errors='ignore') all errors (including no file found,
-            but potentially others) return an empty string or an empty list respectively.
+            but potentially others) return V(None) or an empty list respectively.
           - When V(True) and C(lookup) or C(query) specifies O(ignore:errors='ignore'), no file found will return an empty
-            list and other potential errors return an empty string or empty list depending on the template call
+            list and other potential errors return V(None) or empty list depending on the template call
             (in other words return values of C(lookup) vs C(query)).
     seealso:
       - ref: playbook_task_paths
         description: Search paths used for relative paths/files.
 """
 
+
 EXAMPLES = """
 - name: Set _found_file to the first existing file, raising an error if a file is not found
   ansible.builtin.set_fact:
@@ -137,70 +138,77 @@ RETURN = """
     type: list
     elements: path
 """
-import os
-
-from collections.abc import Mapping, Sequence
 
-from jinja2.exceptions import UndefinedError
+import collections.abc as c
+import os
+import re
+import typing as t
 
-from ansible.errors import AnsibleLookupError, AnsibleUndefinedVariable
-from ansible.module_utils.six import string_types
+from ansible._internal import _task
+from ansible.errors import AnsibleError
 from ansible.plugins.lookup import LookupBase
+from ansible._internal._templating import _jinja_common
+from ansible._internal._templating import _jinja_plugins
+from ansible import template as _template
 from ansible.utils.path import unfrackpath
+from ansible.utils.display import Display
+from ansible.module_utils.datatag import native_type_name
+
 
+def _split_on(target: str, terms: str | list[str], splitters: str) -> list[str]:
+    termlist: list[str] = []
 
-def _splitter(value, chars):
-    chars = set(chars)
-    v = ''
-    for c in value:
-        if c in chars:
-            yield v
-            v = ''
-            continue
-        v += c
-    yield v
+    if isinstance(terms, str):
+        split_terms = re.split(r'[%s]' % ''.join(map(re.escape, splitters)), terms)
 
+        if split_terms == [terms]:
+            termlist = [terms]
+        else:
+            Display().deprecated(
+                msg=f'Automatic splitting of {target} on {splitters!r} is deprecated.',
+                help_text=f'Provide a list of paths for {target} instead.',
+                version='2.23',
+                obj=terms,
+            )
 
-def _split_on(terms, spliters=','):
-    termlist = []
-    if isinstance(terms, string_types):
-        termlist = list(_splitter(terms, spliters))
+            termlist = split_terms
     else:
         # added since options will already listify
-        for t in terms:
-            termlist.extend(_split_on(t, spliters))
+        for term in terms:
+            termlist.extend(_split_on(target, term, splitters))
+
     return termlist
 
 
 class LookupModule(LookupBase):
+    accept_args_markers = True  # terms are allowed to be undefined
+    accept_lazy_markers = True  # terms are allowed to be undefined
 
-    def _process_terms(self, terms, variables, kwargs):
+    def _process_terms(self, terms: c.Sequence, variables: dict[str, t.Any]) -> list[str]:
 
         total_search = []
-        skip = False
 
         # can use a dict instead of list item to pass inline config
         for term in terms:
-            if isinstance(term, Mapping):
+            if isinstance(term, c.Mapping):
                 self.set_options(var_options=variables, direct=term)
                 files = self.get_option('files')
-            elif isinstance(term, string_types):
+                files_description = "the 'files' option"
+            elif isinstance(term, str):
                 files = [term]
-            elif isinstance(term, Sequence):
-                partial, skip = self._process_terms(term, variables, kwargs)
+                files_description = "files"
+            elif isinstance(term, c.Sequence):
+                partial = self._process_terms(term, variables)
                 total_search.extend(partial)
                 continue
             else:
-                raise AnsibleLookupError("Invalid term supplied, can handle string, mapping or list of strings but got: %s for %s" % (type(term), term))
+                raise AnsibleError(f"Invalid term supplied. A string, dict or list is required, not {native_type_name(term)!r}).")
 
             paths = self.get_option('paths')
 
-            # NOTE: this is used as 'global' but  can be set many times?!?!?
-            skip = self.get_option('skip')
-
             # magic extra splitting to create lists
-            filelist = _split_on(files, ',;')
-            pathlist = _split_on(paths, ',:;')
+            filelist = _split_on(files_description, files, ',;')
+            pathlist = _split_on('the "paths" option', paths, ',:;')
 
             # create search structure
             if pathlist:
@@ -208,40 +216,46 @@ class LookupModule(LookupBase):
                     for fn in filelist:
                         f = os.path.join(path, fn)
                         total_search.append(f)
-            elif filelist:
+            else:
                 # NOTE: this is now 'extend', previously it would clobber all options, but we deemed that a bug
                 total_search.extend(filelist)
-            else:
-                total_search.append(term)
 
-        return total_search, skip
+        return total_search
+
+    def run(self, terms: list, variables=None, **kwargs):
+        if (first_marker := _template.get_first_marker_arg((), kwargs)) is not None:
+            first_marker.trip()
 
-    def run(self, terms, variables, **kwargs):
+        if _jinja_plugins._LookupContext.current().invoked_as_with:
+            # we're being invoked by TaskExecutor.get_loop_items(), special backwards compatibility behavior
+            terms = _recurse_terms(terms, omit_undefined=True)  # recursively drop undefined values from terms for backwards compatibility
+
+            # invoked_as_with shouldn't be possible outside a TaskContext
+            te_action = _task.TaskContext.current().task.action  # FIXME: this value has not been templated, it should be (historical problem)...
+
+            # based on the presence of `var`/`template`/`file` in the enclosing task action name, choose a subdir to search
+            for subdir in ['template', 'var', 'file']:
+                if subdir in te_action:
+                    break
+
+            subdir += "s"  # convert to the matching directory name
+        else:
+            terms = _recurse_terms(terms, omit_undefined=False)  # undefined values are only omitted when invoked using `with`
+
+            subdir = None
 
         self.set_options(var_options=variables, direct=kwargs)
 
         if not terms:
             terms = self.get_option('files')
 
-        total_search, skip = self._process_terms(terms, variables, kwargs)
+        total_search = self._process_terms(terms, variables)
 
         # NOTE: during refactor noticed that the 'using a dict' as term
         # is designed to only work with 'one' otherwise inconsistencies will appear.
         # see other notes below.
 
-        # actually search
-        subdir = getattr(self, '_subdir', 'files')
-
-        path = None
         for fn in total_search:
-
-            try:
-                fn = self._templar.template(fn)
-            except (AnsibleUndefinedVariable, UndefinedError):
-                # NOTE: backwards compat ff behaviour is to ignore errors when vars are undefined.
-                #       moved here from task_executor.
-                continue
-
             # get subdir if set by task executor, default to files otherwise
             path = self.find_file_in_search_path(variables, subdir, fn, ignore_missing=True)
 
@@ -249,8 +263,26 @@ class LookupModule(LookupBase):
             if path is not None:
                 return [unfrackpath(path, follow=False)]
 
+        skip = self.get_option('skip')
+
         # if we get here, no file was found
         if skip:
             # NOTE: global skip won't matter, only last 'skip' value in dict term
             return []
-        raise AnsibleLookupError("No file was found when using first_found.")
+
+        raise AnsibleError("No file was found when using first_found.")
+
+
+def _recurse_terms(o: t.Any, omit_undefined: bool) -> t.Any:
+    """Recurse through supported container types, optionally omitting undefined markers and tripping all remaining markers, returning the result."""
+    match o:
+        case dict():
+            return {k: _recurse_terms(v, omit_undefined) for k, v in o.items() if not (omit_undefined and isinstance(v, _jinja_common.UndefinedMarker))}
+        case list():
+            return [_recurse_terms(v, omit_undefined) for v in o if not (omit_undefined and isinstance(v, _jinja_common.UndefinedMarker))]
+        case tuple():
+            return tuple(_recurse_terms(v, omit_undefined) for v in o if not (omit_undefined and isinstance(v, _jinja_common.UndefinedMarker)))
+        case _jinja_common.Marker():
+            o.trip()
+        case _:
+            return o

ansible/plugins/lookup/indexed_items.py

@@ -33,19 +33,10 @@ RETURN = """
     elements: list
 """
 
-from ansible.errors import AnsibleError
 from ansible.plugins.lookup import LookupBase
 
 
 class LookupModule(LookupBase):
-
-    def __init__(self, basedir=None, **kwargs):
-        self.basedir = basedir
-
-    def run(self, terms, variables, **kwargs):
-
-        if not isinstance(terms, list):
-            raise AnsibleError("with_indexed_items expects a list")
-
+    def run(self, terms, variables=None, **kwargs):
         items = self._flatten(terms)
         return list(zip(range(len(items)), items))