ansible-core 2.18.4rc1__py3-none-any.whl → 2.19.0b1__py3-none-any.whl

ansible/plugins/filter/core.py

@@ -4,6 +4,7 @@
 from __future__ import annotations
 
 import base64
+import functools
 import glob
 import hashlib
 import json
@@ -11,31 +12,34 @@ import ntpath
 import os.path
 import re
 import shlex
-import sys
 import time
 import uuid
 import yaml
 import datetime
+import typing as t
 
 from collections.abc import Mapping
 from functools import partial
 from random import Random, SystemRandom, shuffle
 
-from jinja2.filters import pass_environment
+from jinja2.filters import do_map, do_select, do_selectattr, do_reject, do_rejectattr, pass_environment, sync_do_groupby
+from jinja2.environment import Environment
 
-from ansible.errors import AnsibleError, AnsibleFilterError, AnsibleFilterTypeError
-from ansible.module_utils.six import string_types, integer_types, reraise, text_type
+from ansible._internal._templating import _lazy_containers
+from ansible.errors import AnsibleFilterError, AnsibleTypeError
+from ansible.module_utils.datatag import native_type_name
+from ansible.module_utils.common.json import get_encoder, get_decoder
+from ansible.module_utils.six import string_types, integer_types, text_type
 from ansible.module_utils.common.text.converters import to_bytes, to_native, to_text
 from ansible.module_utils.common.collections import is_sequence
 from ansible.module_utils.common.yaml import yaml_load, yaml_load_all
-from ansible.parsing.ajson import AnsibleJSONEncoder
 from ansible.parsing.yaml.dumper import AnsibleDumper
-from ansible.template import recursive_check_defined
+from ansible.plugins import accept_args_markers, accept_lazy_markers
+from ansible._internal._templating._jinja_common import MarkerError, UndefinedMarker, validate_arg_type
 from ansible.utils.display import Display
 from ansible.utils.encrypt import do_encrypt, PASSLIB_AVAILABLE
 from ansible.utils.hashing import md5s, checksum_s
 from ansible.utils.unicode import unicode_wrap
-from ansible.utils.unsafe_proxy import _is_unsafe
 from ansible.utils.vars import merge_hash
 
 display = Display()
@@ -43,53 +47,77 @@ display = Display()
 UUID_NAMESPACE_ANSIBLE = uuid.UUID('361E6D51-FAEC-444A-9079-341386DA8E2E')
 
 
-def to_yaml(a, *args, **kw):
-    '''Make verbose, human-readable yaml'''
-    default_flow_style = kw.pop('default_flow_style', None)
-    try:
-        transformed = yaml.dump(a, Dumper=AnsibleDumper, allow_unicode=True, default_flow_style=default_flow_style, **kw)
-    except Exception as e:
-        raise AnsibleFilterError("to_yaml - %s" % to_native(e), orig_exc=e)
-    return to_text(transformed)
+def to_yaml(a, *_args, default_flow_style: bool | None = None, dump_vault_tags: bool | None = None, **kwargs) -> str:
+    """Serialize input as terse flow-style YAML."""
+    dumper = partial(AnsibleDumper, dump_vault_tags=dump_vault_tags)
 
+    return yaml.dump(a, Dumper=dumper, allow_unicode=True, default_flow_style=default_flow_style, **kwargs)
 
-def to_nice_yaml(a, indent=4, *args, **kw):
-    '''Make verbose, human-readable yaml'''
-    try:
-        transformed = yaml.dump(a, Dumper=AnsibleDumper, indent=indent, allow_unicode=True, default_flow_style=False, **kw)
-    except Exception as e:
-        raise AnsibleFilterError("to_nice_yaml - %s" % to_native(e), orig_exc=e)
-    return to_text(transformed)
+
+def to_nice_yaml(a, indent=4, *_args, default_flow_style=False, **kwargs) -> str:
+    """Serialize input as verbose multi-line YAML."""
+    return to_yaml(a, indent=indent, default_flow_style=default_flow_style, **kwargs)
+
+
+def from_json(a, profile: str | None = None, **kwargs) -> t.Any:
+    """Deserialize JSON with an optional decoder profile."""
+    cls = get_decoder(profile or "tagless")
+
+    return json.loads(a, cls=cls, **kwargs)
 
 
-def to_json(a, *args, **kw):
-    ''' Convert the value to JSON '''
+def to_json(a, profile: str | None = None, vault_to_text: t.Any = ..., preprocess_unsafe: t.Any = ..., **kwargs) -> str:
+    """Serialize as JSON with an optional encoder profile."""
 
-    # defaults for filters
-    if 'vault_to_text' not in kw:
-        kw['vault_to_text'] = True
-    if 'preprocess_unsafe' not in kw:
-        kw['preprocess_unsafe'] = False
+    if profile and vault_to_text is not ...:
+        raise ValueError("Only one of `vault_to_text` or `profile` can be specified.")
 
-    return json.dumps(a, cls=AnsibleJSONEncoder, *args, **kw)
+    if profile and preprocess_unsafe is not ...:
+        raise ValueError("Only one of `preprocess_unsafe` or `profile` can be specified.")
 
+    # deprecated: description='deprecate vault_to_text' core_version='2.23'
+    # deprecated: description='deprecate preprocess_unsafe' core_version='2.23'
 
-def to_nice_json(a, indent=4, sort_keys=True, *args, **kw):
-    '''Make verbose, human-readable JSON'''
+    cls = get_encoder(profile or "tagless")
+
+    return json.dumps(a, cls=cls, **kwargs)
+
+
+def to_nice_json(a, indent=4, sort_keys=True, **kwargs):
+    """Make verbose, human-readable JSON."""
     # TODO separators can be potentially exposed to the user as well
-    kw.pop('separators', None)
-    return to_json(a, indent=indent, sort_keys=sort_keys, separators=(',', ': '), *args, **kw)
+    kwargs.pop('separators', None)
+    return to_json(a, indent=indent, sort_keys=sort_keys, separators=(',', ': '), **kwargs)
+
 
+# CAUTION: Do not put non-string values here since they can have unwanted logical equality, such as 1.0 (equal to 1 and True) or 0.0 (equal to 0 and False).
+_valid_bool_true = {'yes', 'on', 'true', '1'}
+_valid_bool_false = {'no', 'off', 'false', '0'}
 
-def to_bool(a):
-    ''' return a bool for the arg '''
-    if a is None or isinstance(a, bool):
-        return a
-    if isinstance(a, string_types):
-        a = a.lower()
-    if a in ('yes', 'on', '1', 'true', 1):
+
+def to_bool(value: object) -> bool:
+    """Convert well-known input values to a boolean value."""
+    value_to_check: object
+    if isinstance(value, str):
+        value_to_check = value.lower()  # accept mixed case variants
+    elif isinstance(value, int):  # bool is also an int
+        value_to_check = str(value).lower()  # accept int (0, 1) and bool (True, False) -- not just string versions
+    else:
+        value_to_check = value
+
+    if value_to_check in _valid_bool_true:
         return True
-    return False
+
+    if value_to_check in _valid_bool_false:
+        return False
+
+    # if we're still here, the value is unsupported- always fire a deprecation warning
+    result = value_to_check == 1  # backwards compatibility with the old code which checked: value in ('yes', 'on', '1', 'true', 1)
+
+    # NB: update the doc string to reflect reality once this fallback is removed
+    display.deprecated(f'The `bool` filter coerced invalid value {value!r} ({native_type_name(value)}) to {result!r}.', version='2.23')
+
+    return result
 
 
 def to_datetime(string, format="%Y-%m-%d %H:%M:%S"):
@@ -97,7 +125,7 @@ def to_datetime(string, format="%Y-%m-%d %H:%M:%S"):
 
 
 def strftime(string_format, second=None, utc=False):
-    ''' return a date string using string. See https://docs.python.org/3/library/time.html#time.strftime for format '''
+    """ return a date string using string. See https://docs.python.org/3/library/time.html#time.strftime for format """
     if utc:
         timefn = time.gmtime
     else:
@@ -111,19 +139,19 @@ def strftime(string_format, second=None, utc=False):
 
 
 def quote(a):
-    ''' return its argument quoted for shell usage '''
+    """ return its argument quoted for shell usage """
     if a is None:
         a = u''
     return shlex.quote(to_text(a))
 
 
 def fileglob(pathname):
-    ''' return list of matched regular files for glob '''
+    """ return list of matched regular files for glob """
     return [g for g in glob.glob(pathname) if os.path.isfile(g)]
 
 
 def regex_replace(value='', pattern='', replacement='', ignorecase=False, multiline=False, count=0, mandatory_count=0):
-    ''' Perform a `re.sub` returning a string '''
+    """ Perform a `re.sub` returning a string """
 
     value = to_text(value, errors='surrogate_or_strict', nonstring='simplerepr')
 
@@ -141,7 +169,7 @@ def regex_replace(value='', pattern='', replacement='', ignorecase=False, multil
 
 
 def regex_findall(value, regex, multiline=False, ignorecase=False):
-    ''' Perform re.findall and return the list of matches '''
+    """ Perform re.findall and return the list of matches """
 
     value = to_text(value, errors='surrogate_or_strict', nonstring='simplerepr')
 
@@ -154,7 +182,7 @@ def regex_findall(value, regex, multiline=False, ignorecase=False):
 
 
 def regex_search(value, regex, *args, **kwargs):
-    ''' Perform re.search and return the list of matches or a backref '''
+    """ Perform re.search and return the list of matches or a backref """
 
     value = to_text(value, errors='surrogate_or_strict', nonstring='simplerepr')
 
@@ -187,7 +215,7 @@ def regex_search(value, regex, *args, **kwargs):
 
 
 def ternary(value, true_val, false_val, none_val=None):
-    '''  value ? true_val : false_val '''
+    """  value ? true_val : false_val """
     if value is None and none_val is not None:
         return none_val
     elif bool(value):
@@ -220,8 +248,6 @@ def from_yaml(data):
         # The ``text_type`` call here strips any custom
         # string wrapper class, so that CSafeLoader can
         # read the data
-        if _is_unsafe(data):
-            data = data._strip_unsafe()
         return yaml_load(text_type(to_text(data, errors='surrogate_or_strict')))
     return data
 
@@ -231,8 +257,6 @@ def from_yaml_all(data):
         # The ``text_type`` call here strips any custom
         # string wrapper class, so that CSafeLoader can
         # read the data
-        if _is_unsafe(data):
-            data = data._strip_unsafe()
         return yaml_load_all(text_type(to_text(data, errors='surrogate_or_strict')))
     return data
 
@@ -291,26 +315,10 @@ def get_encrypted_password(password, hashtype='sha512', salt=None, salt_size=Non
 
     hashtype = passlib_mapping.get(hashtype, hashtype)
 
-    unknown_passlib_hashtype = False
     if PASSLIB_AVAILABLE and hashtype not in passlib_mapping and hashtype not in passlib_mapping.values():
-        unknown_passlib_hashtype = True
-        display.deprecated(
-            f"Checking for unsupported password_hash passlib hashtype '{hashtype}'. "
-            "This will be an error in the future as all supported hashtypes must be documented.",
-            version='2.19'
-        )
+        raise AnsibleFilterError(f"{hashtype} is not in the list of supported passlib algorithms: {', '.join(passlib_mapping)}")
 
-    try:
-        return do_encrypt(password, hashtype, salt=salt, salt_size=salt_size, rounds=rounds, ident=ident)
-    except AnsibleError as e:
-        reraise(AnsibleFilterError, AnsibleFilterError(to_native(e), orig_exc=e), sys.exc_info()[2])
-    except Exception as e:
-        if unknown_passlib_hashtype:
-            # This can occur if passlib.hash has the hashtype attribute, but it has a different signature than the valid choices.
-            # In 2.19 this will replace the deprecation warning above and the extra exception handling can be deleted.
-            choices = ', '.join(passlib_mapping)
-            raise AnsibleFilterError(f"{hashtype} is not in the list of supported passlib algorithms: {choices}") from e
-        raise
+    return do_encrypt(password, hashtype, salt=salt, salt_size=salt_size, rounds=rounds, ident=ident)
 
 
 def to_uuid(string, namespace=UUID_NAMESPACE_ANSIBLE):
@@ -324,19 +332,21 @@ def to_uuid(string, namespace=UUID_NAMESPACE_ANSIBLE):
     return to_text(uuid.uuid5(uuid_namespace, to_native(string, errors='surrogate_or_strict')))
 
 
-def mandatory(a, msg=None):
+@accept_args_markers
+def mandatory(a: object, msg: str | None = None) -> object:
     """Make a variable mandatory."""
-    from jinja2.runtime import Undefined
+    # DTFIX-RELEASE: deprecate this filter; there are much better ways via undef, etc...
+    #                also remember to remove unit test checking for _undefined_name
+    if isinstance(a, UndefinedMarker):
+        if msg is not None:
+            raise AnsibleFilterError(to_text(msg))
 
-    if isinstance(a, Undefined):
         if a._undefined_name is not None:
-            name = "'%s' " % to_text(a._undefined_name)
+            name = f'{to_text(a._undefined_name)!r} '
         else:
             name = ''
 
-        if msg is not None:
-            raise AnsibleFilterError(to_native(msg))
-        raise AnsibleFilterError("Mandatory variable %s not defined." % name)
+        raise AnsibleFilterError(f"Mandatory variable {name}not defined.")
 
     return a
 
@@ -350,9 +360,6 @@ def combine(*terms, **kwargs):
     # allow the user to do `[dict1, dict2, ...] | combine`
     dictionaries = flatten(terms, levels=1)
 
-    # recursively check that every elements are defined (for jinja2)
-    recursive_check_defined(dictionaries)
-
     if not dictionaries:
         return {}
 
@@ -458,7 +465,7 @@ def comment(text, style='plain', **kw):
 
 
 @pass_environment
-def extract(environment, item, container, morekeys=None):
+def extract(environment: Environment, item, container, morekeys=None):
     if morekeys is None:
         keys = [item]
     elif isinstance(morekeys, list):
@@ -467,18 +474,28 @@ def extract(environment, item, container, morekeys=None):
         keys = [item, morekeys]
 
     value = container
+
     for key in keys:
-        value = environment.getitem(value, key)
+        try:
+            value = environment.getitem(value, key)
+        except MarkerError as ex:
+            value = ex.source
 
     return value
 
 
-def b64encode(string, encoding='utf-8'):
-    return to_text(base64.b64encode(to_bytes(string, encoding=encoding, errors='surrogate_or_strict')))
+def b64encode(string, encoding='utf-8', urlsafe=False):
+    func = base64.b64encode
+    if urlsafe:
+        func = base64.urlsafe_b64encode
+    return to_text(func(to_bytes(string, encoding=encoding, errors='surrogate_or_strict')))
 
 
-def b64decode(string, encoding='utf-8'):
-    return to_text(base64.b64decode(to_bytes(string, errors='surrogate_or_strict')), encoding=encoding)
+def b64decode(string, encoding='utf-8', urlsafe=False):
+    func = base64.b64decode
+    if urlsafe:
+        func = base64.urlsafe_b64decode
+    return to_text(func(to_bytes(string, errors='surrogate_or_strict')), encoding=encoding)
 
 
 def flatten(mylist, levels=None, skip_nulls=True):
@@ -503,14 +520,14 @@ def flatten(mylist, levels=None, skip_nulls=True):
 
 
 def subelements(obj, subelements, skip_missing=False):
-    '''Accepts a dict or list of dicts, and a dotted accessor and produces a product
+    """Accepts a dict or list of dicts, and a dotted accessor and produces a product
     of the element and the results of the dotted accessor
 
     >>> obj = [{"name": "alice", "groups": ["wheel"], "authorized": ["/tmp/alice/onekey.pub"]}]
     >>> subelements(obj, 'groups')
     [({'name': 'alice', 'groups': ['wheel'], 'authorized': ['/tmp/alice/onekey.pub']}, 'wheel')]
 
-    '''
+    """
     if isinstance(obj, dict):
         element_list = list(obj.values())
     elif isinstance(obj, list):
@@ -523,7 +540,7 @@ def subelements(obj, subelements, skip_missing=False):
     elif isinstance(subelements, string_types):
         subelement_list = subelements.split('.')
     else:
-        raise AnsibleFilterTypeError('subelements must be a list or a string')
+        raise AnsibleTypeError('subelements must be a list or a string')
 
     results = []
 
@@ -537,10 +554,10 @@ def subelements(obj, subelements, skip_missing=False):
                     values = []
                     break
                 raise AnsibleFilterError("could not find %r key in iterated item %r" % (subelement, values))
-            except TypeError:
-                raise AnsibleFilterTypeError("the key %s should point to a dictionary, got '%s'" % (subelement, values))
+            except TypeError as ex:
+                raise AnsibleTypeError("the key %s should point to a dictionary, got '%s'" % (subelement, values)) from ex
         if not isinstance(values, list):
-            raise AnsibleFilterTypeError("the key %r should point to a list, got %r" % (subelement, values))
+            raise AnsibleTypeError("the key %r should point to a list, got %r" % (subelement, values))
 
         for value in values:
             results.append((element, value))
@@ -549,11 +566,11 @@ def subelements(obj, subelements, skip_missing=False):
 
 
 def dict_to_list_of_dict_key_value_elements(mydict, key_name='key', value_name='value'):
-    ''' takes a dictionary and transforms it into a list of dictionaries,
-        with each having a 'key' and 'value' keys that correspond to the keys and values of the original '''
+    """ takes a dictionary and transforms it into a list of dictionaries,
+        with each having a 'key' and 'value' keys that correspond to the keys and values of the original """
 
     if not isinstance(mydict, Mapping):
-        raise AnsibleFilterTypeError("dict2items requires a dictionary, got %s instead." % type(mydict))
+        raise AnsibleTypeError("dict2items requires a dictionary, got %s instead." % type(mydict))
 
     ret = []
     for key in mydict:
@@ -562,31 +579,31 @@ def dict_to_list_of_dict_key_value_elements(mydict, key_name='key', value_name='
 
 
 def list_of_dict_key_value_elements_to_dict(mylist, key_name='key', value_name='value'):
-    ''' takes a list of dicts with each having a 'key' and 'value' keys, and transforms the list into a dictionary,
-        effectively as the reverse of dict2items '''
+    """ takes a list of dicts with each having a 'key' and 'value' keys, and transforms the list into a dictionary,
+        effectively as the reverse of dict2items """
 
     if not is_sequence(mylist):
-        raise AnsibleFilterTypeError("items2dict requires a list, got %s instead." % type(mylist))
+        raise AnsibleTypeError("items2dict requires a list, got %s instead." % type(mylist))
 
     try:
         return dict((item[key_name], item[value_name]) for item in mylist)
     except KeyError:
-        raise AnsibleFilterTypeError(
+        raise AnsibleTypeError(
             "items2dict requires each dictionary in the list to contain the keys '%s' and '%s', got %s instead."
             % (key_name, value_name, mylist)
         )
     except TypeError:
-        raise AnsibleFilterTypeError("items2dict requires a list of dictionaries, got %s instead." % mylist)
+        raise AnsibleTypeError("items2dict requires a list of dictionaries, got %s instead." % mylist)
 
 
 def path_join(paths):
-    ''' takes a sequence or a string, and return a concatenation
-        of the different members '''
+    """ takes a sequence or a string, and return a concatenation
+        of the different members """
     if isinstance(paths, string_types):
         return os.path.join(paths)
     if is_sequence(paths):
         return os.path.join(*paths)
-    raise AnsibleFilterTypeError("|path_join expects string or sequence, got %s instead." % type(paths))
+    raise AnsibleTypeError("|path_join expects string or sequence, got %s instead." % type(paths))
 
 
 def commonpath(paths):
@@ -599,13 +616,92 @@ def commonpath(paths):
     :rtype: str
     """
     if not is_sequence(paths):
-        raise AnsibleFilterTypeError("|commonpath expects sequence, got %s instead." % type(paths))
+        raise AnsibleTypeError("|commonpath expects sequence, got %s instead." % type(paths))
 
     return os.path.commonpath(paths)
 
 
+class GroupTuple(t.NamedTuple):
+    """
+    Custom named tuple for the groupby filter with a public interface; silently ignored by unknown type checks.
+    This matches the internal implementation of the _GroupTuple returned by Jinja's built-in groupby filter.
+    """
+
+    grouper: t.Any
+    list: list[t.Any]
+
+    def __repr__(self) -> str:
+        return tuple.__repr__(self)
+
+
+_lazy_containers.register_known_types(GroupTuple)
+
+
+@pass_environment
+def _cleansed_groupby(*args, **kwargs):
+    res = sync_do_groupby(*args, **kwargs)
+    res = [GroupTuple(grouper=g.grouper, list=g.list) for g in res]
+
+    return res
+
+# DTFIX-RELEASE: make these dumb wrappers more dynamic
+
+
+@accept_args_markers
+def ansible_default(
+    value: t.Any,
+    default_value: t.Any = '',
+    boolean: bool = False,
+) -> t.Any:
+    """Updated `default` filter that only coalesces classic undefined objects; other Undefined-derived types (eg, ErrorMarker) pass through."""
+    validate_arg_type('boolean', boolean, bool)
+
+    if isinstance(value, UndefinedMarker):
+        return default_value
+
+    if boolean and not value:
+        return default_value
+
+    return value
+
+
+@accept_lazy_markers
+@functools.wraps(do_map)
+def wrapped_map(*args, **kwargs) -> t.Any:
+    return do_map(*args, **kwargs)
+
+
+@accept_lazy_markers
+@functools.wraps(do_select)
+def wrapped_select(*args, **kwargs) -> t.Any:
+    return do_select(*args, **kwargs)
+
+
+@accept_lazy_markers
+@functools.wraps(do_selectattr)
+def wrapped_selectattr(*args, **kwargs) -> t.Any:
+    return do_selectattr(*args, **kwargs)
+
+
+@accept_lazy_markers
+@functools.wraps(do_reject)
+def wrapped_reject(*args, **kwargs) -> t.Any:
+    return do_reject(*args, **kwargs)
+
+
+@accept_lazy_markers
+@functools.wraps(do_rejectattr)
+def wrapped_rejectattr(*args, **kwargs) -> t.Any:
+    return do_rejectattr(*args, **kwargs)
+
+
+@accept_args_markers
+def type_debug(obj: object) -> str:
+    return native_type_name(obj)
+
+
 class FilterModule(object):
-    ''' Ansible core jinja2 filters '''
+    """ Ansible core jinja2 filters """
 
     def filters(self):
         return {
@@ -619,7 +715,7 @@ class FilterModule(object):
             # json
             'to_json': to_json,
             'to_nice_json': to_nice_json,
-            'from_json': json.loads,
+            'from_json': from_json,
 
             # yaml
             'to_yaml': to_yaml,
@@ -686,7 +782,7 @@ class FilterModule(object):
             'comment': comment,
 
             # debug
-            'type_debug': lambda o: o.__class__.__name__,
+            'type_debug': type_debug,
 
             # Data structures
             'combine': combine,
@@ -696,4 +792,18 @@ class FilterModule(object):
             'items2dict': list_of_dict_key_value_elements_to_dict,
             'subelements': subelements,
             'split': partial(unicode_wrap, text_type.split),
+            # FDI038 - replace this with a standard type compat shim
+            'groupby': _cleansed_groupby,
+
+            # Jinja builtins that need special arg handling
+            # DTFIX-RELEASE: document these now that they're overridden, or hide them so they don't show up as undocumented
+            'd': ansible_default,  # replaces the implementation instead of wrapping it
+            'default': ansible_default,  # replaces the implementation instead of wrapping it
+            'map': wrapped_map,
+            'select': wrapped_select,
+            'selectattr': wrapped_selectattr,
+            'reject': wrapped_reject,
+            'rejectattr': wrapped_rejectattr,
         }
+
+# DTFIX-RELEASE: document protomatter plugins, or hide them from ansible-doc/galaxy (not related to this code, but needed some place to put this comment)