ansible-core 2.18.5rc1__py3-none-any.whl → 2.19.0b1__py3-none-any.whl

ansible/parsing/plugin_docs.py

@@ -4,13 +4,15 @@
 from __future__ import annotations
 
 import ast
-import tokenize
+
+import yaml
 
 from ansible import constants as C
 from ansible.errors import AnsibleError, AnsibleParserError
 from ansible.module_utils.common.text.converters import to_text, to_native
 from ansible.parsing.yaml.loader import AnsibleLoader
 from ansible.utils.display import Display
+from ansible._internal._datatag import _tags
 
 display = Display()
 
@@ -23,33 +25,27 @@ string_to_vars = {
 }
 
 
-def _var2string(value):
-    ''' reverse lookup of the dict above '''
-    for k, v in string_to_vars.items():
-        if v == value:
-            return k
-
-
 def _init_doc_dict():
-    ''' initialize a return dict for docs with the expected structure '''
+    """ initialize a return dict for docs with the expected structure """
     return {k: None for k in string_to_vars.values()}
 
 
 def read_docstring_from_yaml_file(filename, verbose=True, ignore_errors=True):
-    ''' Read docs from 'sidecar' yaml file doc for a plugin '''
+    """ Read docs from 'sidecar' yaml file doc for a plugin """
 
     data = _init_doc_dict()
     file_data = {}
 
     try:
         with open(filename, 'rb') as yamlfile:
-            file_data = AnsibleLoader(yamlfile.read(), file_name=filename).get_single_data()
-    except Exception as e:
-        msg = "Unable to parse yaml file '%s': %s" % (filename, to_native(e))
+            file_data = yaml.load(yamlfile, Loader=AnsibleLoader)
+    except Exception as ex:
+        msg = f"Unable to parse yaml file {filename}"
+        # DTFIX-RELEASE: find a better pattern for this (can we use the new optional error behavior?)
         if not ignore_errors:
-            raise AnsibleParserError(msg, orig_exc=e)
+            raise AnsibleParserError(f'{msg}.') from ex
         elif verbose:
-            display.error(msg)
+            display.error(f'{msg}: {ex}')
 
     if file_data:
         for key in string_to_vars:
@@ -58,74 +54,11 @@ def read_docstring_from_yaml_file(filename, verbose=True, ignore_errors=True):
     return data
 
 
-def read_docstring_from_python_module(filename, verbose=True, ignore_errors=True):
-    """
-    Use tokenization to search for assignment of the documentation variables in the given file.
-    Parse from YAML and return the resulting python structure or None together with examples as plain text.
-    """
-
-    seen = set()
-    data = _init_doc_dict()
-
-    next_string = None
-    with tokenize.open(filename) as f:
-        tokens = tokenize.generate_tokens(f.readline)
-        for token in tokens:
-
-            # found label that looks like variable
-            if token.type == tokenize.NAME:
-
-                # label is expected value, in correct place and has not been seen before
-                if token.start == 1 and token.string in string_to_vars and token.string not in seen:
-                    # next token that is string has the docs
-                    next_string = string_to_vars[token.string]
-                    continue
-
-            # previous token indicated this string is a doc string
-            if next_string is not None and token.type == tokenize.STRING:
-
-                # ensure we only process one case of it
-                seen.add(token.string)
-
-                value = token.string
-
-                # strip string modifiers/delimiters
-                if value.startswith(('r', 'b')):
-                    value = value.lstrip('rb')
-
-                if value.startswith(("'", '"')):
-                    value = value.strip("'\"")
-
-                # actually use the data
-                if next_string == 'plainexamples':
-                    # keep as string, can be yaml, but we let caller deal with it
-                    data[next_string] = to_text(value)
-                else:
-                    # yaml load the data
-                    try:
-                        data[next_string] = AnsibleLoader(value, file_name=filename).get_single_data()
-                    except Exception as e:
-                        msg = "Unable to parse docs '%s' in python file '%s': %s" % (_var2string(next_string), filename, to_native(e))
-                        if not ignore_errors:
-                            raise AnsibleParserError(msg, orig_exc=e)
-                        elif verbose:
-                            display.error(msg)
-
-                next_string = None
-
-    # if nothing else worked, fall back to old method
-    if not seen:
-        data = read_docstring_from_python_file(filename, verbose, ignore_errors)
-
-    return data
-
-
 def read_docstring_from_python_file(filename, verbose=True, ignore_errors=True):
     """
     Use ast to search for assignment of the DOCUMENTATION and EXAMPLES variables in the given file.
     Parse DOCUMENTATION from YAML and return the YAML doc or None together with EXAMPLES, as plain text.
     """
-
     data = _init_doc_dict()
 
     try:
@@ -153,28 +86,30 @@ def read_docstring_from_python_file(filename, verbose=True, ignore_errors=True):
                                 data[varkey] = to_text(child.value.value)
                             else:
                                 # string should be yaml if already not a dict
-                                data[varkey] = AnsibleLoader(child.value.value, file_name=filename).get_single_data()
+                                child_value = _tags.Origin(path=filename, line_num=child.value.lineno).tag(child.value.value)
+                                data[varkey] = yaml.load(child_value, Loader=AnsibleLoader)
 
                         display.debug('Documentation assigned: %s' % varkey)
 
-    except Exception as e:
-        msg = "Unable to parse documentation in python file '%s': %s" % (filename, to_native(e))
+    except Exception as ex:
+        msg = f"Unable to parse documentation in python file {filename!r}"
+        # DTFIX-RELEASE: better pattern to conditionally raise/display
         if not ignore_errors:
-            raise AnsibleParserError(msg, orig_exc=e)
+            raise AnsibleParserError(f'{msg}.') from ex
         elif verbose:
-            display.error(msg)
+            display.error(f'{msg}: {ex}.')
 
     return data
 
 
 def read_docstring(filename, verbose=True, ignore_errors=True):
-    ''' returns a documentation dictionary from Ansible plugin docstrings '''
+    """ returns a documentation dictionary from Ansible plugin docstrings """
 
     # NOTE: adjacency of doc file to code file is responsibility of caller
     if filename.endswith(C.YAML_DOC_EXTENSIONS):
         docstring = read_docstring_from_yaml_file(filename, verbose=verbose, ignore_errors=ignore_errors)
     elif filename.endswith(C.PYTHON_DOC_EXTENSIONS):
-        docstring = read_docstring_from_python_module(filename, verbose=verbose, ignore_errors=ignore_errors)
+        docstring = read_docstring_from_python_file(filename, verbose=verbose, ignore_errors=ignore_errors)
     elif not ignore_errors:
         raise AnsibleError("Unknown documentation format: %s" % to_native(filename))
 
@@ -221,6 +156,6 @@ def read_docstub(filename):
                 in_documentation = True
 
     short_description = r''.join(doc_stub).strip().rstrip('.')
-    data = AnsibleLoader(short_description, file_name=filename).get_single_data()
+    data = yaml.load(_tags.Origin(path=str(filename)).tag(short_description), Loader=AnsibleLoader)
 
     return data

ansible/parsing/quoting.py

@@ -23,7 +23,7 @@ def is_quoted(data):
 
 
 def unquote(data):
-    ''' removes first and last quotes from a string, if the string starts and ends with the same quotes '''
+    """ removes first and last quotes from a string, if the string starts and ends with the same quotes """
     if is_quoted(data):
         return data[1:-1]
     return data

ansible/parsing/splitter.py

@@ -22,19 +22,21 @@ import re
 
 from ansible.errors import AnsibleParserError
 from ansible.module_utils.common.text.converters import to_text
+from ansible.module_utils._internal._datatag import AnsibleTagHelper
+from ansible._internal._datatag._tags import Origin, TrustedAsTemplate
 from ansible.parsing.quoting import unquote
 
 
 # Decode escapes adapted from rspeer's answer here:
 # http://stackoverflow.com/questions/4020539/process-escape-sequences-in-a-string-in-python
 _HEXCHAR = '[a-fA-F0-9]'
-_ESCAPE_SEQUENCE_RE = re.compile(r'''
+_ESCAPE_SEQUENCE_RE = re.compile(r"""
     ( \\U{0}           # 8-digit hex escapes
     | \\u{1}           # 4-digit hex escapes
     | \\x{2}           # 2-digit hex escapes
     | \\N\{{[^}}]+\}}  # Unicode characters by name
     | \\[\\'"abfnrtv]  # Single-character escapes
-    )'''.format(_HEXCHAR * 8, _HEXCHAR * 4, _HEXCHAR * 2), re.UNICODE | re.VERBOSE)
+    )""".format(_HEXCHAR * 8, _HEXCHAR * 4, _HEXCHAR * 2), re.UNICODE | re.VERBOSE)
 
 
 def _decode_escapes(s):
@@ -45,12 +47,19 @@ def _decode_escapes(s):
 
 
 def parse_kv(args, check_raw=False):
-    '''
+    """
     Convert a string of key/value items to a dict. If any free-form params
     are found and the check_raw option is set to True, they will be added
     to a new parameter called '_raw_params'. If check_raw is not enabled,
     they will simply be ignored.
-    '''
+    """
+
+    tags = []
+    if origin_tag := Origin.get_tag(args):
+        # NB: adjusting the column number is left as an exercise for the reader
+        tags.append(origin_tag)
+    if trusted_tag := TrustedAsTemplate.get_tag(args):
+        tags.append(trusted_tag)
 
     args = to_text(args, nonstring='passthru')
 
@@ -90,14 +99,20 @@ def parse_kv(args, check_raw=False):
         if len(raw_params) > 0:
             options[u'_raw_params'] = join_args(raw_params)
 
+    if tags:
+        options = {AnsibleTagHelper.tag(k, tags): AnsibleTagHelper.tag(v, tags) for k, v in options.items()}
+
+    if origin_tag:
+        options = origin_tag.tag(options)
+
     return options
 
 
 def _get_quote_state(token, quote_char):
-    '''
+    """
     the goal of this block is to determine if the quoted string
     is unterminated in which case it needs to be put back together
-    '''
+    """
     # the char before the current one, used to see if
     # the current character is escaped
     prev_char = None
@@ -114,11 +129,11 @@ def _get_quote_state(token, quote_char):
 
 
 def _count_jinja2_blocks(token, cur_depth, open_token, close_token):
-    '''
+    """
     this function counts the number of opening/closing blocks for a
     given opening/closing type and adjusts the current depth for that
     block based on the difference
-    '''
+    """
     num_open = token.count(open_token)
     num_close = token.count(close_token)
     if num_open != num_close:
@@ -129,10 +144,10 @@ def _count_jinja2_blocks(token, cur_depth, open_token, close_token):
 
 
 def join_args(s):
-    '''
+    """
     Join the original cmd based on manipulations by split_args().
     This retains the original newlines and whitespaces.
-    '''
+    """
     result = ''
     for p in s:
         if len(result) == 0 or result.endswith('\n'):
@@ -143,7 +158,7 @@ def join_args(s):
 
 
 def split_args(args):
-    '''
+    """
     Splits args on whitespace, but intelligently reassembles
     those that may have been split over a jinja2 block or quotes.
 
@@ -156,7 +171,7 @@ def split_args(args):
 
     Basically this is a variation shlex that has some more intelligence for
     how Ansible needs to use it.
-    '''
+    """
 
     if not args:
         return []

ansible/parsing/utils/addresses.py

@@ -23,21 +23,21 @@ from ansible.errors import AnsibleParserError, AnsibleError
 # Components that match a numeric or alphanumeric begin:end or begin:end:step
 # range expression inside square brackets.
 
-numeric_range = r'''
+numeric_range = r"""
     \[
         (?:[0-9]+:[0-9]+)               # numeric begin:end
         (?::[0-9]+)?                    # numeric :step (optional)
     \]
-'''
+"""
 
-hexadecimal_range = r'''
+hexadecimal_range = r"""
     \[
         (?:[0-9a-f]+:[0-9a-f]+)         # hexadecimal begin:end
         (?::[0-9]+)?                    # numeric :step (optional)
     \]
-'''
+"""
 
-alphanumeric_range = r'''
+alphanumeric_range = r"""
     \[
         (?:
             [a-z]:[a-z]|                # one-char alphabetic range
@@ -45,56 +45,56 @@ alphanumeric_range = r'''
         )
         (?::[0-9]+)?                    # numeric :step (optional)
     \]
-'''
+"""
 
 # Components that match a 16-bit portion of an IPv6 address in hexadecimal
 # notation (0..ffff) or an 8-bit portion of an IPv4 address in decimal notation
 # (0..255) or an [x:y(:z)] numeric range.
 
-ipv6_component = r'''
+ipv6_component = r"""
     (?:
         [0-9a-f]{{1,4}}|                # 0..ffff
         {range}                         # or a numeric range
     )
-'''.format(range=hexadecimal_range)
+""".format(range=hexadecimal_range)
 
-ipv4_component = r'''
+ipv4_component = r"""
     (?:
         [01]?[0-9]{{1,2}}|              # 0..199
         2[0-4][0-9]|                    # 200..249
         25[0-5]|                        # 250..255
         {range}                         # or a numeric range
     )
-'''.format(range=numeric_range)
+""".format(range=numeric_range)
 
 # A hostname label, e.g. 'foo' in 'foo.example.com'. Consists of alphanumeric
 # characters plus dashes (and underscores) or valid ranges. The label may not
 # start or end with a hyphen or an underscore. This is interpolated into the
 # hostname pattern below. We don't try to enforce the 63-char length limit.
 
-label = r'''
+label = r"""
     (?:[\w]|{range})                    # Starts with an alphanumeric or a range
     (?:[\w_-]|{range})*                 # Then zero or more of the same or [_-]
     (?<![_-])                           # ...as long as it didn't end with [_-]
-'''.format(range=alphanumeric_range)
+""".format(range=alphanumeric_range)
 
 patterns = {
     # This matches a square-bracketed expression with a port specification. What
     # is inside the square brackets is validated later.
 
     'bracketed_hostport': re.compile(
-        r'''^
+        r"""^
             \[(.+)\]                    # [host identifier]
             :([0-9]+)                   # :port number
             $
-        ''', re.X
+        """, re.X
     ),
 
     # This matches a bare IPv4 address or hostname (or host pattern including
     # [x:y(:z)] ranges) with a port specification.
 
     'hostport': re.compile(
-        r'''^
+        r"""^
             ((?:                        # We want to match:
                 [^:\[\]]                # (a non-range character
                 |                       # ...or...
@@ -102,16 +102,16 @@ patterns = {
             )*)                         # repeated as many times as possible
             :([0-9]+)                   # followed by a port number
             $
-        ''', re.X
+        """, re.X
     ),
 
     # This matches an IPv4 address, but also permits range expressions.
 
     'ipv4': re.compile(
-        r'''^
+        r"""^
             (?:{i4}\.){{3}}{i4}         # Three parts followed by dots plus one
             $
-        '''.format(i4=ipv4_component), re.X | re.I
+        """.format(i4=ipv4_component), re.X | re.I
     ),
 
     # This matches an IPv6 address, but also permits range expressions.
@@ -125,8 +125,8 @@ patterns = {
     # accept ranges in place of each component.
 
     'ipv6': re.compile(
-        r'''^
-            (?:{0}:){{7}}{0}|           # uncompressed: 1:2:3:4:5:6:7:8
+        r"""^
+            ((?:{0}:){{7}}{0}|          # uncompressed: 1:2:3:4:5:6:7:8
             (?:{0}:){{1,6}}:|           # compressed variants, which are all
             (?:{0}:)(?::{0}){{1,6}}|    # a::b for various lengths of a,b
             (?:{0}:){{2}}(?::{0}){{1,5}}|
@@ -139,9 +139,9 @@ patterns = {
                                         # ipv4-in-ipv6 variants
             (?:0:){{6}}(?:{0}\.){{3}}{0}|
             ::(?:ffff:)?(?:{0}\.){{3}}{0}|
-            (?:0:){{5}}ffff:(?:{0}\.){{3}}{0}
+            (?:0:){{5}}ffff:(?:{0}\.){{3}}{0})
             $
-        '''.format(ipv6_component), re.X | re.I
+        """.format(ipv6_component), re.X | re.I
     ),
 
     # This matches a hostname or host pattern including [x:y(:z)] ranges.
@@ -155,11 +155,11 @@ patterns = {
     # 253 characters total) or make any attempt to process IDNs.
 
     'hostname': re.compile(
-        r'''^
+        r"""^
             {label}                     # We must have at least one label
             (?:\.{label})*              # Followed by zero or more .labels
             $
-        '''.format(label=label), re.X | re.I | re.UNICODE
+        """.format(label=label), re.X | re.I | re.UNICODE
     ),
 
 }

ansible/parsing/utils/yaml.py

@@ -6,77 +6,48 @@
 from __future__ import annotations
 
 import json
+import typing as t
 
-from yaml import YAMLError
+import yaml
 
-from ansible.errors import AnsibleParserError
-from ansible.errors.yaml_strings import YAML_SYNTAX_ERROR
-from ansible.module_utils.common.text.converters import to_native
+from ansible.errors import AnsibleJSONParserError
+from ansible._internal._errors import _utils
+from ansible.parsing.vault import VaultSecret
 from ansible.parsing.yaml.loader import AnsibleLoader
-from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject
-from ansible.parsing.ajson import AnsibleJSONDecoder
+from ansible._internal._yaml._errors import AnsibleYAMLParserError
+from ansible._internal._datatag._tags import Origin
+from ansible._internal._json._profiles import _legacy
 
 
-__all__ = ('from_yaml',)
+def from_yaml(
+    data: str,
+    file_name: str | None = None,
+    show_content: bool = True,
+    vault_secrets: list[tuple[str, VaultSecret]] | None = None,  # deprecated: description='Deprecate vault_secrets, it has no effect.' core_version='2.23'
+    json_only: bool = False,
+) -> t.Any:
+    """Creates a Python data structure from the given data, which can be either a JSON or YAML string."""
+    # FUTURE: provide Ansible-specific top-level APIs to expose JSON and YAML serialization/deserialization to hide the error handling logic
+    #         once those are in place, defer deprecate this entire function
 
+    origin = Origin.get_or_create_tag(data, file_name)
 
-def _handle_error(json_exc, yaml_exc, file_name, show_content):
-    '''
-    Optionally constructs an object (AnsibleBaseYAMLObject) to encapsulate the
-    file name/position where a YAML exception occurred, and raises an AnsibleParserError
-    to display the syntax exception information.
-    '''
+    data = origin.tag(data)
 
-    # if the YAML exception contains a problem mark, use it to construct
-    # an object the error class can use to display the faulty line
-    err_obj = None
-    if hasattr(yaml_exc, 'problem_mark'):
-        err_obj = AnsibleBaseYAMLObject()
-        err_obj.ansible_pos = (file_name, yaml_exc.problem_mark.line + 1, yaml_exc.problem_mark.column + 1)
-
-    n_yaml_syntax_error = YAML_SYNTAX_ERROR % to_native(getattr(yaml_exc, 'problem', u''))
-    n_err_msg = 'We were unable to read either as JSON nor YAML, these are the errors we got from each:\n' \
-                'JSON: %s\n\n%s' % (to_native(json_exc), n_yaml_syntax_error)
-
-    raise AnsibleParserError(n_err_msg, obj=err_obj, show_content=show_content, orig_exc=yaml_exc)
-
-
-def _safe_load(stream, file_name=None, vault_secrets=None):
-    ''' Implements yaml.safe_load(), except using our custom loader class. '''
-
-    loader = AnsibleLoader(stream, file_name, vault_secrets)
-    try:
-        return loader.get_single_data()
-    finally:
+    with _utils.RedactAnnotatedSourceContext.when(not show_content):
         try:
-            loader.dispose()
-        except AttributeError:
-            pass  # older versions of yaml don't have dispose function, ignore
-
-
-def from_yaml(data, file_name='<string>', show_content=True, vault_secrets=None, json_only=False):
-    '''
-    Creates a python datastructure from the given data, which can be either
-    a JSON or YAML string.
-    '''
-    new_data = None
-
-    try:
-        # in case we have to deal with vaults
-        AnsibleJSONDecoder.set_secrets(vault_secrets)
-
-        # we first try to load this data as JSON.
-        # Fixes issues with extra vars json strings not being parsed correctly by the yaml parser
-        new_data = json.loads(data, cls=AnsibleJSONDecoder)
-    except Exception as json_exc:
+            # we first try to load this data as JSON.
+            # Fixes issues with extra vars json strings not being parsed correctly by the yaml parser
+            return json.loads(data, cls=_legacy.Decoder)
+        except Exception as ex:
+            json_ex = ex
 
         if json_only:
-            raise AnsibleParserError(to_native(json_exc), orig_exc=json_exc)
+            AnsibleJSONParserError.handle_exception(json_ex, origin=origin)
 
-        # must not be JSON, let the rest try
         try:
-            new_data = _safe_load(data, file_name=file_name, vault_secrets=vault_secrets)
-        except YAMLError as yaml_exc:
-            _handle_error(json_exc, yaml_exc, file_name, show_content)
-
-    return new_data
+            return yaml.load(data, Loader=AnsibleLoader)  # type: ignore[arg-type]
+        except Exception as yaml_ex:
+            # DTFIX-RELEASE: how can we indicate in Origin that the data is in-memory only, to support context information -- is that useful?
+            #        we'd need to pass data to handle_exception so it could be used as the content instead of reading from disk
+            AnsibleYAMLParserError.handle_exception(yaml_ex, origin=origin)