py2docfx 0.1.11.dev1830301__py3-none-any.whl → 0.1.11.dev1860114__py3-none-any.whl

py2docfx/venv/template/Lib/site-packages/jinja2/lexer.py

@@ -1,869 +0,0 @@
-"""Implements a Jinja / Python combination lexer. The ``Lexer`` class
-is used to do some preprocessing. It filters out invalid operators like
-the bitshift operators we don't allow in templates. It separates
-template code and python code in expressions.
-"""
-import re
-import typing as t
-from ast import literal_eval
-from collections import deque
-from sys import intern
-
-from ._identifier import pattern as name_re
-from .exceptions import TemplateSyntaxError
-from .utils import LRUCache
-
-if t.TYPE_CHECKING:
-    import typing_extensions as te
-    from .environment import Environment
-
-# cache for the lexers. Exists in order to be able to have multiple
-# environments with the same lexer
-_lexer_cache: t.MutableMapping[t.Tuple, "Lexer"] = LRUCache(50)  # type: ignore
-
-# static regular expressions
-whitespace_re = re.compile(r"\s+")
-newline_re = re.compile(r"(\r\n|\r|\n)")
-string_re = re.compile(
-    r"('([^'\\]*(?:\\.[^'\\]*)*)'" r'|"([^"\\]*(?:\\.[^"\\]*)*)")', re.S
-)
-integer_re = re.compile(
-    r"""
-    (
-        0b(_?[0-1])+ # binary
-    |
-        0o(_?[0-7])+ # octal
-    |
-        0x(_?[\da-f])+ # hex
-    |
-        [1-9](_?\d)* # decimal
-    |
-        0(_?0)* # decimal zero
-    )
-    """,
-    re.IGNORECASE | re.VERBOSE,
-)
-float_re = re.compile(
-    r"""
-    (?<!\.)  # doesn't start with a .
-    (\d+_)*\d+  # digits, possibly _ separated
-    (
-        (\.(\d+_)*\d+)?  # optional fractional part
-        e[+\-]?(\d+_)*\d+  # exponent part
-    |
-        \.(\d+_)*\d+  # required fractional part
-    )
-    """,
-    re.IGNORECASE | re.VERBOSE,
-)
-
-# internal the tokens and keep references to them
-TOKEN_ADD = intern("add")
-TOKEN_ASSIGN = intern("assign")
-TOKEN_COLON = intern("colon")
-TOKEN_COMMA = intern("comma")
-TOKEN_DIV = intern("div")
-TOKEN_DOT = intern("dot")
-TOKEN_EQ = intern("eq")
-TOKEN_FLOORDIV = intern("floordiv")
-TOKEN_GT = intern("gt")
-TOKEN_GTEQ = intern("gteq")
-TOKEN_LBRACE = intern("lbrace")
-TOKEN_LBRACKET = intern("lbracket")
-TOKEN_LPAREN = intern("lparen")
-TOKEN_LT = intern("lt")
-TOKEN_LTEQ = intern("lteq")
-TOKEN_MOD = intern("mod")
-TOKEN_MUL = intern("mul")
-TOKEN_NE = intern("ne")
-TOKEN_PIPE = intern("pipe")
-TOKEN_POW = intern("pow")
-TOKEN_RBRACE = intern("rbrace")
-TOKEN_RBRACKET = intern("rbracket")
-TOKEN_RPAREN = intern("rparen")
-TOKEN_SEMICOLON = intern("semicolon")
-TOKEN_SUB = intern("sub")
-TOKEN_TILDE = intern("tilde")
-TOKEN_WHITESPACE = intern("whitespace")
-TOKEN_FLOAT = intern("float")
-TOKEN_INTEGER = intern("integer")
-TOKEN_NAME = intern("name")
-TOKEN_STRING = intern("string")
-TOKEN_OPERATOR = intern("operator")
-TOKEN_BLOCK_BEGIN = intern("block_begin")
-TOKEN_BLOCK_END = intern("block_end")
-TOKEN_VARIABLE_BEGIN = intern("variable_begin")
-TOKEN_VARIABLE_END = intern("variable_end")
-TOKEN_RAW_BEGIN = intern("raw_begin")
-TOKEN_RAW_END = intern("raw_end")
-TOKEN_COMMENT_BEGIN = intern("comment_begin")
-TOKEN_COMMENT_END = intern("comment_end")
-TOKEN_COMMENT = intern("comment")
-TOKEN_LINESTATEMENT_BEGIN = intern("linestatement_begin")
-TOKEN_LINESTATEMENT_END = intern("linestatement_end")
-TOKEN_LINECOMMENT_BEGIN = intern("linecomment_begin")
-TOKEN_LINECOMMENT_END = intern("linecomment_end")
-TOKEN_LINECOMMENT = intern("linecomment")
-TOKEN_DATA = intern("data")
-TOKEN_INITIAL = intern("initial")
-TOKEN_EOF = intern("eof")
-
-# bind operators to token types
-operators = {
-    "+": TOKEN_ADD,
-    "-": TOKEN_SUB,
-    "/": TOKEN_DIV,
-    "//": TOKEN_FLOORDIV,
-    "*": TOKEN_MUL,
-    "%": TOKEN_MOD,
-    "**": TOKEN_POW,
-    "~": TOKEN_TILDE,
-    "[": TOKEN_LBRACKET,
-    "]": TOKEN_RBRACKET,
-    "(": TOKEN_LPAREN,
-    ")": TOKEN_RPAREN,
-    "{": TOKEN_LBRACE,
-    "}": TOKEN_RBRACE,
-    "==": TOKEN_EQ,
-    "!=": TOKEN_NE,
-    ">": TOKEN_GT,
-    ">=": TOKEN_GTEQ,
-    "<": TOKEN_LT,
-    "<=": TOKEN_LTEQ,
-    "=": TOKEN_ASSIGN,
-    ".": TOKEN_DOT,
-    ":": TOKEN_COLON,
-    "|": TOKEN_PIPE,
-    ",": TOKEN_COMMA,
-    ";": TOKEN_SEMICOLON,
-}
-
-reverse_operators = {v: k for k, v in operators.items()}
-assert len(operators) == len(reverse_operators), "operators dropped"
-operator_re = re.compile(
-    f"({'|'.join(re.escape(x) for x in sorted(operators, key=lambda x: -len(x)))})"
-)
-
-ignored_tokens = frozenset(
-    [
-        TOKEN_COMMENT_BEGIN,
-        TOKEN_COMMENT,
-        TOKEN_COMMENT_END,
-        TOKEN_WHITESPACE,
-        TOKEN_LINECOMMENT_BEGIN,
-        TOKEN_LINECOMMENT_END,
-        TOKEN_LINECOMMENT,
-    ]
-)
-ignore_if_empty = frozenset(
-    [TOKEN_WHITESPACE, TOKEN_DATA, TOKEN_COMMENT, TOKEN_LINECOMMENT]
-)
-
-
-def _describe_token_type(token_type: str) -> str:
-    if token_type in reverse_operators:
-        return reverse_operators[token_type]
-
-    return {
-        TOKEN_COMMENT_BEGIN: "begin of comment",
-        TOKEN_COMMENT_END: "end of comment",
-        TOKEN_COMMENT: "comment",
-        TOKEN_LINECOMMENT: "comment",
-        TOKEN_BLOCK_BEGIN: "begin of statement block",
-        TOKEN_BLOCK_END: "end of statement block",
-        TOKEN_VARIABLE_BEGIN: "begin of print statement",
-        TOKEN_VARIABLE_END: "end of print statement",
-        TOKEN_LINESTATEMENT_BEGIN: "begin of line statement",
-        TOKEN_LINESTATEMENT_END: "end of line statement",
-        TOKEN_DATA: "template data / text",
-        TOKEN_EOF: "end of template",
-    }.get(token_type, token_type)
-
-
-def describe_token(token: "Token") -> str:
-    """Returns a description of the token."""
-    if token.type == TOKEN_NAME:
-        return token.value
-
-    return _describe_token_type(token.type)
-
-
-def describe_token_expr(expr: str) -> str:
-    """Like `describe_token` but for token expressions."""
-    if ":" in expr:
-        type, value = expr.split(":", 1)
-
-        if type == TOKEN_NAME:
-            return value
-    else:
-        type = expr
-
-    return _describe_token_type(type)
-
-
-def count_newlines(value: str) -> int:
-    """Count the number of newline characters in the string.  This is
-    useful for extensions that filter a stream.
-    """
-    return len(newline_re.findall(value))
-
-
-def compile_rules(environment: "Environment") -> t.List[t.Tuple[str, str]]:
-    """Compiles all the rules from the environment into a list of rules."""
-    e = re.escape
-    rules = [
-        (
-            len(environment.comment_start_string),
-            TOKEN_COMMENT_BEGIN,
-            e(environment.comment_start_string),
-        ),
-        (
-            len(environment.block_start_string),
-            TOKEN_BLOCK_BEGIN,
-            e(environment.block_start_string),
-        ),
-        (
-            len(environment.variable_start_string),
-            TOKEN_VARIABLE_BEGIN,
-            e(environment.variable_start_string),
-        ),
-    ]
-
-    if environment.line_statement_prefix is not None:
-        rules.append(
-            (
-                len(environment.line_statement_prefix),
-                TOKEN_LINESTATEMENT_BEGIN,
-                r"^[ \t\v]*" + e(environment.line_statement_prefix),
-            )
-        )
-    if environment.line_comment_prefix is not None:
-        rules.append(
-            (
-                len(environment.line_comment_prefix),
-                TOKEN_LINECOMMENT_BEGIN,
-                r"(?:^|(?<=\S))[^\S\r\n]*" + e(environment.line_comment_prefix),
-            )
-        )
-
-    return [x[1:] for x in sorted(rules, reverse=True)]
-
-
-class Failure:
-    """Class that raises a `TemplateSyntaxError` if called.
-    Used by the `Lexer` to specify known errors.
-    """
-
-    def __init__(
-        self, message: str, cls: t.Type[TemplateSyntaxError] = TemplateSyntaxError
-    ) -> None:
-        self.message = message
-        self.error_class = cls
-
-    def __call__(self, lineno: int, filename: str) -> "te.NoReturn":
-        raise self.error_class(self.message, lineno, filename)
-
-
-class Token(t.NamedTuple):
-    lineno: int
-    type: str
-    value: str
-
-    def __str__(self) -> str:
-        return describe_token(self)
-
-    def test(self, expr: str) -> bool:
-        """Test a token against a token expression.  This can either be a
-        token type or ``'token_type:token_value'``.  This can only test
-        against string values and types.
-        """
-        # here we do a regular string equality check as test_any is usually
-        # passed an iterable of not interned strings.
-        if self.type == expr:
-            return True
-
-        if ":" in expr:
-            return expr.split(":", 1) == [self.type, self.value]
-
-        return False
-
-    def test_any(self, *iterable: str) -> bool:
-        """Test against multiple token expressions."""
-        return any(self.test(expr) for expr in iterable)
-
-
-class TokenStreamIterator:
-    """The iterator for tokenstreams.  Iterate over the stream
-    until the eof token is reached.
-    """
-
-    def __init__(self, stream: "TokenStream") -> None:
-        self.stream = stream
-
-    def __iter__(self) -> "TokenStreamIterator":
-        return self
-
-    def __next__(self) -> Token:
-        token = self.stream.current
-
-        if token.type is TOKEN_EOF:
-            self.stream.close()
-            raise StopIteration
-
-        next(self.stream)
-        return token
-
-
-class TokenStream:
-    """A token stream is an iterable that yields :class:`Token`\\s.  The
-    parser however does not iterate over it but calls :meth:`next` to go
-    one token ahead.  The current active token is stored as :attr:`current`.
-    """
-
-    def __init__(
-        self,
-        generator: t.Iterable[Token],
-        name: t.Optional[str],
-        filename: t.Optional[str],
-    ):
-        self._iter = iter(generator)
-        self._pushed: "te.Deque[Token]" = deque()
-        self.name = name
-        self.filename = filename
-        self.closed = False
-        self.current = Token(1, TOKEN_INITIAL, "")
-        next(self)
-
-    def __iter__(self) -> TokenStreamIterator:
-        return TokenStreamIterator(self)
-
-    def __bool__(self) -> bool:
-        return bool(self._pushed) or self.current.type is not TOKEN_EOF
-
-    @property
-    def eos(self) -> bool:
-        """Are we at the end of the stream?"""
-        return not self
-
-    def push(self, token: Token) -> None:
-        """Push a token back to the stream."""
-        self._pushed.append(token)
-
-    def look(self) -> Token:
-        """Look at the next token."""
-        old_token = next(self)
-        result = self.current
-        self.push(result)
-        self.current = old_token
-        return result
-
-    def skip(self, n: int = 1) -> None:
-        """Got n tokens ahead."""
-        for _ in range(n):
-            next(self)
-
-    def next_if(self, expr: str) -> t.Optional[Token]:
-        """Perform the token test and return the token if it matched.
-        Otherwise the return value is `None`.
-        """
-        if self.current.test(expr):
-            return next(self)
-
-        return None
-
-    def skip_if(self, expr: str) -> bool:
-        """Like :meth:`next_if` but only returns `True` or `False`."""
-        return self.next_if(expr) is not None
-
-    def __next__(self) -> Token:
-        """Go one token ahead and return the old one.
-
-        Use the built-in :func:`next` instead of calling this directly.
-        """
-        rv = self.current
-
-        if self._pushed:
-            self.current = self._pushed.popleft()
-        elif self.current.type is not TOKEN_EOF:
-            try:
-                self.current = next(self._iter)
-            except StopIteration:
-                self.close()
-
-        return rv
-
-    def close(self) -> None:
-        """Close the stream."""
-        self.current = Token(self.current.lineno, TOKEN_EOF, "")
-        self._iter = iter(())
-        self.closed = True
-
-    def expect(self, expr: str) -> Token:
-        """Expect a given token type and return it.  This accepts the same
-        argument as :meth:`jinja2.lexer.Token.test`.
-        """
-        if not self.current.test(expr):
-            expr = describe_token_expr(expr)
-
-            if self.current.type is TOKEN_EOF:
-                raise TemplateSyntaxError(
-                    f"unexpected end of template, expected {expr!r}.",
-                    self.current.lineno,
-                    self.name,
-                    self.filename,
-                )
-
-            raise TemplateSyntaxError(
-                f"expected token {expr!r}, got {describe_token(self.current)!r}",
-                self.current.lineno,
-                self.name,
-                self.filename,
-            )
-
-        return next(self)
-
-
-def get_lexer(environment: "Environment") -> "Lexer":
-    """Return a lexer which is probably cached."""
-    key = (
-        environment.block_start_string,
-        environment.block_end_string,
-        environment.variable_start_string,
-        environment.variable_end_string,
-        environment.comment_start_string,
-        environment.comment_end_string,
-        environment.line_statement_prefix,
-        environment.line_comment_prefix,
-        environment.trim_blocks,
-        environment.lstrip_blocks,
-        environment.newline_sequence,
-        environment.keep_trailing_newline,
-    )
-    lexer = _lexer_cache.get(key)
-
-    if lexer is None:
-        _lexer_cache[key] = lexer = Lexer(environment)
-
-    return lexer
-
-
-class OptionalLStrip(tuple):
-    """A special tuple for marking a point in the state that can have
-    lstrip applied.
-    """
-
-    __slots__ = ()
-
-    # Even though it looks like a no-op, creating instances fails
-    # without this.
-    def __new__(cls, *members, **kwargs):  # type: ignore
-        return super().__new__(cls, members)
-
-
-class _Rule(t.NamedTuple):
-    pattern: t.Pattern[str]
-    tokens: t.Union[str, t.Tuple[str, ...], t.Tuple[Failure]]
-    command: t.Optional[str]
-
-
-class Lexer:
-    """Class that implements a lexer for a given environment. Automatically
-    created by the environment class, usually you don't have to do that.
-
-    Note that the lexer is not automatically bound to an environment.
-    Multiple environments can share the same lexer.
-    """
-
-    def __init__(self, environment: "Environment") -> None:
-        # shortcuts
-        e = re.escape
-
-        def c(x: str) -> t.Pattern[str]:
-            return re.compile(x, re.M | re.S)
-
-        # lexing rules for tags
-        tag_rules: t.List[_Rule] = [
-            _Rule(whitespace_re, TOKEN_WHITESPACE, None),
-            _Rule(float_re, TOKEN_FLOAT, None),
-            _Rule(integer_re, TOKEN_INTEGER, None),
-            _Rule(name_re, TOKEN_NAME, None),
-            _Rule(string_re, TOKEN_STRING, None),
-            _Rule(operator_re, TOKEN_OPERATOR, None),
-        ]
-
-        # assemble the root lexing rule. because "|" is ungreedy
-        # we have to sort by length so that the lexer continues working
-        # as expected when we have parsing rules like <% for block and
-        # <%= for variables. (if someone wants asp like syntax)
-        # variables are just part of the rules if variable processing
-        # is required.
-        root_tag_rules = compile_rules(environment)
-
-        block_start_re = e(environment.block_start_string)
-        block_end_re = e(environment.block_end_string)
-        comment_end_re = e(environment.comment_end_string)
-        variable_end_re = e(environment.variable_end_string)
-
-        # block suffix if trimming is enabled
-        block_suffix_re = "\\n?" if environment.trim_blocks else ""
-
-        # If lstrip is enabled, it should not be applied if there is any
-        # non-whitespace between the newline and block.
-        self.lstrip_unless_re = c(r"[^ \t]") if environment.lstrip_blocks else None
-
-        self.newline_sequence = environment.newline_sequence
-        self.keep_trailing_newline = environment.keep_trailing_newline
-
-        root_raw_re = (
-            fr"(?P<raw_begin>{block_start_re}(\-|\+|)\s*raw\s*"
-            fr"(?:\-{block_end_re}\s*|{block_end_re}))"
-        )
-        root_parts_re = "|".join(
-            [root_raw_re] + [fr"(?P<{n}>{r}(\-|\+|))" for n, r in root_tag_rules]
-        )
-
-        # global lexing rules
-        self.rules: t.Dict[str, t.List[_Rule]] = {
-            "root": [
-                # directives
-                _Rule(
-                    c(fr"(.*?)(?:{root_parts_re})"),
-                    OptionalLStrip(TOKEN_DATA, "#bygroup"),  # type: ignore
-                    "#bygroup",
-                ),
-                # data
-                _Rule(c(".+"), TOKEN_DATA, None),
-            ],
-            # comments
-            TOKEN_COMMENT_BEGIN: [
-                _Rule(
-                    c(
-                        fr"(.*?)((?:\+{comment_end_re}|\-{comment_end_re}\s*"
-                        fr"|{comment_end_re}{block_suffix_re}))"
-                    ),
-                    (TOKEN_COMMENT, TOKEN_COMMENT_END),
-                    "#pop",
-                ),
-                _Rule(c(r"(.)"), (Failure("Missing end of comment tag"),), None),
-            ],
-            # blocks
-            TOKEN_BLOCK_BEGIN: [
-                _Rule(
-                    c(
-                        fr"(?:\+{block_end_re}|\-{block_end_re}\s*"
-                        fr"|{block_end_re}{block_suffix_re})"
-                    ),
-                    TOKEN_BLOCK_END,
-                    "#pop",
-                ),
-            ]
-            + tag_rules,
-            # variables
-            TOKEN_VARIABLE_BEGIN: [
-                _Rule(
-                    c(fr"\-{variable_end_re}\s*|{variable_end_re}"),
-                    TOKEN_VARIABLE_END,
-                    "#pop",
-                )
-            ]
-            + tag_rules,
-            # raw block
-            TOKEN_RAW_BEGIN: [
-                _Rule(
-                    c(
-                        fr"(.*?)((?:{block_start_re}(\-|\+|))\s*endraw\s*"
-                        fr"(?:\+{block_end_re}|\-{block_end_re}\s*"
-                        fr"|{block_end_re}{block_suffix_re}))"
-                    ),
-                    OptionalLStrip(TOKEN_DATA, TOKEN_RAW_END),  # type: ignore
-                    "#pop",
-                ),
-                _Rule(c(r"(.)"), (Failure("Missing end of raw directive"),), None),
-            ],
-            # line statements
-            TOKEN_LINESTATEMENT_BEGIN: [
-                _Rule(c(r"\s*(\n|$)"), TOKEN_LINESTATEMENT_END, "#pop")
-            ]
-            + tag_rules,
-            # line comments
-            TOKEN_LINECOMMENT_BEGIN: [
-                _Rule(
-                    c(r"(.*?)()(?=\n|$)"),
-                    (TOKEN_LINECOMMENT, TOKEN_LINECOMMENT_END),
-                    "#pop",
-                )
-            ],
-        }
-
-    def _normalize_newlines(self, value: str) -> str:
-        """Replace all newlines with the configured sequence in strings
-        and template data.
-        """
-        return newline_re.sub(self.newline_sequence, value)
-
-    def tokenize(
-        self,
-        source: str,
-        name: t.Optional[str] = None,
-        filename: t.Optional[str] = None,
-        state: t.Optional[str] = None,
-    ) -> TokenStream:
-        """Calls tokeniter + tokenize and wraps it in a token stream."""
-        stream = self.tokeniter(source, name, filename, state)
-        return TokenStream(self.wrap(stream, name, filename), name, filename)
-
-    def wrap(
-        self,
-        stream: t.Iterable[t.Tuple[int, str, str]],
-        name: t.Optional[str] = None,
-        filename: t.Optional[str] = None,
-    ) -> t.Iterator[Token]:
-        """This is called with the stream as returned by `tokenize` and wraps
-        every token in a :class:`Token` and converts the value.
-        """
-        for lineno, token, value_str in stream:
-            if token in ignored_tokens:
-                continue
-
-            value: t.Any = value_str
-
-            if token == TOKEN_LINESTATEMENT_BEGIN:
-                token = TOKEN_BLOCK_BEGIN
-            elif token == TOKEN_LINESTATEMENT_END:
-                token = TOKEN_BLOCK_END
-            # we are not interested in those tokens in the parser
-            elif token in (TOKEN_RAW_BEGIN, TOKEN_RAW_END):
-                continue
-            elif token == TOKEN_DATA:
-                value = self._normalize_newlines(value_str)
-            elif token == "keyword":
-                token = value_str
-            elif token == TOKEN_NAME:
-                value = value_str
-
-                if not value.isidentifier():
-                    raise TemplateSyntaxError(
-                        "Invalid character in identifier", lineno, name, filename
-                    )
-            elif token == TOKEN_STRING:
-                # try to unescape string
-                try:
-                    value = (
-                        self._normalize_newlines(value_str[1:-1])
-                        .encode("ascii", "backslashreplace")
-                        .decode("unicode-escape")
-                    )
-                except Exception as e:
-                    msg = str(e).split(":")[-1].strip()
-                    raise TemplateSyntaxError(msg, lineno, name, filename) from e
-            elif token == TOKEN_INTEGER:
-                value = int(value_str.replace("_", ""), 0)
-            elif token == TOKEN_FLOAT:
-                # remove all "_" first to support more Python versions
-                value = literal_eval(value_str.replace("_", ""))
-            elif token == TOKEN_OPERATOR:
-                token = operators[value_str]
-
-            yield Token(lineno, token, value)
-
-    def tokeniter(
-        self,
-        source: str,
-        name: t.Optional[str],
-        filename: t.Optional[str] = None,
-        state: t.Optional[str] = None,
-    ) -> t.Iterator[t.Tuple[int, str, str]]:
-        """This method tokenizes the text and returns the tokens in a
-        generator. Use this method if you just want to tokenize a template.
-
-        .. versionchanged:: 3.0
-            Only ``\\n``, ``\\r\\n`` and ``\\r`` are treated as line
-            breaks.
-        """
-        lines = newline_re.split(source)[::2]
-
-        if not self.keep_trailing_newline and lines[-1] == "":
-            del lines[-1]
-
-        source = "\n".join(lines)
-        pos = 0
-        lineno = 1
-        stack = ["root"]
-
-        if state is not None and state != "root":
-            assert state in ("variable", "block"), "invalid state"
-            stack.append(state + "_begin")
-
-        statetokens = self.rules[stack[-1]]
-        source_length = len(source)
-        balancing_stack: t.List[str] = []
-        lstrip_unless_re = self.lstrip_unless_re
-        newlines_stripped = 0
-        line_starting = True
-
-        while True:
-            # tokenizer loop
-            for regex, tokens, new_state in statetokens:
-                m = regex.match(source, pos)
-
-                # if no match we try again with the next rule
-                if m is None:
-                    continue
-
-                # we only match blocks and variables if braces / parentheses
-                # are balanced. continue parsing with the lower rule which
-                # is the operator rule. do this only if the end tags look
-                # like operators
-                if balancing_stack and tokens in (
-                    TOKEN_VARIABLE_END,
-                    TOKEN_BLOCK_END,
-                    TOKEN_LINESTATEMENT_END,
-                ):
-                    continue
-
-                # tuples support more options
-                if isinstance(tokens, tuple):
-                    groups = m.groups()
-
-                    if isinstance(tokens, OptionalLStrip):
-                        # Rule supports lstrip. Match will look like
-                        # text, block type, whitespace control, type, control, ...
-                        text = groups[0]
-                        # Skipping the text and first type, every other group is the
-                        # whitespace control for each type. One of the groups will be
-                        # -, +, or empty string instead of None.
-                        strip_sign = next(g for g in groups[2::2] if g is not None)
-
-                        if strip_sign == "-":
-                            # Strip all whitespace between the text and the tag.
-                            stripped = text.rstrip()
-                            newlines_stripped = text[len(stripped) :].count("\n")
-                            groups = [stripped, *groups[1:]]
-                        elif (
-                            # Not marked for preserving whitespace.
-                            strip_sign != "+"
-                            # lstrip is enabled.
-                            and lstrip_unless_re is not None
-                            # Not a variable expression.
-                            and not m.groupdict().get(TOKEN_VARIABLE_BEGIN)
-                        ):
-                            # The start of text between the last newline and the tag.
-                            l_pos = text.rfind("\n") + 1
-
-                            if l_pos > 0 or line_starting:
-                                # If there's only whitespace between the newline and the
-                                # tag, strip it.
-                                if not lstrip_unless_re.search(text, l_pos):
-                                    groups = [text[:l_pos], *groups[1:]]
-
-                    for idx, token in enumerate(tokens):
-                        # failure group
-                        if token.__class__ is Failure:
-                            raise token(lineno, filename)
-                        # bygroup is a bit more complex, in that case we
-                        # yield for the current token the first named
-                        # group that matched
-                        elif token == "#bygroup":
-                            for key, value in m.groupdict().items():
-                                if value is not None:
-                                    yield lineno, key, value
-                                    lineno += value.count("\n")
-                                    break
-                            else:
-                                raise RuntimeError(
-                                    f"{regex!r} wanted to resolve the token dynamically"
-                                    " but no group matched"
-                                )
-                        # normal group
-                        else:
-                            data = groups[idx]
-
-                            if data or token not in ignore_if_empty:
-                                yield lineno, token, data
-
-                            lineno += data.count("\n") + newlines_stripped
-                            newlines_stripped = 0
-
-                # strings as token just are yielded as it.
-                else:
-                    data = m.group()
-
-                    # update brace/parentheses balance
-                    if tokens == TOKEN_OPERATOR:
-                        if data == "{":
-                            balancing_stack.append("}")
-                        elif data == "(":
-                            balancing_stack.append(")")
-                        elif data == "[":
-                            balancing_stack.append("]")
-                        elif data in ("}", ")", "]"):
-                            if not balancing_stack:
-                                raise TemplateSyntaxError(
-                                    f"unexpected '{data}'", lineno, name, filename
-                                )
-
-                            expected_op = balancing_stack.pop()
-
-                            if expected_op != data:
-                                raise TemplateSyntaxError(
-                                    f"unexpected '{data}', expected '{expected_op}'",
-                                    lineno,
-                                    name,
-                                    filename,
-                                )
-
-                    # yield items
-                    if data or tokens not in ignore_if_empty:
-                        yield lineno, tokens, data
-
-                    lineno += data.count("\n")
-
-                line_starting = m.group()[-1:] == "\n"
-                # fetch new position into new variable so that we can check
-                # if there is a internal parsing error which would result
-                # in an infinite loop
-                pos2 = m.end()
-
-                # handle state changes
-                if new_state is not None:
-                    # remove the uppermost state
-                    if new_state == "#pop":
-                        stack.pop()
-                    # resolve the new state by group checking
-                    elif new_state == "#bygroup":
-                        for key, value in m.groupdict().items():
-                            if value is not None:
-                                stack.append(key)
-                                break
-                        else:
-                            raise RuntimeError(
-                                f"{regex!r} wanted to resolve the new state dynamically"
-                                f" but no group matched"
-                            )
-                    # direct state name given
-                    else:
-                        stack.append(new_state)
-
-                    statetokens = self.rules[stack[-1]]
-                # we are still at the same position and no stack change.
-                # this means a loop without break condition, avoid that and
-                # raise error
-                elif pos2 == pos:
-                    raise RuntimeError(
-                        f"{regex!r} yielded empty string without stack change"
-                    )
-
-                # publish new function and start again
-                pos = pos2
-                break
-            # if loop terminated without break we haven't found a single match
-            # either we are at the end of the file or we have a problem
-            else:
-                # end of text
-                if pos >= source_length:
-                    return
-
-                # something went wrong
-                raise TemplateSyntaxError(
-                    f"unexpected char {source[pos]!r} at {pos}", lineno, name, filename
-                )