pbi-parsers 0.7.8__py3-none-any.whl

pbi_parsers/dax/formatter.py

@@ -0,0 +1,215 @@
+import string
+import textwrap
+from typing import Any
+
+from .exprs import (
+    AddSubExpression,
+    AddSubUnaryExpression,
+    ArrayExpression,
+    ColumnExpression,
+    ComparisonExpression,
+    ConcatenationExpression,
+    DivMulExpression,
+    ExponentExpression,
+    Expression,
+    FunctionExpression,
+    HierarchyExpression,
+    IdentifierExpression,
+    InExpression,
+    KeywordExpression,
+    LiteralNumberExpression,
+    LiteralStringExpression,
+    LogicalExpression,
+    MeasureExpression,
+    NoneExpression,
+    ParenthesesExpression,
+    ReturnExpression,
+    TableExpression,
+    VariableExpression,
+)
+from .tokens import Token
+
+MAX_ARGUMENT_LENGTH = 40  # Maximum length of arguments before formatting them on new lines
+
+
+def format_comments(comments: list[Token], indent_chars: int) -> str:
+    """Concatenates a list of comments into a single string."""
+    base = "\n".join(comment.text_slice.get_text().strip() for comment in comments)
+    return textwrap.indent(base, " " * indent_chars)
+
+
+class Formatter:
+    """Formats a DAX expression into a standardized format."""
+
+    def __init__(self, expression: "Expression") -> None:
+        self.expression = expression
+
+    def format(self) -> str:
+        return self._format_helper(self.expression)
+
+    @classmethod
+    def _format_helper(cls, expr: Expression) -> str:
+        mapper: Any = {
+            AddSubExpression: cls._format_add_sub,
+            AddSubUnaryExpression: cls._format_add_sub_unary,
+            ArrayExpression: cls._format_array,
+            ComparisonExpression: cls._format_comparison,
+            ColumnExpression: cls._format_column,
+            ConcatenationExpression: cls._format_concatenation,
+            DivMulExpression: cls._format_div_mul,
+            ExponentExpression: cls._format_exponent,
+            FunctionExpression: cls._format_function,
+            HierarchyExpression: cls._format_hierarchy,
+            IdentifierExpression: cls._format_identifier,
+            InExpression: cls._format_in,
+            KeywordExpression: cls._format_keyword,
+            LiteralNumberExpression: cls._format_literal_number,
+            LiteralStringExpression: cls._format_literal_string,
+            LogicalExpression: cls._format_logical,
+            MeasureExpression: cls._format_measure,
+            NoneExpression: lambda _: "",
+            ParenthesesExpression: cls._format_parens,
+            ReturnExpression: cls._format_return,
+            TableExpression: cls._format_table,
+            VariableExpression: cls._format_variable,
+        }
+        if type(expr) in mapper:
+            base_format = mapper[type(expr)](expr)
+            if expr.pre_comments:
+                base_format = f"{format_comments(expr.pre_comments, 0)}\n{base_format}"
+            if expr.post_comments:
+                base_format = f"{base_format}  {format_comments(expr.post_comments, 0)}"
+            return base_format
+
+        msg = f"Unsupported expression type: {type(expr).__name__}"
+        raise TypeError(msg)
+
+    @classmethod
+    def _format_add_sub(cls, expr: AddSubExpression) -> str:
+        left = cls._format_helper(expr.left)
+        right = cls._format_helper(expr.right)
+        return f"""{left} {expr.operator.text} {right}"""
+
+    @classmethod
+    def _format_add_sub_unary(cls, expr: AddSubUnaryExpression) -> str:
+        return f"{expr.operator.text}{cls._format_helper(expr.number)}"
+
+    @classmethod
+    def _format_array(cls, expr: ArrayExpression) -> str:
+        elements = ",\n".join(cls._format_helper(el) for el in expr.elements)
+        elements = textwrap.indent(elements, " " * 4)[4:]
+        return f"""{{
+    {elements}
+}}
+"""
+
+    @classmethod
+    def _format_comparison(cls, expr: ComparisonExpression) -> str:
+        left = cls._format_helper(expr.left)
+        right = cls._format_helper(expr.right)
+        return f"""{left} {expr.operator.text} {right}"""
+
+    @classmethod
+    def _format_column(cls, expr: ColumnExpression) -> str:
+        table = expr.table.text
+        if table.startswith("'") and all(c in string.ascii_letters + string.digits + "_" for c in table[1:-1]):
+            table = table[1:-1]
+        column = expr.column.text
+        return f"{table}{column}"
+
+    @classmethod
+    def _format_concatenation(cls, expr: ConcatenationExpression) -> str:
+        left = cls._format_helper(expr.left)
+        right = cls._format_helper(expr.right)
+        return f"""{left} {expr.operator.text} {right}"""
+
+    @classmethod
+    def _format_div_mul(cls, expr: DivMulExpression) -> str:
+        left = cls._format_helper(expr.left)
+        right = cls._format_helper(expr.right)
+        return f"""{left} {expr.operator.text} {right}"""
+
+    @classmethod
+    def _format_exponent(cls, expr: ExponentExpression) -> str:
+        base = cls._format_helper(expr.base)
+        power = cls._format_helper(expr.power)
+        return f"""{base}^{power}"""
+
+    @classmethod
+    def _format_function(cls, expr: FunctionExpression) -> str:
+        name = "".join(token.text for token in expr.name_parts)
+        args = [cls._format_helper(arg) for arg in expr.args]
+        if sum(len(x) for x in args) < MAX_ARGUMENT_LENGTH:
+            arg_str = ", ".join(args)
+            return f"{name}({arg_str})"
+        arg_str = textwrap.indent(",\n".join(args), " " * 4)[4:]
+        return f"""
+{name}(
+    {arg_str}
+)""".strip()
+
+    @classmethod
+    def _format_hierarchy(cls, expr: HierarchyExpression) -> str:
+        table = expr.table.text
+        if table.startswith("'") and all(c in string.ascii_letters + string.digits + "_" for c in table[1:-1]):
+            table = table[1:-1]
+        return f"{table}{expr.column.text}.{expr.level.text}"
+
+    @classmethod
+    def _format_identifier(cls, expr: IdentifierExpression) -> str:
+        return expr.name.text
+
+    @classmethod
+    def _format_in(cls, expr: InExpression) -> str:
+        value = cls._format_helper(expr.value)
+        array = cls._format_helper(expr.array)
+        return f"""{value} IN {array}"""
+
+    @classmethod
+    def _format_keyword(cls, expr: KeywordExpression) -> str:
+        return expr.name.text
+
+    @classmethod
+    def _format_literal_string(cls, expr: LiteralStringExpression) -> str:
+        return expr.value.text
+
+    @classmethod
+    def _format_literal_number(cls, expr: LiteralNumberExpression) -> str:
+        return expr.value.text
+
+    @classmethod
+    def _format_logical(cls, expr: LogicalExpression) -> str:
+        left = cls._format_helper(expr.left)
+        right = cls._format_helper(expr.right)
+        return f"""{left} {expr.operator.text} {right}"""
+
+    @classmethod
+    def _format_measure(cls, expr: MeasureExpression) -> str:
+        return expr.name.text
+
+    @classmethod
+    def _format_parens(cls, expr: ParenthesesExpression) -> str:
+        inner = cls._format_helper(expr.inner_statement)
+        return f"({inner})"
+
+    @classmethod
+    def _format_return(cls, expr: ReturnExpression) -> str:
+        variable_strs = "\n".join(cls._format_helper(var) for var in expr.variable_statements)
+        return_statement: str = cls._format_helper(expr.ret)
+        return f"""
+{variable_strs}
+RETURN {return_statement}
+""".strip()
+
+    @classmethod
+    def _format_table(cls, expr: TableExpression) -> str:
+        table_name = expr.name.text
+        if table_name.startswith("'") and all(
+            c in string.ascii_letters + string.digits + "_" for c in table_name[1:-1]
+        ):
+            table_name = table_name[1:-1]
+        return table_name
+
+    @classmethod
+    def _format_variable(cls, expr: VariableExpression) -> str:
+        return f"{expr.var_name.text} = {cls._format_helper(expr.statement)}"

pbi_parsers/dax/lexer.py

@@ -0,0 +1,222 @@
+from pbi_parsers.base import BaseLexer
+from pbi_parsers.base.tokens import TextSlice
+
+from .tokens import KEYWORD_MAPPING, Token, TokenType
+
+WHITESPACE = ["\n", "\r", "\t", " ", "\f", "\v"]
+
+
+class Lexer(BaseLexer):
+    def scan(self) -> tuple[Token]:
+        return super().scan()  # type: ignore[override]
+
+    def create_token(self, tok_type: TokenType, start_pos: int) -> Token:
+        """Create a new token with the given type and text."""
+        text_slice = TextSlice(
+            full_text=self.source,
+            start=start_pos,
+            end=self.current_position,
+        )
+        return Token(tok_type=tok_type, text_slice=text_slice)
+
+    def _match_in(self, start_pos: int) -> Token | None:
+        if self.match(
+            "in ",
+            case_insensitive=True,
+        ):  # I have found no case where "in" is not followed by a space
+            # this allows us to avoid matching with the "int" function
+            self.advance(-1)  # leave the space to be consumed by whitespace handling
+            return self.create_token(
+                tok_type=TokenType.IN,
+                start_pos=start_pos,
+            )
+        return None
+
+    def _match_keyword(self, start_pos: int) -> Token | None:
+        for keyword, token_type in KEYWORD_MAPPING.items():
+            if self.match(keyword, case_insensitive=True):
+                return self.create_token(
+                    tok_type=token_type,
+                    start_pos=start_pos,
+                )
+        return None
+
+    def _match_whitespace(self, start_pos: int) -> Token | None:
+        if self.match(lambda c: c in WHITESPACE):
+            while self.match(lambda c: c in WHITESPACE):
+                pass
+            return self.create_token(
+                tok_type=TokenType.WHITESPACE,
+                start_pos=start_pos,
+            )
+        return None
+
+    def _match_var(self, start_pos: int) -> Token | None:
+        if self.match("var", case_insensitive=True):
+            return self.create_token(
+                tok_type=TokenType.VARIABLE,
+                start_pos=start_pos,
+            )
+        return None
+
+    def _match_return(self, start_pos: int) -> Token | None:
+        if self.match("return", case_insensitive=True):
+            return self.create_token(
+                tok_type=TokenType.RETURN,
+                start_pos=start_pos,
+            )
+        return None
+
+    def _match_period(self, start_pos: int) -> Token | None:
+        if self.match("."):
+            # must come before number literal to avoid conflict
+            return self.create_token(
+                tok_type=TokenType.PERIOD,
+                start_pos=start_pos,
+            )
+        return None
+
+    def _match_number_literal(self, start_pos: int) -> Token | None:
+        if self.match(
+            lambda c: c.isdigit() or c == ".",
+        ):  # must come before unquoted identifier to avoid conflict
+            while self.match(lambda c: c.isdigit() or c in {".", "e", "E"}):
+                pass
+            return self.create_token(
+                tok_type=TokenType.NUMBER_LITERAL,
+                start_pos=start_pos,
+            )
+        return None
+
+    def _match_unquoted_identifier(self, start_pos: int) -> Token | None:
+        if self.match(lambda c: c.isalnum() or c == "_"):
+            while self.match(lambda c: c.isalnum() or c == "_"):
+                pass
+            return self.create_token(
+                tok_type=TokenType.UNQUOTED_IDENTIFIER,
+                start_pos=start_pos,
+            )
+        return None
+
+    def _match_single_quoted_identifier(self, start_pos: int) -> Token | None:
+        if self.match("'"):
+            while self.match(lambda c: c != "'"):
+                pass
+            if self.match("'"):
+                return self.create_token(
+                    tok_type=TokenType.SINGLE_QUOTED_IDENTIFIER,
+                    start_pos=start_pos,
+                )
+            msg = "Unterminated string literal"
+            raise ValueError(msg)
+        return None
+
+    def _match_bracketed_identifier(self, start_pos: int) -> Token | None:
+        if self.match("["):
+            while self.match(lambda c: c != "]"):
+                pass
+            if self.match("]"):
+                return self.create_token(
+                    tok_type=TokenType.BRACKETED_IDENTIFIER,
+                    start_pos=start_pos,
+                )
+            msg = "Unterminated bracketed identifier"
+            raise ValueError(msg)
+        return None
+
+    def _match_string_literal(self, start_pos: int) -> Token | None:
+        if self.match('"'):
+            while self.match(lambda c: c != '"') or self.match('""'):
+                pass
+            if self.match('"'):
+                return self.create_token(
+                    tok_type=TokenType.STRING_LITERAL,
+                    start_pos=start_pos,
+                )
+            msg = "Unterminated string literal"
+            raise ValueError(msg)
+        return None
+
+    def _match_single_line_comment(self, start_pos: int) -> Token | None:
+        if self.match("//") or self.match("--"):
+            while self.match(lambda c: c not in {"\n", ""}):
+                pass
+            return self.create_token(
+                tok_type=TokenType.SINGLE_LINE_COMMENT,
+                start_pos=start_pos,
+            )
+        return None
+
+    def _match_multi_line_comment(self, start_pos: int) -> Token | None:
+        if not self.match("/*"):
+            return None
+
+        while not self.at_end():
+            if self.match("*/", chunk=2):
+                return self.create_token(
+                    tok_type=TokenType.MULTI_LINE_COMMENT,
+                    start_pos=start_pos,
+                )
+            self.advance()
+
+        msg = "Unterminated multi-line comment"
+        raise ValueError(msg)
+
+    def _match_token(self, start_pos: int) -> Token | None:
+        fixed_character_mapping = {
+            "(": TokenType.LEFT_PAREN,
+            ")": TokenType.RIGHT_PAREN,
+            ",": TokenType.COMMA,
+            "==": TokenType.EQUAL_SIGN,
+            "=": TokenType.EQUAL_SIGN,
+            "{": TokenType.LEFT_CURLY_BRACE,
+            "}": TokenType.RIGHT_CURLY_BRACE,
+            "<>": TokenType.NOT_EQUAL_SIGN,
+            "<=": TokenType.COMPARISON_OPERATOR,
+            "<": TokenType.COMPARISON_OPERATOR,
+            ">=": TokenType.COMPARISON_OPERATOR,
+            ">": TokenType.COMPARISON_OPERATOR,
+            "||": TokenType.DOUBLE_PIPE_OPERATOR,
+            "&&": TokenType.DOUBLE_AMPERSAND_OPERATOR,
+            "&": TokenType.AMPERSAND_OPERATOR,
+            "+": TokenType.PLUS_SIGN,
+            "-": TokenType.MINUS_SIGN,
+            "^": TokenType.EXPONENTIATION_SIGN,
+            "*": TokenType.MULTIPLY_SIGN,
+            "%": TokenType.MODULUS_SIGN,
+            "/": TokenType.DIVIDE_SIGN,
+        }
+
+        for char, token_type in fixed_character_mapping.items():
+            if self.match(char):
+                return self.create_token(tok_type=token_type, start_pos=start_pos)
+        return None
+
+    def scan_helper(self) -> Token:
+        start_pos: int = self.current_position
+
+        if not self.peek():
+            return Token()
+
+        for candidate_func in (
+            self._match_in,
+            self._match_keyword,
+            self._match_whitespace,
+            self._match_var,
+            self._match_return,
+            self._match_period,
+            self._match_number_literal,
+            self._match_unquoted_identifier,
+            self._match_single_quoted_identifier,
+            self._match_bracketed_identifier,
+            self._match_string_literal,
+            self._match_single_line_comment,
+            self._match_multi_line_comment,
+            self._match_token,
+        ):
+            match_candidate = candidate_func(start_pos)
+            if match_candidate:
+                return match_candidate
+
+        msg = f"Unexpected character: {self.peek()} at position {self.current_position}"
+        raise ValueError(msg)

pbi_parsers/dax/main.py

@@ -0,0 +1,63 @@
+from collections.abc import Iterable
+
+from .exprs._base import Expression
+from .formatter import Formatter
+from .lexer import Lexer
+from .parser import Parser
+from .tokens import Token, TokenType
+
+
+def remove_non_executing_tokens(tokens: Iterable[Token]) -> list[Token]:
+    """Removes tokens that are not executed in the DAX expression.
+
+    Args:
+        tokens (Iterable[Token]): List of tokens to filter.
+
+    Returns:
+        list[Token]: Filtered list of tokens that are executed.
+
+    """
+    return list(
+        filter(
+            lambda x: x.tok_type
+            not in {
+                TokenType.WHITESPACE,
+                TokenType.SINGLE_LINE_COMMENT,
+                TokenType.MULTI_LINE_COMMENT,
+            },
+            tokens,
+        ),
+    )
+
+
+def to_ast(text: str) -> Expression | None:
+    """Converts a DAX expression string into an AST (Abstract Syntax Tree).
+
+    Args:
+        text (str): The DAX expression to parse.
+
+    Returns:
+        Expression | None: when matched, returns the root node of the AST representing the DAX expression.
+            When not matched, returns None.
+
+    """
+    tokens = Lexer(text).scan()
+    tokens = remove_non_executing_tokens(tokens)
+    parser = Parser(tokens)
+    return parser.to_ast()
+
+
+def format_expression(text: str) -> str:
+    """Formats a DAX expression string into a more readable format.
+
+    Args:
+        text (str): The DAX expression to format.
+
+    Returns:
+        str: The formatted DAX expression.
+
+    """
+    ast = to_ast(text)
+    if ast is None:
+        return text
+    return Formatter(ast).format()

pbi_parsers/dax/parser.py

@@ -0,0 +1,66 @@
+from typing import TYPE_CHECKING, Any
+
+from .tokens import Token, TokenType
+
+if TYPE_CHECKING:
+    from .exprs import Expression
+
+EOF_TOKEN = Token()
+
+
+class Parser:
+    __tokens: list[Token]
+    index: int = 0
+    cache: dict[Any, Any]
+
+    def __init__(self, tokens: list[Token]) -> None:
+        self.__tokens = tokens
+        self.index = 0
+        self.cache = {}
+
+    def peek(self, forward: int = 0) -> Token:
+        """Peek at the next token without advancing the index.
+
+        Args:
+            forward (int): How many tokens to look ahead. Defaults to 0.
+
+        Returns:
+            Token: The token at the current index + forward.
+
+        """
+        if self.index + forward >= len(self.__tokens):
+            return EOF_TOKEN
+        return self.__tokens[self.index + forward]
+
+    def remaining(self) -> list[Token]:
+        """Returns the remaining tokens from the current index.
+
+        Returns:
+            list[Token]: The list of tokens from the current index to the end.
+
+        """
+        return self.__tokens[self.index :]
+
+    def to_ast(self) -> "Expression | None":
+        """Parse the tokens and return the root expression.
+
+        Raises:
+            ValueError: If no valid expression is found in the token stream.
+
+        """
+        from .exprs import any_expression_match  # noqa: PLC0415
+
+        ret = any_expression_match(self)
+        if ret is None:
+            msg = "No valid expression found in the token stream."
+            raise ValueError(msg)
+        assert self.peek().tok_type == TokenType.EOF
+        return ret
+
+    def consume(self) -> Token:
+        """Returns the next token and advances the index."""
+        if self.index >= len(self.__tokens):
+            return EOF_TOKEN
+        ret = self.__tokens[self.index]
+        self.index += 1
+        return ret

pbi_parsers/dax/tokens.py

@@ -0,0 +1,54 @@
+from dataclasses import dataclass
+from enum import Enum, auto
+
+from pbi_parsers.base import BaseToken
+
+
+class TokenType(Enum):
+    AMPERSAND_OPERATOR = auto()
+    ASC = auto()
+    BRACKETED_IDENTIFIER = auto()
+    COMMA = auto()
+    COMPARISON_OPERATOR = auto()
+    DESC = auto()
+    DIVIDE_SIGN = auto()
+    DOUBLE_AMPERSAND_OPERATOR = auto()
+    DOUBLE_PIPE_OPERATOR = auto()
+    EOF = auto()
+    EQUAL_SIGN = auto()
+    EXPONENTIATION_SIGN = auto()
+    FALSE = auto()
+    IN = auto()
+    LEFT_CURLY_BRACE = auto()
+    LEFT_PAREN = auto()
+    MINUS_SIGN = auto()
+    MODULUS_SIGN = auto()
+    MULTIPLY_SIGN = auto()
+    MULTI_LINE_COMMENT = auto()
+    NOT_EQUAL_SIGN = auto()
+    NUMBER_LITERAL = auto()
+    PERIOD = auto()
+    PLUS_SIGN = auto()
+    RETURN = auto()
+    RIGHT_CURLY_BRACE = auto()
+    RIGHT_PAREN = auto()
+    SINGLE_LINE_COMMENT = auto()
+    SINGLE_QUOTED_IDENTIFIER = auto()
+    STRING_LITERAL = auto()
+    TRUE = auto()
+    UNQUOTED_IDENTIFIER = auto()
+    VARIABLE = auto()
+    WHITESPACE = auto()
+
+
+KEYWORD_MAPPING = {
+    "TRUE": TokenType.TRUE,
+    "FALSE": TokenType.FALSE,
+    "ASC": TokenType.ASC,
+    "DESC": TokenType.DESC,
+}
+
+
+@dataclass
+class Token(BaseToken):
+    tok_type: TokenType = TokenType.EOF