pbi-parsers 0.7.8__py3-none-any.whl

pbi_parsers/__init__.py

@@ -0,0 +1,9 @@
+from . import dax, pq
+
+__version__ = "0.7.8"
+
+
+__all__ = [
+    "dax",
+    "pq",
+]

pbi_parsers/base/__init__.py

@@ -0,0 +1,7 @@
+from .lexer import BaseLexer
+from .tokens import BaseToken
+
+__all__ = [
+    "BaseLexer",
+    "BaseToken",
+]

pbi_parsers/base/lexer.py

@@ -0,0 +1,127 @@
+from collections.abc import Callable
+
+from .tokens import BaseToken
+
+MAX_POSITION = 1_000_000
+
+
+class BaseLexer:
+    source: str
+    start_position: int
+    current_position: int
+    tokens: list[BaseToken]
+
+    def __init__(self, source: str) -> None:
+        self.source = source
+        self.start_position = 0
+        self.current_position = 0
+        self.tokens = []
+
+    def scan_helper(self) -> BaseToken:
+        """Contains the orchestration logic for converting tokens into expressions."""
+        msg = "Subclasses should implement match_tokens method."
+        raise NotImplementedError(msg)
+
+    def match(
+        self,
+        matcher: Callable[[str], bool] | str,
+        chunk: int = 1,
+        *,
+        case_insensitive: bool = True,
+    ) -> bool:
+        """Match a string or a callable matcher against the current position in the source.
+
+        Args:
+        ----
+            matcher (Callable[[str], bool] | str): A string to match or a callable that
+                takes a string and returns a boolean.
+            chunk (int): The number of characters to check from the current position.
+            case_insensitive (bool): If True, perform a case-insensitive match __only__ for strings.
+
+        """
+        if isinstance(matcher, str):
+            chunk = len(matcher)
+
+        string_chunk = self.peek(chunk)
+        if not string_chunk:
+            return False
+
+        if isinstance(matcher, str):
+            if case_insensitive:
+                string_chunk = string_chunk.lower()
+                matcher = matcher.lower()
+            if string_chunk == matcher:
+                self.advance(chunk)
+                return True
+            return False
+
+        if matcher(string_chunk):
+            self.advance(chunk)
+            return True
+        return False
+
+    def peek(self, chunk: int = 1) -> str:
+        """Returns the next chunk of text from the current position. Defaults to a single character.
+
+        Args:
+            chunk (int): The number of characters to return from the current position.
+
+        Returns:
+            str: The next chunk of text from the current position.
+
+        """
+        return (
+            self.source[self.current_position : self.current_position + chunk]
+            if self.current_position < len(self.source)
+            else ""
+        )
+
+    def remaining(self) -> str:
+        """Returns the remaining text from the current position to the end of the source.
+
+        Only used for testing and debugging purposes.
+
+        Returns:
+            str: The remaining text from the current position to the end of the source.
+
+        """
+        return self.source[self.current_position :]
+
+    def advance(self, chunk: int = 1) -> None:
+        """Advances the current position by the specified chunk size.
+
+        Generally used alongside peek to consume characters.
+
+        Args:
+            chunk (int): The number of characters to advance the current position.
+
+        Raises:
+            ValueError: If the current position exceeds a predefined MAX_POSITION (1,000,000 characters).
+                This is to avoid errors with the lexer causing the process to hang
+
+        """
+        if self.current_position > MAX_POSITION:
+            msg = f"Current position exceeds {MAX_POSITION:,} characters."
+            raise ValueError(msg)
+        self.current_position += chunk
+
+    def scan(self) -> tuple[BaseToken, ...]:
+        """Repeatedly calls scan_helper until the end of the source is reached.
+
+        Returns:
+            tuple[BaseToken, ...]: A tuple of tokens scanned from the source.
+
+        """
+        while not self.at_end():
+            self.tokens.append(self.scan_helper())
+        return tuple(self.tokens)
+
+    def at_end(self) -> bool:
+        """Checks if the current position is at (or beyond) the end of the source.
+
+        Returns:
+            bool: True if the current position is at or beyond the end of the source, False
+                otherwise.
+
+        """
+        return self.current_position >= len(self.source)

pbi_parsers/base/tokens.py

@@ -0,0 +1,61 @@
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class TextSlice:
+    full_text: str = ""
+    start: int = -1
+    end: int = -1
+
+    def __eq__(self, other: object) -> bool:
+        """Checks equality based on the text slice."""
+        if not isinstance(other, TextSlice):
+            return NotImplemented
+        return self.full_text == other.full_text and self.start == other.start and self.end == other.end
+
+    def __hash__(self) -> int:
+        """Returns a hash based on the text slice."""
+        return hash((self.full_text, self.start, self.end))
+
+    def __repr__(self) -> str:
+        """Returns a string representation of the TextSlice."""
+        return f"TextSlice(text='{self.get_text()}', start={self.start}, end={self.end})"
+
+    def get_text(self) -> str:
+        """Returns the text slice."""
+        return self.full_text[self.start : self.end]
+
+
+@dataclass
+class BaseToken:
+    tok_type: Any
+    text_slice: TextSlice = field(default_factory=TextSlice)
+
+    def __repr__(self) -> str:
+        pretty_text = self.text_slice.get_text().replace("\n", "\\n").replace("\r", "\\r")
+        return f"Token(type={self.tok_type.name}, text='{pretty_text}')"
+
+    @property
+    def text(self) -> str:
+        """Returns the text underlying the token."""
+        return self.text_slice.get_text()
+
+    def position(self) -> tuple[int, int]:
+        """Returns the start and end positions of the token.
+
+        Returns:
+            tuple[int, int]: A tuple containing the start and end positions of the token within the source text.
+
+        """
+        return self.text_slice.start, self.text_slice.end
+
+    def __eq__(self, other: object) -> bool:
+        """Checks equality based on token type and text slice."""
+        if not isinstance(other, BaseToken):
+            return NotImplemented
+        return self.tok_type == other.tok_type and self.text_slice == other.text_slice
+
+    def __hash__(self) -> int:
+        """Returns a hash based on token type and text slice."""
+        return hash((self.tok_type, self.text_slice))

pbi_parsers/dax/__init__.py

@@ -0,0 +1,22 @@
+from . import exprs, utils
+from .exprs import Expression
+from .formatter import Formatter
+from .lexer import Lexer
+from .main import format_expression, to_ast
+from .parser import Parser
+from .tokens import Token, TokenType
+from .utils import highlight_section
+
+__all__ = [
+    "Expression",
+    "Formatter",
+    "Lexer",
+    "Parser",
+    "Token",
+    "TokenType",
+    "exprs",
+    "format_expression",
+    "highlight_section",
+    "to_ast",
+    "utils",
+]

pbi_parsers/dax/exprs/__init__.py

@@ -0,0 +1,107 @@
+from typing import TYPE_CHECKING
+
+from ._base import Expression
+from .add_sub import AddSubExpression
+from .add_sub_unary import AddSubUnaryExpression
+from .array import ArrayExpression
+from .column import ColumnExpression
+from .comparison import ComparisonExpression
+from .concatenation import ConcatenationExpression
+from .div_mul import DivMulExpression
+from .exponent import ExponentExpression
+from .function import FunctionExpression
+from .hierarchy import HierarchyExpression
+from .identifier import IdentifierExpression
+from .ins import InExpression
+from .keyword import KeywordExpression
+from .literal_number import LiteralNumberExpression
+from .literal_string import LiteralStringExpression
+from .logical import LogicalExpression
+from .measure import MeasureExpression
+from .none import NoneExpression
+from .parens import ParenthesesExpression
+from .returns import ReturnExpression
+from .table import TableExpression
+from .variable import VariableExpression
+
+if TYPE_CHECKING:
+    from pbi_parsers.dax.parser import Parser
+
+# Bool/AddSub/DivMul must be in this order to ensure correct precedence. They must also be above all other expressions.
+# Column expression must be before table and identifier expressions to ensure correct precedence.
+# identifer must be before table to ensure correct precedence.
+
+# operator precedence (https://learn.microsoft.com/en-us/dax/dax-operator-reference). This is from tightest to loosest:
+# unary +,-
+# ^
+# *,/
+# +,-
+# &
+# &&, || (note: this is not specified in the docs, so I'm guessing here)
+# =, ==, <>, <, <=, >, >=
+# IN (note: this is not specified in the docs, so I'm guessing here)
+# NOT
+
+EXPRESSION_HIERARCHY = (
+    # Operators, must come first
+    InExpression,
+    LogicalExpression,
+    ComparisonExpression,
+    ConcatenationExpression,
+    AddSubExpression,
+    DivMulExpression,
+    ExponentExpression,
+    AddSubUnaryExpression,
+    # For performance, the ones with a defined prefix
+    ReturnExpression,  # must come before VariableExpression
+    VariableExpression,
+    ParenthesesExpression,
+    ArrayExpression,
+    FunctionExpression,
+    MeasureExpression,
+    HierarchyExpression,
+    ColumnExpression,
+    KeywordExpression,
+    IdentifierExpression,
+    TableExpression,  # Technically, it's partially ambiguous with IdentifierExpression
+    LiteralStringExpression,
+    LiteralNumberExpression,
+)
+
+
+def any_expression_match(parser: "Parser", skip_first: int = 0) -> Expression | None:
+    """Matches any expression type.
+
+    This is a utility function to simplify the matching process in other expressions.
+    """
+    for expr in EXPRESSION_HIERARCHY[skip_first:]:
+        if match := expr.match(parser):
+            return match
+    return None
+
+
+__all__ = [
+    "AddSubExpression",
+    "AddSubUnaryExpression",
+    "ArrayExpression",
+    "ColumnExpression",
+    "ComparisonExpression",
+    "ConcatenationExpression",
+    "DivMulExpression",
+    "ExponentExpression",
+    "Expression",
+    "FunctionExpression",
+    "HierarchyExpression",
+    "IdentifierExpression",
+    "InExpression",
+    "KeywordExpression",
+    "LiteralNumberExpression",
+    "LiteralStringExpression",
+    "LogicalExpression",
+    "MeasureExpression",
+    "NoneExpression",
+    "ParenthesesExpression",
+    "ReturnExpression",
+    "TableExpression",
+    "VariableExpression",
+]

pbi_parsers/dax/exprs/_base.py

@@ -0,0 +1,46 @@
+from typing import TYPE_CHECKING, Any
+
+from pbi_parsers.dax.tokens import TokenType
+
+if TYPE_CHECKING:
+    from pbi_parsers.dax.parser import Parser
+
+
+class Expression:
+    pre_comments: list[Any] = []
+    post_comments: list[Any] = []
+
+    def pprint(self) -> str:
+        msg = "Subclasses should implement this method."
+        raise NotImplementedError(msg)
+
+    @classmethod
+    def match(cls, parser: "Parser") -> "Expression | None":
+        """Attempt to match the current tokens to this expression type.
+
+        Returns an instance of the expression if matched, otherwise None.
+        """
+        msg = "Subclasses should implement this method."
+        raise NotImplementedError(msg)
+
+    @staticmethod
+    def match_tokens(parser: "Parser", match_tokens: list[TokenType]) -> bool:
+        return all(parser.peek(i).tok_type == token_type for i, token_type in enumerate(match_tokens))
+
+    def __repr__(self) -> str:
+        return self.pprint()
+
+    def children(self) -> list["Expression"]:
+        """Returns a list of child expressions."""
+        msg = "This method should be implemented by subclasses."
+        raise NotImplementedError(msg)
+
+    def position(self) -> tuple[int, int]:
+        """Returns the start and end positions of the expression in the source code."""
+        msg = "This method should be implemented by subclasses."
+        raise NotImplementedError(msg)
+
+    def full_text(self) -> str:
+        """Returns the full text of the expression."""
+        msg = "This method should be implemented by subclasses."
+        raise NotImplementedError(msg)

pbi_parsers/dax/exprs/_utils.py

@@ -0,0 +1,45 @@
+from collections.abc import Callable
+from typing import ParamSpec, TypeVar
+
+from pbi_parsers.dax.exprs._base import Expression
+from pbi_parsers.dax.parser import Parser
+from pbi_parsers.dax.tokens import TokenType
+
+P = ParamSpec("P")  # Represents the parameters of the decorated function
+R = TypeVar("R")  # Represents the return type of the decorated function
+
+
+def lexer_reset(func: Callable[P, R]) -> Callable[P, R]:
+    def lexer_reset_inner(*args: P.args, **kwargs: P.kwargs) -> R:
+        parser = args[1]
+        if not isinstance(parser, Parser):
+            msg = f"Expected the second argument to be a Parser instance, got {type(parser)}"
+            raise TypeError(msg)
+        idx = parser.index
+
+        pre_comments = []
+        while parser.peek().tok_type in {TokenType.SINGLE_LINE_COMMENT, TokenType.MULTI_LINE_COMMENT}:
+            pre_comments.append(parser.consume())
+
+        # Speed up of a bazillion
+        cached_val, cached_index = parser.cache.get((idx, id(func)), (None, -1))
+        if cached_val is not None:
+            parser.index = cached_index
+            return cached_val
+
+        ret = func(*args, **kwargs)
+
+        post_comments = []
+        while parser.peek().tok_type in {TokenType.SINGLE_LINE_COMMENT, TokenType.MULTI_LINE_COMMENT}:
+            post_comments.append(parser.consume())
+
+        if isinstance(ret, Expression):
+            ret.pre_comments = pre_comments
+            ret.post_comments = post_comments
+
+        parser.cache[idx, id(func)] = (ret, parser.index)
+        if ret is None:
+            parser.index = idx
+        return ret
+
+    return lexer_reset_inner

pbi_parsers/dax/exprs/add_sub.py

@@ -0,0 +1,73 @@
+import textwrap
+from typing import TYPE_CHECKING
+
+from pbi_parsers.dax.tokens import Token, TokenType
+
+from ._base import Expression
+from ._utils import lexer_reset
+
+if TYPE_CHECKING:
+    from pbi_parsers.dax.parser import Parser
+
+
+class AddSubExpression(Expression):
+    """Represents an addition or subtraction expression.
+
+    Examples:
+        1 + 2
+        func() - 3
+
+    """
+
+    operator: Token
+    left: Expression
+    right: Expression
+
+    def __init__(self, operator: Token, left: Expression, right: Expression) -> None:
+        self.operator = operator
+        self.left = left
+        self.right = right
+
+    @classmethod
+    @lexer_reset
+    def match(cls, parser: "Parser") -> "AddSubExpression | None":
+        from . import EXPRESSION_HIERARCHY, any_expression_match  # noqa: PLC0415
+
+        skip_index = EXPRESSION_HIERARCHY.index(AddSubExpression)
+
+        left_term = any_expression_match(parser=parser, skip_first=skip_index + 1)
+        operator = parser.consume()
+
+        if not left_term:
+            return None
+        if operator.tok_type not in {
+            TokenType.PLUS_SIGN,
+            TokenType.MINUS_SIGN,
+        }:
+            return None
+
+        right_term = any_expression_match(parser=parser, skip_first=skip_index)
+        if right_term is None:
+            msg = f"Expected a right term after operator {operator.text}, found: {parser.peek()}"
+            raise ValueError(msg)
+        return AddSubExpression(operator=operator, left=left_term, right=right_term)
+
+    def pprint(self) -> str:
+        op_str = "Add" if self.operator.text == "+" else "Sub"
+        left_str = textwrap.indent(self.left.pprint(), " " * 10).lstrip()
+        right_str = textwrap.indent(self.right.pprint(), " " * 11).lstrip()
+        return f"""
+{op_str} (
+    left: {left_str},
+    right: {right_str}
+)""".strip()
+
+    def children(self) -> list[Expression]:
+        """Returns a list of child expressions."""
+        return [self.left, self.right]
+
+    def position(self) -> tuple[int, int]:
+        return self.left.position()[0], self.right.position()[1]
+
+    def full_text(self) -> str:
+        return self.operator.text_slice.full_text

pbi_parsers/dax/exprs/add_sub_unary.py

@@ -0,0 +1,72 @@
+import textwrap
+from typing import TYPE_CHECKING
+
+from pbi_parsers.dax.tokens import Token, TokenType
+
+from ._base import Expression
+from ._utils import lexer_reset
+
+if TYPE_CHECKING:
+    from pbi_parsers.dax.parser import Parser
+
+
+class AddSubUnaryExpression(Expression):
+    """Represents an addition or subtraction expression.
+
+    Examples:
+        +2
+        -func()
+
+    """
+
+    operator: Token
+    number: Expression
+
+    def __init__(self, operator: Token, number: Expression) -> None:
+        self.operator = operator
+        self.number = number
+
+    @classmethod
+    @lexer_reset
+    def match(cls, parser: "Parser") -> "AddSubUnaryExpression | None":
+        from . import EXPRESSION_HIERARCHY, any_expression_match  # noqa: PLC0415
+
+        skip_index = EXPRESSION_HIERARCHY.index(
+            AddSubUnaryExpression,
+        )  # intentionally inclusive of self to allow +-++- chains
+
+        operator = parser.consume()
+
+        if operator.tok_type not in {
+            TokenType.PLUS_SIGN,
+            TokenType.MINUS_SIGN,
+        }:
+            return None
+
+        # Handle chained +-++-+ prefixes
+        number: Expression | None = any_expression_match(
+            parser=parser,
+            skip_first=skip_index,
+        )
+        if number is None:
+            msg = f"Expected a right term after operator {operator.text}, found: {parser.peek()}"
+            raise ValueError(msg)
+        return AddSubUnaryExpression(operator=operator, number=number)
+
+    def pprint(self) -> str:
+        number = textwrap.indent(self.number.pprint(), " " * 12).lstrip()
+        return f"""
+Number (
+    sign: {self.operator.text},
+    number: {number},
+)""".strip()
+
+    def children(self) -> list[Expression]:
+        """Returns a list of child expressions."""
+        return [self.number]
+
+    def position(self) -> tuple[int, int]:
+        return self.operator.text_slice.start, self.number.position()[1]
+
+    def full_text(self) -> str:
+        return self.operator.text_slice.full_text

pbi_parsers/dax/exprs/array.py

@@ -0,0 +1,75 @@
+import textwrap
+from typing import TYPE_CHECKING
+
+from pbi_parsers.dax.tokens import Token, TokenType
+
+from ._base import Expression
+from ._utils import lexer_reset
+
+if TYPE_CHECKING:
+    from pbi_parsers.dax.parser import Parser
+
+
+class ArrayExpression(Expression):
+    """Represents an array expression.
+
+    Examples:
+        {1, 2, 3}
+        {func(), 4, 5}
+
+    """
+
+    elements: list[Expression]
+    braces: tuple[Token, Token]
+
+    def __init__(self, elements: list[Expression], braces: tuple[Token, Token]) -> None:
+        self.elements: list[Expression] = elements
+        self.braces = braces
+
+    def pprint(self) -> str:
+        elements = ",\n".join(element.pprint() for element in self.elements)
+        elements = textwrap.indent(elements, " " * 14)[14:]
+        return f"""
+Array (
+    elements: {elements}
+)        """.strip()
+
+    @classmethod
+    @lexer_reset
+    def match(cls, parser: "Parser") -> "ArrayExpression | None":
+        from . import any_expression_match  # noqa: PLC0415
+
+        left_brace = parser.consume()
+        if left_brace.tok_type != TokenType.LEFT_CURLY_BRACE:
+            return None
+
+        elements: list[Expression] = []
+
+        while not cls.match_tokens(parser, [TokenType.RIGHT_CURLY_BRACE]):
+            # We gotta handle operators next :(
+            element = any_expression_match(parser)
+            if element is not None:
+                elements.append(element)
+            else:
+                msg = f"Unexpected token sequence: {parser.peek()}, {parser.index}"
+                raise ValueError(msg)
+
+            if not cls.match_tokens(parser, [TokenType.RIGHT_CURLY_BRACE]):
+                assert parser.consume().tok_type == TokenType.COMMA
+
+        right_brace = parser.consume()
+        if right_brace.tok_type != TokenType.RIGHT_CURLY_BRACE:
+            msg = f"Expected a right curly brace, found: {right_brace}"
+            raise ValueError(msg)
+
+        return ArrayExpression(elements=elements, braces=(left_brace, right_brace))
+
+    def children(self) -> list[Expression]:
+        """Returns a list of child expressions."""
+        return self.elements
+
+    def position(self) -> tuple[int, int]:
+        return self.braces[0].text_slice.start, self.braces[1].text_slice.end
+
+    def full_text(self) -> str:
+        return self.braces[0].text_slice.full_text