pbi-parsers 0.8.1__tar.gz → 0.8.3__tar.gz

{pbi_parsers-0.8.1 → pbi_parsers-0.8.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pbi_parsers
-Version: 0.8.1
+Version: 0.8.3
 Summary: Power BI lexer, parsers, and formatters for DAX and M (Power Query) languages
 Project-URL: Homepage, https://github.com/douglassimonsen/pbi_parsers
 Project-URL: Documentation, https://douglassimonsen.github.io/pbi_parsers/

{pbi_parsers-0.8.1 → pbi_parsers-0.8.3}/pbi_parsers/__init__.py

@@ -1,6 +1,6 @@
 from . import dax, pq
 
-__version__ = "0.8.1"
+__version__ = "0.8.3"
 
 
 __all__ = [

{pbi_parsers-0.8.1 → pbi_parsers-0.8.3}/pbi_parsers/dax/tokens.py

@@ -2,6 +2,7 @@ from dataclasses import dataclass
 from enum import Enum, auto
 
 from pbi_parsers.base import BaseToken
+from pbi_parsers.base.tokens import TextSlice
 
 
 class TokenType(Enum):
@@ -40,6 +41,9 @@ class TokenType(Enum):
     VARIABLE = auto()
     WHITESPACE = auto()
 
+    UNKNOWN = auto()
+    """unknown is used when someone replaces a token with a str"""
+
 
 KEYWORD_MAPPING = {
     "TRUE": TokenType.TRUE,
@@ -52,3 +56,17 @@ KEYWORD_MAPPING = {
 @dataclass
 class Token(BaseToken):
     tok_type: TokenType = TokenType.EOF
+
+    @staticmethod
+    def from_str(value: str, tok_type: TokenType = TokenType.UNKNOWN) -> "Token":
+        tok_type = KEYWORD_MAPPING.get(value, tok_type)
+        return Token(
+            tok_type=tok_type,
+            text_slice=TextSlice(value, 0, len(value)),
+        )
+
+    def add_token_before(self, text: str, tok_type: TokenType) -> None:
+        super().add_token_before(text, tok_type)
+
+    def add_token_after(self, text: str, tok_type: TokenType) -> None:
+        super().add_token_after(text, tok_type)

{pbi_parsers-0.8.1 → pbi_parsers-0.8.3}/pbi_parsers/dax/utils.py

@@ -4,7 +4,7 @@ import jinja2
 from colorama import Fore, Style
 
 from .exprs import Expression
-from .tokens import Token
+from .tokens import Token, TokenType
 
 T = TypeVar("T", bound=Expression)
 
@@ -118,3 +118,30 @@ def highlight_section(node: Expression | Token | list[Token] | list[Expression])
     position = node.position()
     full_text = node.text_slice.full_text if isinstance(node, Token) else node.full_text()
     return Context(position, full_text)
+
+
+def get_inner_text(tok: Token) -> str:
+    """Returns the inner text of a token, stripping any surrounding quotes or brackets.
+
+    Args:
+        tok (Token): The token to extract inner text from.
+
+    Returns:
+        str: The inner text of the token.
+
+    Raises:
+        ValueError: If the token type does not support inner text extraction.
+
+    """
+    if tok.tok_type in {
+        TokenType.BRACKETED_IDENTIFIER,
+        TokenType.SINGLE_QUOTED_IDENTIFIER,
+    }:
+        return tok.text[1:-1]
+    if tok.tok_type in {
+        TokenType.STRING_LITERAL,
+        TokenType.UNQUOTED_IDENTIFIER,
+    }:
+        return tok.text
+    msg = f"Token type {tok.tok_type} does not have inner text"
+    raise ValueError(msg)

{pbi_parsers-0.8.1 → pbi_parsers-0.8.3}/pbi_parsers/pq/tokens.py

@@ -2,6 +2,7 @@ from dataclasses import dataclass
 from enum import Enum
 
 from pbi_parsers.base import BaseToken
+from pbi_parsers.base.tokens import TextSlice
 
 
 class TokenType(Enum):
@@ -52,12 +53,27 @@ class TokenType(Enum):
     IS = 46
     AS = 47
     EXCLAMATION_POINT = 48
+    UNKNOWN = 99
+    """unknown is used when someone replaces a token with a str"""
 
 
 @dataclass
 class Token(BaseToken):
     tok_type: TokenType = TokenType.EOF
 
+    @staticmethod
+    def from_str(value: str, tok_type: TokenType = TokenType.UNKNOWN) -> "Token":
+        return Token(
+            tok_type=tok_type,
+            text_slice=TextSlice(value, 0, len(value)),
+        )
+
+    def add_token_before(self, text: str, tok_type: TokenType) -> None:
+        super().add_token_before(text, tok_type)
+
+    def add_token_after(self, text: str, tok_type: TokenType) -> None:
+        super().add_token_after(text, tok_type)
+
 
 # These are tokens that could also be used as identifiers in expressions.
 TEXT_TOKENS = (