pbi-parsers 0.8.1__py3-none-any.whl → 0.8.3__py3-none-any.whl

pbi_parsers/__init__.py

@@ -1,6 +1,6 @@
 from . import dax, pq
 
-__version__ = "0.8.1"
+__version__ = "0.8.3"
 
 
 __all__ = [

pbi_parsers/dax/tokens.py

@@ -2,6 +2,7 @@ from dataclasses import dataclass
 from enum import Enum, auto
 
 from pbi_parsers.base import BaseToken
+from pbi_parsers.base.tokens import TextSlice
 
 
 class TokenType(Enum):
@@ -40,6 +41,9 @@ class TokenType(Enum):
     VARIABLE = auto()
     WHITESPACE = auto()
 
+    UNKNOWN = auto()
+    """unknown is used when someone replaces a token with a str"""
+
 
 KEYWORD_MAPPING = {
     "TRUE": TokenType.TRUE,
@@ -52,3 +56,17 @@ KEYWORD_MAPPING = {
 @dataclass
 class Token(BaseToken):
     tok_type: TokenType = TokenType.EOF
+
+    @staticmethod
+    def from_str(value: str, tok_type: TokenType = TokenType.UNKNOWN) -> "Token":
+        tok_type = KEYWORD_MAPPING.get(value, tok_type)
+        return Token(
+            tok_type=tok_type,
+            text_slice=TextSlice(value, 0, len(value)),
+        )
+
+    def add_token_before(self, text: str, tok_type: TokenType) -> None:
+        super().add_token_before(text, tok_type)
+
+    def add_token_after(self, text: str, tok_type: TokenType) -> None:
+        super().add_token_after(text, tok_type)

pbi_parsers/dax/utils.py

@@ -4,7 +4,7 @@ import jinja2
 from colorama import Fore, Style
 
 from .exprs import Expression
-from .tokens import Token
+from .tokens import Token, TokenType
 
 T = TypeVar("T", bound=Expression)
 
@@ -118,3 +118,30 @@ def highlight_section(node: Expression | Token | list[Token] | list[Expression])
     position = node.position()
     full_text = node.text_slice.full_text if isinstance(node, Token) else node.full_text()
     return Context(position, full_text)
+
+
+def get_inner_text(tok: Token) -> str:
+    """Returns the inner text of a token, stripping any surrounding quotes or brackets.
+
+    Args:
+        tok (Token): The token to extract inner text from.
+
+    Returns:
+        str: The inner text of the token.
+
+    Raises:
+        ValueError: If the token type does not support inner text extraction.
+
+    """
+    if tok.tok_type in {
+        TokenType.BRACKETED_IDENTIFIER,
+        TokenType.SINGLE_QUOTED_IDENTIFIER,
+    }:
+        return tok.text[1:-1]
+    if tok.tok_type in {
+        TokenType.STRING_LITERAL,
+        TokenType.UNQUOTED_IDENTIFIER,
+    }:
+        return tok.text
+    msg = f"Token type {tok.tok_type} does not have inner text"
+    raise ValueError(msg)

pbi_parsers/pq/tokens.py

@@ -2,6 +2,7 @@ from dataclasses import dataclass
 from enum import Enum
 
 from pbi_parsers.base import BaseToken
+from pbi_parsers.base.tokens import TextSlice
 
 
 class TokenType(Enum):
@@ -52,12 +53,27 @@ class TokenType(Enum):
     IS = 46
     AS = 47
     EXCLAMATION_POINT = 48
+    UNKNOWN = 99
+    """unknown is used when someone replaces a token with a str"""
 
 
 @dataclass
 class Token(BaseToken):
     tok_type: TokenType = TokenType.EOF
 
+    @staticmethod
+    def from_str(value: str, tok_type: TokenType = TokenType.UNKNOWN) -> "Token":
+        return Token(
+            tok_type=tok_type,
+            text_slice=TextSlice(value, 0, len(value)),
+        )
+
+    def add_token_before(self, text: str, tok_type: TokenType) -> None:
+        super().add_token_before(text, tok_type)
+
+    def add_token_after(self, text: str, tok_type: TokenType) -> None:
+        super().add_token_after(text, tok_type)
+
 
 # These are tokens that could also be used as identifiers in expressions.
 TEXT_TOKENS = (

{pbi_parsers-0.8.1.dist-info → pbi_parsers-0.8.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pbi_parsers
-Version: 0.8.1
+Version: 0.8.3
 Summary: Power BI lexer, parsers, and formatters for DAX and M (Power Query) languages
 Project-URL: Homepage, https://github.com/douglassimonsen/pbi_parsers
 Project-URL: Documentation, https://douglassimonsen.github.io/pbi_parsers/

{pbi_parsers-0.8.1.dist-info → pbi_parsers-0.8.3.dist-info}/RECORD

@@ -1,4 +1,4 @@
-pbi_parsers/__init__.py,sha256=ljbk7gxiajzYkx2vu3YTjMDWJjYoFMUhfrwtK2iJbI4,91
+pbi_parsers/__init__.py,sha256=bYOBpDdxjZNL99olLw0gryn94zqaskT48sEBk-mrqs4,91
 pbi_parsers/base/__init__.py,sha256=U7QpzFFD9A4wK3ZHin6xg5fPgTca0y5KC-O7nrZ-flM,115
 pbi_parsers/base/lexer.py,sha256=Rl2cWlySJblFHvwW8oMVAjnCh83Zjo4N7jW3pkRZZO0,4335
 pbi_parsers/base/tokens.py,sha256=c1PIAU5oRxZbI74SY5KT3AZ3WrcukzXtuiLPrZU4f2o,7017
@@ -7,8 +7,8 @@ pbi_parsers/dax/formatter.py,sha256=jOFnwcgQGIzsmi5sfkKoB_pFEGjDPd8E_pwMPwudmy4,
 pbi_parsers/dax/lexer.py,sha256=2_pERJSrSYd8VujOe9TxJa9R2Ex8mvP-bCotH7uVBZY,8025
 pbi_parsers/dax/main.py,sha256=FG35XCAPEooXoJShSgOnmQ0py-h_MPtOfnLpQWy61is,1657
 pbi_parsers/dax/parser.py,sha256=QLKrIBcxZ26TGhTHpeKcTGEHEHUDLC6IgpxxrdJzdek,1821
-pbi_parsers/dax/tokens.py,sha256=nY1laCbL8vwALpJ4jcd8k4lAscqOwqdw3dFuj4_KKVk,1234
-pbi_parsers/dax/utils.py,sha256=OURPa-b6Ldn0_KKXPdLIPA3Zdc12OfbbFd2X5SocCek,4402
+pbi_parsers/dax/tokens.py,sha256=ela2bX-iXIIsEho5G07Vqsiefc4oEv7mYLJEfWD95ZI,1887
+pbi_parsers/dax/utils.py,sha256=fyGZY_6ASLtH7Hwk89Ti0Bvu49kAmcUE3D8c7QPyzks,5160
 pbi_parsers/dax/exprs/__init__.py,sha256=OUfiXzZYp5HkTPE9x595MMxpsgG1IvsED8p8spAKZuk,3432
 pbi_parsers/dax/exprs/_base.py,sha256=bMHLICgAUOqAKl_S9d6V8kk62cqM1jynUY2-gBJlEcs,2732
 pbi_parsers/dax/exprs/_utils.py,sha256=BxxRCtsqpL9t330ZfBnkm1lq7B_ejAoG89Ot2bLYJig,2123
@@ -39,7 +39,7 @@ pbi_parsers/pq/formatter.py,sha256=gcqj_aP8o5V10ULi5hdGhy3aAOy829jTKAfzH4mZewA,3
 pbi_parsers/pq/lexer.py,sha256=YOo4chz1N06FLO7cU4-wSoemIzfwG30NeUSJhJB-yOE,8093
 pbi_parsers/pq/main.py,sha256=4k5ZT-dRv5g2jjFgL1ckSpLR36wzClxe1YjiiIiBMu8,1649
 pbi_parsers/pq/parser.py,sha256=Fy8cqAGvGv1oVg4vYWJAGHZSWimEJ3wTtL5eqIkfOA8,1885
-pbi_parsers/pq/tokens.py,sha256=tll_fijLQ2reUJZIgyTW_a6ewuxiw9dva4dH9zx4GZ0,1637
+pbi_parsers/pq/tokens.py,sha256=vmcZcK_0E99HSR5FCk8LUrxTVT2D5ZeGX-AWPHSZdVc,2245
 pbi_parsers/pq/exprs/__init__.py,sha256=wV-G51GagUAkA6_uVjsNA5JskO2JN3xXJPjKtzCH5rU,2845
 pbi_parsers/pq/exprs/_base.py,sha256=GcfWW3rannZBvw4LyjdiGbWGJ1nctw-m5k6LGkX7Wk4,1118
 pbi_parsers/pq/exprs/_utils.py,sha256=kUCWSzCSy7HMKOWcGjmO4R1WiYHP23afXmq67A0ZAXY,1065
@@ -73,7 +73,7 @@ pbi_parsers/pq/exprs/statement.py,sha256=JSg48pGAU3Ka2pt4lzVsYlVOqGeF_ARGm8Ajf0l
 pbi_parsers/pq/exprs/try_expr.py,sha256=UcnqfA-t9S1LVrKqeNUT8n4JJcO-ZQZoJrxAdjJ-GMA,1692
 pbi_parsers/pq/exprs/type_expr.py,sha256=hH5ubrIJaxwQsopNJHUZ4ByS1rHEgv2Tf8ocYqSukXM,2570
 pbi_parsers/pq/exprs/variable.py,sha256=wp4t0QHIGA264sXnWp7XVe1H8MJzMIOaoLNBQe-dfNk,1602
-pbi_parsers-0.8.1.dist-info/METADATA,sha256=-B23BQU7BCehF7DUTBcDeD9gH8ndy730y41H3HxUicU,2906
-pbi_parsers-0.8.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-pbi_parsers-0.8.1.dist-info/licenses/LICENSE,sha256=Sn0IfXOE4B0iL9lZXmGmRuTGyJeCtefxcfws0bLjp2g,1072
-pbi_parsers-0.8.1.dist-info/RECORD,,
+pbi_parsers-0.8.3.dist-info/METADATA,sha256=69HKbe_XOTTk3mtD8hfc--HTkZjUU1V_yBOjx9-qUdM,2906
+pbi_parsers-0.8.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+pbi_parsers-0.8.3.dist-info/licenses/LICENSE,sha256=Sn0IfXOE4B0iL9lZXmGmRuTGyJeCtefxcfws0bLjp2g,1072
+pbi_parsers-0.8.3.dist-info/RECORD,,