pbi-parsers 0.8.1__tar.gz → 0.8.2__tar.gz

{pbi_parsers-0.8.1 → pbi_parsers-0.8.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pbi_parsers
-Version: 0.8.1
+Version: 0.8.2
 Summary: Power BI lexer, parsers, and formatters for DAX and M (Power Query) languages
 Project-URL: Homepage, https://github.com/douglassimonsen/pbi_parsers
 Project-URL: Documentation, https://douglassimonsen.github.io/pbi_parsers/

{pbi_parsers-0.8.1 → pbi_parsers-0.8.2}/pbi_parsers/__init__.py

@@ -1,6 +1,6 @@
 from . import dax, pq
 
-__version__ = "0.8.1"
+__version__ = "0.8.2"
 
 
 __all__ = [

{pbi_parsers-0.8.1 → pbi_parsers-0.8.2}/pbi_parsers/dax/tokens.py

@@ -2,6 +2,7 @@ from dataclasses import dataclass
 from enum import Enum, auto
 
 from pbi_parsers.base import BaseToken
+from pbi_parsers.base.tokens import TextSlice
 
 
 class TokenType(Enum):
@@ -39,6 +40,8 @@ class TokenType(Enum):
     UNQUOTED_IDENTIFIER = auto()
     VARIABLE = auto()
     WHITESPACE = auto()
+    UNKNOWN = auto()
+    """unknown is used when someone replaces a token with a str"""
 
 
 KEYWORD_MAPPING = {
@@ -52,3 +55,17 @@ KEYWORD_MAPPING = {
 @dataclass
 class Token(BaseToken):
     tok_type: TokenType = TokenType.EOF
+
+    @staticmethod
+    def from_str(value: str, tok_type: TokenType = TokenType.UNKNOWN) -> "Token":
+        tok_type = KEYWORD_MAPPING.get(value, tok_type)
+        return Token(
+            tok_type=tok_type,
+            text_slice=TextSlice(value, 0, len(value)),
+        )
+
+    def add_token_before(self, text: str, tok_type: TokenType) -> None:
+        super().add_token_before(text, tok_type)
+
+    def add_token_after(self, text: str, tok_type: TokenType) -> None:
+        super().add_token_after(text, tok_type)

{pbi_parsers-0.8.1 → pbi_parsers-0.8.2}/pbi_parsers/pq/tokens.py

@@ -3,6 +3,8 @@ from enum import Enum
 
 from pbi_parsers.base import BaseToken
 
+from ..base.tokens import TextSlice
+
 
 class TokenType(Enum):
     LET = 1
@@ -52,12 +54,27 @@ class TokenType(Enum):
     IS = 46
     AS = 47
     EXCLAMATION_POINT = 48
+    UNKNOWN = 99
+    """unknown is used when someone replaces a token with a str"""
 
 
 @dataclass
 class Token(BaseToken):
     tok_type: TokenType = TokenType.EOF
 
+    @staticmethod
+    def from_str(value: str, tok_type: TokenType = TokenType.UNKNOWN) -> "Token":
+        return Token(
+            tok_type=tok_type,
+            text_slice=TextSlice(value, 0, len(value)),
+        )
+
+    def add_token_before(self, text: str, tok_type: TokenType) -> None:
+        super().add_token_before(text, tok_type)
+
+    def add_token_after(self, text: str, tok_type: TokenType) -> None:
+        super().add_token_after(text, tok_type)
+
 
 # These are tokens that could also be used as identifiers in expressions.
 TEXT_TOKENS = (