pbi-parsers 0.8.1__py3-none-any.whl → 0.8.2__py3-none-any.whl

pbi_parsers/__init__.py

@@ -1,6 +1,6 @@
 from . import dax, pq
 
-__version__ = "0.8.1"
+__version__ = "0.8.2"
 
 
 __all__ = [

pbi_parsers/dax/tokens.py

@@ -2,6 +2,7 @@ from dataclasses import dataclass
 from enum import Enum, auto
 
 from pbi_parsers.base import BaseToken
+from pbi_parsers.base.tokens import TextSlice
 
 
 class TokenType(Enum):
@@ -39,6 +40,8 @@ class TokenType(Enum):
     UNQUOTED_IDENTIFIER = auto()
     VARIABLE = auto()
     WHITESPACE = auto()
+    UNKNOWN = auto()
+    """unknown is used when someone replaces a token with a str"""
 
 
 KEYWORD_MAPPING = {
@@ -52,3 +55,17 @@ KEYWORD_MAPPING = {
 @dataclass
 class Token(BaseToken):
     tok_type: TokenType = TokenType.EOF
+
+    @staticmethod
+    def from_str(value: str, tok_type: TokenType = TokenType.UNKNOWN) -> "Token":
+        tok_type = KEYWORD_MAPPING.get(value, tok_type)
+        return Token(
+            tok_type=tok_type,
+            text_slice=TextSlice(value, 0, len(value)),
+        )
+
+    def add_token_before(self, text: str, tok_type: TokenType) -> None:
+        super().add_token_before(text, tok_type)
+
+    def add_token_after(self, text: str, tok_type: TokenType) -> None:
+        super().add_token_after(text, tok_type)

pbi_parsers/pq/tokens.py

@@ -3,6 +3,8 @@ from enum import Enum
 
 from pbi_parsers.base import BaseToken
 
+from ..base.tokens import TextSlice
+
 
 class TokenType(Enum):
     LET = 1
@@ -52,12 +54,27 @@ class TokenType(Enum):
     IS = 46
     AS = 47
     EXCLAMATION_POINT = 48
+    UNKNOWN = 99
+    """unknown is used when someone replaces a token with a str"""
 
 
 @dataclass
 class Token(BaseToken):
     tok_type: TokenType = TokenType.EOF
 
+    @staticmethod
+    def from_str(value: str, tok_type: TokenType = TokenType.UNKNOWN) -> "Token":
+        return Token(
+            tok_type=tok_type,
+            text_slice=TextSlice(value, 0, len(value)),
+        )
+
+    def add_token_before(self, text: str, tok_type: TokenType) -> None:
+        super().add_token_before(text, tok_type)
+
+    def add_token_after(self, text: str, tok_type: TokenType) -> None:
+        super().add_token_after(text, tok_type)
+
 
 # These are tokens that could also be used as identifiers in expressions.
 TEXT_TOKENS = (

{pbi_parsers-0.8.1.dist-info → pbi_parsers-0.8.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pbi_parsers
-Version: 0.8.1
+Version: 0.8.2
 Summary: Power BI lexer, parsers, and formatters for DAX and M (Power Query) languages
 Project-URL: Homepage, https://github.com/douglassimonsen/pbi_parsers
 Project-URL: Documentation, https://douglassimonsen.github.io/pbi_parsers/

{pbi_parsers-0.8.1.dist-info → pbi_parsers-0.8.2.dist-info}/RECORD

@@ -1,4 +1,4 @@
-pbi_parsers/__init__.py,sha256=ljbk7gxiajzYkx2vu3YTjMDWJjYoFMUhfrwtK2iJbI4,91
+pbi_parsers/__init__.py,sha256=o6tdbz0lucvp3ShZEGQdvolSfiDl90As5VtF8yWSzBc,91
 pbi_parsers/base/__init__.py,sha256=U7QpzFFD9A4wK3ZHin6xg5fPgTca0y5KC-O7nrZ-flM,115
 pbi_parsers/base/lexer.py,sha256=Rl2cWlySJblFHvwW8oMVAjnCh83Zjo4N7jW3pkRZZO0,4335
 pbi_parsers/base/tokens.py,sha256=c1PIAU5oRxZbI74SY5KT3AZ3WrcukzXtuiLPrZU4f2o,7017
@@ -7,7 +7,7 @@ pbi_parsers/dax/formatter.py,sha256=jOFnwcgQGIzsmi5sfkKoB_pFEGjDPd8E_pwMPwudmy4,
 pbi_parsers/dax/lexer.py,sha256=2_pERJSrSYd8VujOe9TxJa9R2Ex8mvP-bCotH7uVBZY,8025
 pbi_parsers/dax/main.py,sha256=FG35XCAPEooXoJShSgOnmQ0py-h_MPtOfnLpQWy61is,1657
 pbi_parsers/dax/parser.py,sha256=QLKrIBcxZ26TGhTHpeKcTGEHEHUDLC6IgpxxrdJzdek,1821
-pbi_parsers/dax/tokens.py,sha256=nY1laCbL8vwALpJ4jcd8k4lAscqOwqdw3dFuj4_KKVk,1234
+pbi_parsers/dax/tokens.py,sha256=vWkHbxWg_tU_vkzCayrK6K8W2CQPPghY0sshRwFAvoc,1886
 pbi_parsers/dax/utils.py,sha256=OURPa-b6Ldn0_KKXPdLIPA3Zdc12OfbbFd2X5SocCek,4402
 pbi_parsers/dax/exprs/__init__.py,sha256=OUfiXzZYp5HkTPE9x595MMxpsgG1IvsED8p8spAKZuk,3432
 pbi_parsers/dax/exprs/_base.py,sha256=bMHLICgAUOqAKl_S9d6V8kk62cqM1jynUY2-gBJlEcs,2732
@@ -39,7 +39,7 @@ pbi_parsers/pq/formatter.py,sha256=gcqj_aP8o5V10ULi5hdGhy3aAOy829jTKAfzH4mZewA,3
 pbi_parsers/pq/lexer.py,sha256=YOo4chz1N06FLO7cU4-wSoemIzfwG30NeUSJhJB-yOE,8093
 pbi_parsers/pq/main.py,sha256=4k5ZT-dRv5g2jjFgL1ckSpLR36wzClxe1YjiiIiBMu8,1649
 pbi_parsers/pq/parser.py,sha256=Fy8cqAGvGv1oVg4vYWJAGHZSWimEJ3wTtL5eqIkfOA8,1885
-pbi_parsers/pq/tokens.py,sha256=tll_fijLQ2reUJZIgyTW_a6ewuxiw9dva4dH9zx4GZ0,1637
+pbi_parsers/pq/tokens.py,sha256=WHaAIvGaoj22b-YbuFF_t0h5fLdaia3ZdNTGRXK8PiA,2237
 pbi_parsers/pq/exprs/__init__.py,sha256=wV-G51GagUAkA6_uVjsNA5JskO2JN3xXJPjKtzCH5rU,2845
 pbi_parsers/pq/exprs/_base.py,sha256=GcfWW3rannZBvw4LyjdiGbWGJ1nctw-m5k6LGkX7Wk4,1118
 pbi_parsers/pq/exprs/_utils.py,sha256=kUCWSzCSy7HMKOWcGjmO4R1WiYHP23afXmq67A0ZAXY,1065
@@ -73,7 +73,7 @@ pbi_parsers/pq/exprs/statement.py,sha256=JSg48pGAU3Ka2pt4lzVsYlVOqGeF_ARGm8Ajf0l
 pbi_parsers/pq/exprs/try_expr.py,sha256=UcnqfA-t9S1LVrKqeNUT8n4JJcO-ZQZoJrxAdjJ-GMA,1692
 pbi_parsers/pq/exprs/type_expr.py,sha256=hH5ubrIJaxwQsopNJHUZ4ByS1rHEgv2Tf8ocYqSukXM,2570
 pbi_parsers/pq/exprs/variable.py,sha256=wp4t0QHIGA264sXnWp7XVe1H8MJzMIOaoLNBQe-dfNk,1602
-pbi_parsers-0.8.1.dist-info/METADATA,sha256=-B23BQU7BCehF7DUTBcDeD9gH8ndy730y41H3HxUicU,2906
-pbi_parsers-0.8.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-pbi_parsers-0.8.1.dist-info/licenses/LICENSE,sha256=Sn0IfXOE4B0iL9lZXmGmRuTGyJeCtefxcfws0bLjp2g,1072
-pbi_parsers-0.8.1.dist-info/RECORD,,
+pbi_parsers-0.8.2.dist-info/METADATA,sha256=lw6AvVHgL0ERGlP7LEZtl343oezahwNwDJ4FJCXFbGM,2906
+pbi_parsers-0.8.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+pbi_parsers-0.8.2.dist-info/licenses/LICENSE,sha256=Sn0IfXOE4B0iL9lZXmGmRuTGyJeCtefxcfws0bLjp2g,1072
+pbi_parsers-0.8.2.dist-info/RECORD,,