pbi-parsers 0.8.0__py3-none-any.whl → 0.8.2__py3-none-any.whl

pbi_parsers/__init__.py

@@ -1,6 +1,6 @@
 from . import dax, pq
 
-__version__ = "0.8.0"
+__version__ = "0.8.2"
 
 
 __all__ = [

pbi_parsers/base/lexer.py

@@ -118,6 +118,9 @@ class BaseLexer:
         """
         while not self.at_end():
             self.tokens.append(self.scan_helper())
+        for a_tok, b_tok in zip(self.tokens, self.tokens[1:], strict=False):
+            a_tok.next_token = b_tok
+            b_tok.prior_token = a_tok
         return tuple(self.tokens)
 
     def scan_helper(self) -> BaseToken:

pbi_parsers/base/tokens.py

@@ -31,6 +31,8 @@ class TextSlice:
 class BaseToken:
     tok_type: Any
     text_slice: TextSlice = field(default_factory=TextSlice)
+    prior_token: "BaseToken | None" = field(repr=False, default=None)
+    next_token: "BaseToken | None" = field(repr=False, default=None)
 
     def __eq__(self, other: object) -> bool:
         """Checks equality based on token type and text slice."""
@@ -64,3 +66,131 @@ class BaseToken:
 
         """
         return self.text_slice.get_text()
+
+    def add_token_before(self, text: str, tok_type: Any) -> None:
+        """Adds a token before the current token in the linked list.
+
+        Args:
+            text (str): The text to add before the current token.
+            tok_type (Any): The type of the token to add.
+
+        """
+        new_global_text = (
+            self.text_slice.full_text[: self.text_slice.start]
+            + text
+            + self.text_slice.full_text[self.text_slice.start :]
+        )
+        self._update_full_text(new_global_text)
+
+        length = len(text)
+        tok = BaseToken(
+            tok_type=tok_type,
+            text_slice=TextSlice(
+                full_text=new_global_text,
+                start=self.text_slice.start,
+                end=self.text_slice.start + length,
+            ),
+            prior_token=self.prior_token,
+            next_token=self,
+        )
+        if self.prior_token:
+            self.prior_token.next_token = tok
+        self.prior_token = tok
+
+        # prior_token because we need to update the current token's position as well
+        curr_tok = self.prior_token
+        while curr_tok := curr_tok.next_token:
+            curr_tok.text_slice.start += length
+            curr_tok.text_slice.end += length
+
+    def add_token_after(self, text: str, tok_type: Any) -> None:
+        """Adds a token after the current token in the linked list.
+
+        Args:
+            text (str): The text to add before the current token.
+            tok_type (Any): The type of the token to add.
+
+        """
+        new_global_text = (
+            self.text_slice.full_text[: self.text_slice.end] + text + self.text_slice.full_text[self.text_slice.end :]
+        )
+        self._update_full_text(new_global_text)
+
+        length = len(text)
+        tok = BaseToken(
+            tok_type=tok_type,
+            text_slice=TextSlice(
+                full_text=new_global_text,
+                start=self.text_slice.end,
+                end=self.text_slice.end + length,
+            ),
+            prior_token=self,
+            next_token=self.next_token,
+        )
+        if self.next_token:
+            self.next_token.prior_token = tok
+        self.next_token = tok
+
+        # prior_token because we need to update the current token's position as well
+        curr_tok = self
+        while curr_tok := curr_tok.next_token:
+            curr_tok.text_slice.start += length
+            curr_tok.text_slice.end += length
+
+    def remove(self) -> None:
+        """Removes the current token from the linked list."""
+        new_global_text = (
+            self.text_slice.full_text[: self.text_slice.start] + self.text_slice.full_text[self.text_slice.end :]
+        )
+        self._update_full_text(new_global_text)
+
+        curr_tok = self
+        length = len(self.text)
+        while curr_tok := curr_tok.next_token:
+            curr_tok.text_slice.start -= length
+            curr_tok.text_slice.end -= length
+
+        if self.prior_token:
+            self.prior_token.next_token = self.next_token
+        if self.next_token:
+            self.next_token.prior_token = self.prior_token
+        self.prior_token = None
+        self.next_token = None
+
+    def replace(self, new_text: str) -> None:
+        """Replaces the text of the current token with new text.
+
+        Args:
+            new_text (str): The new text to replace the current token's text.
+
+        """
+        new_global_text = (
+            self.text_slice.full_text[: self.text_slice.start]
+            + new_text
+            + self.text_slice.full_text[self.text_slice.end :]
+        )
+        self._update_full_text(new_global_text)
+
+        len_diff = len(new_text) - len(self.text)
+        # Adjust the positions of subsequent tokens
+        self.text_slice.end += len_diff
+        current = self.next_token
+        while current:
+            current.text_slice.start += len_diff
+            current.text_slice.end += len_diff
+            current = current.next_token
+
+    def _update_full_text(self, new_full_text: str) -> None:
+        """Updates the full text of the token and adjusts the text slice accordingly.
+
+        Args:
+            new_full_text (str): The new full text to update the token's text slice.
+
+        """
+        self.text_slice.full_text = new_full_text
+        curr_tok = self
+        while curr_tok := curr_tok.next_token:
+            curr_tok.text_slice.full_text = new_full_text
+        curr_tok = self
+        while curr_tok := curr_tok.prior_token:
+            curr_tok.text_slice.full_text = new_full_text

pbi_parsers/dax/tokens.py

@@ -2,6 +2,7 @@ from dataclasses import dataclass
 from enum import Enum, auto
 
 from pbi_parsers.base import BaseToken
+from pbi_parsers.base.tokens import TextSlice
 
 
 class TokenType(Enum):
@@ -39,6 +40,8 @@ class TokenType(Enum):
     UNQUOTED_IDENTIFIER = auto()
     VARIABLE = auto()
     WHITESPACE = auto()
+    UNKNOWN = auto()
+    """unknown is used when someone replaces a token with a str"""
 
 
 KEYWORD_MAPPING = {
@@ -52,3 +55,17 @@ KEYWORD_MAPPING = {
 @dataclass
 class Token(BaseToken):
     tok_type: TokenType = TokenType.EOF
+
+    @staticmethod
+    def from_str(value: str, tok_type: TokenType = TokenType.UNKNOWN) -> "Token":
+        tok_type = KEYWORD_MAPPING.get(value, tok_type)
+        return Token(
+            tok_type=tok_type,
+            text_slice=TextSlice(value, 0, len(value)),
+        )
+
+    def add_token_before(self, text: str, tok_type: TokenType) -> None:
+        super().add_token_before(text, tok_type)
+
+    def add_token_after(self, text: str, tok_type: TokenType) -> None:
+        super().add_token_after(text, tok_type)

pbi_parsers/pq/tokens.py

@@ -3,6 +3,8 @@ from enum import Enum
 
 from pbi_parsers.base import BaseToken
 
+from ..base.tokens import TextSlice
+
 
 class TokenType(Enum):
     LET = 1
@@ -52,12 +54,27 @@ class TokenType(Enum):
     IS = 46
     AS = 47
     EXCLAMATION_POINT = 48
+    UNKNOWN = 99
+    """unknown is used when someone replaces a token with a str"""
 
 
 @dataclass
 class Token(BaseToken):
     tok_type: TokenType = TokenType.EOF
 
+    @staticmethod
+    def from_str(value: str, tok_type: TokenType = TokenType.UNKNOWN) -> "Token":
+        return Token(
+            tok_type=tok_type,
+            text_slice=TextSlice(value, 0, len(value)),
+        )
+
+    def add_token_before(self, text: str, tok_type: TokenType) -> None:
+        super().add_token_before(text, tok_type)
+
+    def add_token_after(self, text: str, tok_type: TokenType) -> None:
+        super().add_token_after(text, tok_type)
+
 
 # These are tokens that could also be used as identifiers in expressions.
 TEXT_TOKENS = (

{pbi_parsers-0.8.0.dist-info → pbi_parsers-0.8.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pbi_parsers
-Version: 0.8.0
+Version: 0.8.2
 Summary: Power BI lexer, parsers, and formatters for DAX and M (Power Query) languages
 Project-URL: Homepage, https://github.com/douglassimonsen/pbi_parsers
 Project-URL: Documentation, https://douglassimonsen.github.io/pbi_parsers/

{pbi_parsers-0.8.0.dist-info → pbi_parsers-0.8.2.dist-info}/RECORD

@@ -1,13 +1,13 @@
-pbi_parsers/__init__.py,sha256=cFx9LLUu1MM9UFcbhuA1oauYK8JiPDwU0ohEoJnrI_I,91
+pbi_parsers/__init__.py,sha256=o6tdbz0lucvp3ShZEGQdvolSfiDl90As5VtF8yWSzBc,91
 pbi_parsers/base/__init__.py,sha256=U7QpzFFD9A4wK3ZHin6xg5fPgTca0y5KC-O7nrZ-flM,115
-pbi_parsers/base/lexer.py,sha256=5iOkdYzJ9wGwz7I4rDlc7slrRNUPzi0oFFkDxN1d62M,4180
-pbi_parsers/base/tokens.py,sha256=slIVl4673xXomqHMgrn1ApHs8YRvcC2yQgs1zfSpe1U,2220
+pbi_parsers/base/lexer.py,sha256=Rl2cWlySJblFHvwW8oMVAjnCh83Zjo4N7jW3pkRZZO0,4335
+pbi_parsers/base/tokens.py,sha256=c1PIAU5oRxZbI74SY5KT3AZ3WrcukzXtuiLPrZU4f2o,7017
 pbi_parsers/dax/__init__.py,sha256=w8tfYFRwfjndq-QNnYQO3cu8fri4-OlG-edxUAKunF4,479
 pbi_parsers/dax/formatter.py,sha256=jOFnwcgQGIzsmi5sfkKoB_pFEGjDPd8E_pwMPwudmy4,7674
 pbi_parsers/dax/lexer.py,sha256=2_pERJSrSYd8VujOe9TxJa9R2Ex8mvP-bCotH7uVBZY,8025
 pbi_parsers/dax/main.py,sha256=FG35XCAPEooXoJShSgOnmQ0py-h_MPtOfnLpQWy61is,1657
 pbi_parsers/dax/parser.py,sha256=QLKrIBcxZ26TGhTHpeKcTGEHEHUDLC6IgpxxrdJzdek,1821
-pbi_parsers/dax/tokens.py,sha256=nY1laCbL8vwALpJ4jcd8k4lAscqOwqdw3dFuj4_KKVk,1234
+pbi_parsers/dax/tokens.py,sha256=vWkHbxWg_tU_vkzCayrK6K8W2CQPPghY0sshRwFAvoc,1886
 pbi_parsers/dax/utils.py,sha256=OURPa-b6Ldn0_KKXPdLIPA3Zdc12OfbbFd2X5SocCek,4402
 pbi_parsers/dax/exprs/__init__.py,sha256=OUfiXzZYp5HkTPE9x595MMxpsgG1IvsED8p8spAKZuk,3432
 pbi_parsers/dax/exprs/_base.py,sha256=bMHLICgAUOqAKl_S9d6V8kk62cqM1jynUY2-gBJlEcs,2732
@@ -39,7 +39,7 @@ pbi_parsers/pq/formatter.py,sha256=gcqj_aP8o5V10ULi5hdGhy3aAOy829jTKAfzH4mZewA,3
 pbi_parsers/pq/lexer.py,sha256=YOo4chz1N06FLO7cU4-wSoemIzfwG30NeUSJhJB-yOE,8093
 pbi_parsers/pq/main.py,sha256=4k5ZT-dRv5g2jjFgL1ckSpLR36wzClxe1YjiiIiBMu8,1649
 pbi_parsers/pq/parser.py,sha256=Fy8cqAGvGv1oVg4vYWJAGHZSWimEJ3wTtL5eqIkfOA8,1885
-pbi_parsers/pq/tokens.py,sha256=tll_fijLQ2reUJZIgyTW_a6ewuxiw9dva4dH9zx4GZ0,1637
+pbi_parsers/pq/tokens.py,sha256=WHaAIvGaoj22b-YbuFF_t0h5fLdaia3ZdNTGRXK8PiA,2237
 pbi_parsers/pq/exprs/__init__.py,sha256=wV-G51GagUAkA6_uVjsNA5JskO2JN3xXJPjKtzCH5rU,2845
 pbi_parsers/pq/exprs/_base.py,sha256=GcfWW3rannZBvw4LyjdiGbWGJ1nctw-m5k6LGkX7Wk4,1118
 pbi_parsers/pq/exprs/_utils.py,sha256=kUCWSzCSy7HMKOWcGjmO4R1WiYHP23afXmq67A0ZAXY,1065
@@ -73,7 +73,7 @@ pbi_parsers/pq/exprs/statement.py,sha256=JSg48pGAU3Ka2pt4lzVsYlVOqGeF_ARGm8Ajf0l
 pbi_parsers/pq/exprs/try_expr.py,sha256=UcnqfA-t9S1LVrKqeNUT8n4JJcO-ZQZoJrxAdjJ-GMA,1692
 pbi_parsers/pq/exprs/type_expr.py,sha256=hH5ubrIJaxwQsopNJHUZ4ByS1rHEgv2Tf8ocYqSukXM,2570
 pbi_parsers/pq/exprs/variable.py,sha256=wp4t0QHIGA264sXnWp7XVe1H8MJzMIOaoLNBQe-dfNk,1602
-pbi_parsers-0.8.0.dist-info/METADATA,sha256=o3D15LUYr8IlrK-TNjptPgLFlSvwHF7BqmkXfOPFJtA,2906
-pbi_parsers-0.8.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-pbi_parsers-0.8.0.dist-info/licenses/LICENSE,sha256=Sn0IfXOE4B0iL9lZXmGmRuTGyJeCtefxcfws0bLjp2g,1072
-pbi_parsers-0.8.0.dist-info/RECORD,,
+pbi_parsers-0.8.2.dist-info/METADATA,sha256=lw6AvVHgL0ERGlP7LEZtl343oezahwNwDJ4FJCXFbGM,2906
+pbi_parsers-0.8.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+pbi_parsers-0.8.2.dist-info/licenses/LICENSE,sha256=Sn0IfXOE4B0iL9lZXmGmRuTGyJeCtefxcfws0bLjp2g,1072
+pbi_parsers-0.8.2.dist-info/RECORD,,