pbi-parsers 0.7.8__tar.gz

pbi_parsers-0.7.8/.github/workflows/publish-to-pypi.yml

@@ -0,0 +1,56 @@
+name: Publish Python Package to PyPI
+
+on:
+  push
+
+jobs:
+  build:
+    name: Build Package
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.x"
+
+      - name: Install pypa/build
+        run: >-
+          python3 -m
+          pip install
+          build
+          --user
+
+      - name: Build a binary wheel and a source tarball
+        run: python3 -m build
+
+      - name: Store the distribution packages
+        uses: actions/upload-artifact@v4
+        with:
+          name: python-package-distributions
+          path: dist/
+
+  publish-to-pypi:
+    name: Publish Package to PyPI
+    needs:
+      - build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/pbi_parsers # Replace <package-name> with your PyPI project name
+    permissions:
+      id-token: write # IMPORTANT: mandatory for trusted publishing
+
+    steps:
+      - name: Download all the dists
+        uses: actions/download-artifact@v4
+        with:
+          name: python-package-distributions
+          path: dist/
+
+      - name: Publish Package to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

pbi_parsers-0.7.8/.gitignore

@@ -0,0 +1,5 @@
+__pycache__
+venv
+dist
+docs/site
+.env

pbi_parsers-0.7.8/PKG-INFO

@@ -0,0 +1,66 @@
+Metadata-Version: 2.4
+Name: pbi_parsers
+Version: 0.7.8
+Summary: Power BI lexer, parsers, and formatters for DAX and M (Power Query) languages
+Requires-Python: >=3.11.0
+Requires-Dist: jinja2>=3.1.6
+Provides-Extra: dev
+Requires-Dist: build>=1.2.2; extra == 'dev'
+Requires-Dist: pre-commit>=3.8.0; extra == 'dev'
+Requires-Dist: ruff>=0.12.7; extra == 'dev'
+Provides-Extra: docs
+Requires-Dist: mkdocs-material>=9.6.16; extra == 'docs'
+Requires-Dist: mkdocs>=1.6.1; extra == 'docs'
+Requires-Dist: mkdocstrings-python>=0.30.0; extra == 'docs'
+Description-Content-Type: text/markdown
+
+# Overview
+
+Based on [Crafting Interpreters](https://timothya.com/pdfs/crafting-interpreters.pdf). Library provides lexers, parsers, and formatters for DAX and Power Query (M) languages. Designed to support code introspection and analysis, not execution. This enables developement of [ruff](https://github.com/astral-sh/ruff)-equivalent tools for DAX and Power Query. It also enables extracting metadata from DAX and Power Query code, such PQ source types (Excel, SQL, etc.) and DAX lineage dependencies.
+
+For more information, see the [docs](https://douglassimonsen.github.io/pbi_parsers/)
+
+# Installation
+
+```shell
+python -m pip install pbi_parsers
+```
+
+# Dev Instructions
+
+
+## Set Up
+
+```shell
+python -m venv venv
+venv\Scripts\activate
+python -m pip install .
+pre-commit install
+```
+
+
+# Running the Documentation Server
+
+```shell
+python -m pip install .[docs]
+mkdocs serve -f docs/mkdocs.yml
+```
+
+## Deploy docs to Github Pages
+
+```shell
+mkdocs  gh-deploy --clean -f docs/mkdocs.yml
+```
+
+## Testing
+
+```shell
+
+pip install -e .
+```
+
+# Build Wheel
+
+```shell
+python -m build .
+```

pbi_parsers-0.7.8/README.md

@@ -0,0 +1,50 @@
+# Overview
+
+Based on [Crafting Interpreters](https://timothya.com/pdfs/crafting-interpreters.pdf). Library provides lexers, parsers, and formatters for DAX and Power Query (M) languages. Designed to support code introspection and analysis, not execution. This enables developement of [ruff](https://github.com/astral-sh/ruff)-equivalent tools for DAX and Power Query. It also enables extracting metadata from DAX and Power Query code, such PQ source types (Excel, SQL, etc.) and DAX lineage dependencies.
+
+For more information, see the [docs](https://douglassimonsen.github.io/pbi_parsers/)
+
+# Installation
+
+```shell
+python -m pip install pbi_parsers
+```
+
+# Dev Instructions
+
+
+## Set Up
+
+```shell
+python -m venv venv
+venv\Scripts\activate
+python -m pip install .
+pre-commit install
+```
+
+
+# Running the Documentation Server
+
+```shell
+python -m pip install .[docs]
+mkdocs serve -f docs/mkdocs.yml
+```
+
+## Deploy docs to Github Pages
+
+```shell
+mkdocs  gh-deploy --clean -f docs/mkdocs.yml
+```
+
+## Testing
+
+```shell
+
+pip install -e .
+```
+
+# Build Wheel
+
+```shell
+python -m build .
+```

pbi_parsers-0.7.8/docs/docs/api/dax/formatter.md

@@ -0,0 +1 @@
+::: pbi_parsers.dax.Formatter

pbi_parsers-0.7.8/docs/docs/api/dax/lexer.md

@@ -0,0 +1 @@
+::: pbi_parsers.dax.Lexer

pbi_parsers-0.7.8/docs/docs/api/dax/parser.md

@@ -0,0 +1 @@
+::: pbi_parsers.dax.Parser

pbi_parsers-0.7.8/docs/docs/api/pq/formatter.md

@@ -0,0 +1 @@
+::: pbi_parsers.pq.Formatter

pbi_parsers-0.7.8/docs/docs/api/pq/lexer.md

@@ -0,0 +1 @@
+::: pbi_parsers.pq.Lexer

pbi_parsers-0.7.8/docs/docs/api/pq/parser.md

@@ -0,0 +1 @@
+::: pbi_parsers.pq.Parser

pbi_parsers-0.7.8/docs/docs/api/shared/lexer.md

@@ -0,0 +1 @@
+::: pbi_parsers.base.BaseLexer

pbi_parsers-0.7.8/docs/docs/api/shared/text_slice.md

@@ -0,0 +1 @@
+::: pbi_parsers.base.tokens.TextSlice

pbi_parsers-0.7.8/docs/docs/api/shared/token.md

@@ -0,0 +1 @@
+::: pbi_parsers.base.BaseToken

pbi_parsers-0.7.8/docs/docs/index.md

@@ -0,0 +1,151 @@
+# Overview
+
+Based on [Crafting Interpreters](https://timothya.com/pdfs/crafting-interpreters.pdf). Library provides lexers, parsers, and formatters for DAX and Power Query (M) languages. Designed to support code introspection and analysis, not execution. This enables development of [ruff](https://github.com/astral-sh/ruff)-equivalent tools for DAX and Power Query. It also enables extracting metadata from DAX and Power Query code, such PQ source types (Excel, SQL, etc.) and DAX lineage dependencies.
+
+# Installation
+
+```shell
+python -m pip install pbi_parsers
+```
+
+## Functionality
+
+!!! info "Rust Implementation"
+    Although the library is primarily implemented in Python, there are plans to implement a Rust version for performance and efficiency.
+
+- DAX
+    * [x] Lexer
+    * [x] Parser
+    * [x] Formatter
+    * [x] Testing
+    * [ ] Rust Implementation
+- Power Query (M)
+    * [x] Lexer
+    * [x] Parser
+    * [ ] Formatter
+    * [ ] Testing
+    * [ ] Rust Implementation
+  
+
+## Examples
+
+!!! info "Formatting DAX Expressions"
+    Like `ruff` for Python, this library can format DAX expressions to improve readability and maintainability.
+
+    ```python
+    from pbi_parsers.dax import format_expression
+
+    input_dax = """
+    func.name(arg1 + 1 + 2  + 3, func(), func(10000000000000), arg2)
+    """
+    formatted_dax = format_expression(input_dax)
+    print(formatted_dax)
+    # Output:
+    # func.name(
+    #     arg1 + 1 + 2 + 3,
+    #     func(),
+    #     func(10000000000000),
+    #     arg2
+    # )
+    ```
+
+!!! info "Creating AST Trees from DAX Expressions"
+    The library can parse DAX expressions into Abstract Syntax Trees (ASTs) for further analysis or manipulation.
+
+    ```python
+    from pbi_parsers.dax import to_ast
+
+    input_dax = """
+    func.name(arg1 + 1 + 2  + 3, func(), func(10000000000000), arg2)
+    """
+    ast = to_ast(input_dax)
+    print(ast)
+    # Output: 
+    # Function (
+    #     name: func.name,
+    #     args: Add (
+    #               left: Identifier (arg1),
+    #               right: Add (
+    #                          left: Number (1),
+    #                          right: Add (
+    #                                     left: Number (2),
+    #                                     right: Number (3)
+    #                                 )
+    #                      )
+    #           ),
+    #           Function (
+    #               name: func,
+    #               args:
+    #           ),
+    #           Function (
+    #               name: func,
+    #               args: Number (10000000000000)
+    #           ),
+    #           Identifier (arg2)
+    # )
+    ```
+
+!!! info "Highlighting DAX Sections"
+    The library can highlight sections of DAX code, making it easier to identify and analyze specific parts of the code.
+
+    Note: in the console, the caret (`^`) will be yellow and the line number will be cyan.
+
+    ```python
+    from pbi_parsers.dax import highlight_section, to_ast
+
+    input_dax = """
+    func.name(
+        arg1 + 
+        1 +
+            2 + 3,
+        func(),
+        func(10000000000000),
+        arg2
+    )
+    """
+    ast = to_ast(input_dax)
+    assert ast is not None, "AST should not be None"
+    section = ast.args[0].right.left  # the "1" in "arg1 + 1 + 2 + 3"
+    highlighted = highlight_section(section)
+    print(highlighted.to_console())
+
+    # Output:
+    # 1 | func.name(
+    # 2 |     arg1 +
+    # 3 |       1 +
+    #         ^
+    # 4 |         2 + 3,
+    # 5 |     func(),
+    ```
+
+    Highlighting a larger section:
+
+    ```python
+    from pbi_parsers.dax import highlight_section, to_ast
+
+    input_dax = """
+    func.name(
+        arg1 + 
+        1 +
+            2 + 3,
+        func(),
+        func(10000000000000),
+        arg2
+    )
+    """
+    ast = to_ast(input_dax)
+    assert ast is not None, "AST should not be None"
+    section = ast.args[0].right # The "1 + 2" in "arg1 + 1 + 2 + 3"
+    highlighted = highlight_section(section)
+    print(highlighted.to_console())
+    # Output:
+    # 1 | func.name(
+    # 2 |     arg1 +
+    # 3 |       1 +
+    #         ^^^
+    # 4 |         2 + 3,
+    # ^^^^^^^^^^^^^^^^^
+    # 5 |     func(),
+    # 6 |     func(10000000000000),
+
+    ```

pbi_parsers-0.7.8/docs/mkdocs.yml

@@ -0,0 +1,53 @@
+site_name: "pbi_parsers Documentation"
+
+nav:
+  - Home: index.md
+  - API:
+      - DAX:
+          - Lexer: api/dax/lexer.md
+          - Parser: api/dax/parser.md
+          - Formatter: api/dax/formatter.md
+      - PQ:
+          - Lexer: api/pq/lexer.md
+          - Parser: api/pq/parser.md
+          - Formatter: api/pq/formatter.md
+      - Shared:
+          - Lexer: api/shared/lexer.md
+          - Token: api/shared/token.md
+          - Text Slice: api/shared/text_slice.md
+
+theme:
+  name: "material"
+  features:
+    - content.code.copy
+    - content.code.select
+    - content.code.annotate
+
+plugins:
+  - search
+  - mkdocstrings:
+      default_handler: python
+      handlers:
+        python:
+          options:
+            backlinks: tree
+            separate_signature: true
+            show_signature_annotations: true
+
+markdown_extensions:
+  - pymdownx.highlight:
+      anchor_linenums: true
+      line_spans: __span
+      pygments_lang_class: true
+  - admonition
+  - pymdownx.details
+  - pymdownx.inlinehilite
+  - pymdownx.snippets
+  - pymdownx.superfences:
+      custom_fences:
+        - name: mermaid
+          class: mermaid
+          format: !!python/name:pymdownx.superfences.fence_code_format
+
+  - pymdownx.tasklist:
+      custom_checkbox: true

pbi_parsers-0.7.8/pbi_parsers/__init__.py

@@ -0,0 +1,9 @@
+from . import dax, pq
+
+__version__ = "0.7.8"
+
+
+__all__ = [
+    "dax",
+    "pq",
+]

pbi_parsers-0.7.8/pbi_parsers/base/__init__.py

@@ -0,0 +1,7 @@
+from .lexer import BaseLexer
+from .tokens import BaseToken
+
+__all__ = [
+    "BaseLexer",
+    "BaseToken",
+]

pbi_parsers-0.7.8/pbi_parsers/base/lexer.py

@@ -0,0 +1,127 @@
+from collections.abc import Callable
+
+from .tokens import BaseToken
+
+MAX_POSITION = 1_000_000
+
+
+class BaseLexer:
+    source: str
+    start_position: int
+    current_position: int
+    tokens: list[BaseToken]
+
+    def __init__(self, source: str) -> None:
+        self.source = source
+        self.start_position = 0
+        self.current_position = 0
+        self.tokens = []
+
+    def scan_helper(self) -> BaseToken:
+        """Contains the orchestration logic for converting tokens into expressions."""
+        msg = "Subclasses should implement match_tokens method."
+        raise NotImplementedError(msg)
+
+    def match(
+        self,
+        matcher: Callable[[str], bool] | str,
+        chunk: int = 1,
+        *,
+        case_insensitive: bool = True,
+    ) -> bool:
+        """Match a string or a callable matcher against the current position in the source.
+
+        Args:
+        ----
+            matcher (Callable[[str], bool] | str): A string to match or a callable that
+                takes a string and returns a boolean.
+            chunk (int): The number of characters to check from the current position.
+            case_insensitive (bool): If True, perform a case-insensitive match __only__ for strings.
+
+        """
+        if isinstance(matcher, str):
+            chunk = len(matcher)
+
+        string_chunk = self.peek(chunk)
+        if not string_chunk:
+            return False
+
+        if isinstance(matcher, str):
+            if case_insensitive:
+                string_chunk = string_chunk.lower()
+                matcher = matcher.lower()
+            if string_chunk == matcher:
+                self.advance(chunk)
+                return True
+            return False
+
+        if matcher(string_chunk):
+            self.advance(chunk)
+            return True
+        return False
+
+    def peek(self, chunk: int = 1) -> str:
+        """Returns the next chunk of text from the current position. Defaults to a single character.
+
+        Args:
+            chunk (int): The number of characters to return from the current position.
+
+        Returns:
+            str: The next chunk of text from the current position.
+
+        """
+        return (
+            self.source[self.current_position : self.current_position + chunk]
+            if self.current_position < len(self.source)
+            else ""
+        )
+
+    def remaining(self) -> str:
+        """Returns the remaining text from the current position to the end of the source.
+
+        Only used for testing and debugging purposes.
+
+        Returns:
+            str: The remaining text from the current position to the end of the source.
+
+        """
+        return self.source[self.current_position :]
+
+    def advance(self, chunk: int = 1) -> None:
+        """Advances the current position by the specified chunk size.
+
+        Generally used alongside peek to consume characters.
+
+        Args:
+            chunk (int): The number of characters to advance the current position.
+
+        Raises:
+            ValueError: If the current position exceeds a predefined MAX_POSITION (1,000,000 characters).
+                This is to avoid errors with the lexer causing the process to hang
+
+        """
+        if self.current_position > MAX_POSITION:
+            msg = f"Current position exceeds {MAX_POSITION:,} characters."
+            raise ValueError(msg)
+        self.current_position += chunk
+
+    def scan(self) -> tuple[BaseToken, ...]:
+        """Repeatedly calls scan_helper until the end of the source is reached.
+
+        Returns:
+            tuple[BaseToken, ...]: A tuple of tokens scanned from the source.
+
+        """
+        while not self.at_end():
+            self.tokens.append(self.scan_helper())
+        return tuple(self.tokens)
+
+    def at_end(self) -> bool:
+        """Checks if the current position is at (or beyond) the end of the source.
+
+        Returns:
+            bool: True if the current position is at or beyond the end of the source, False
+                otherwise.
+
+        """
+        return self.current_position >= len(self.source)

pbi_parsers-0.7.8/pbi_parsers/base/tokens.py

@@ -0,0 +1,61 @@
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class TextSlice:
+    full_text: str = ""
+    start: int = -1
+    end: int = -1
+
+    def __eq__(self, other: object) -> bool:
+        """Checks equality based on the text slice."""
+        if not isinstance(other, TextSlice):
+            return NotImplemented
+        return self.full_text == other.full_text and self.start == other.start and self.end == other.end
+
+    def __hash__(self) -> int:
+        """Returns a hash based on the text slice."""
+        return hash((self.full_text, self.start, self.end))
+
+    def __repr__(self) -> str:
+        """Returns a string representation of the TextSlice."""
+        return f"TextSlice(text='{self.get_text()}', start={self.start}, end={self.end})"
+
+    def get_text(self) -> str:
+        """Returns the text slice."""
+        return self.full_text[self.start : self.end]
+
+
+@dataclass
+class BaseToken:
+    tok_type: Any
+    text_slice: TextSlice = field(default_factory=TextSlice)
+
+    def __repr__(self) -> str:
+        pretty_text = self.text_slice.get_text().replace("\n", "\\n").replace("\r", "\\r")
+        return f"Token(type={self.tok_type.name}, text='{pretty_text}')"
+
+    @property
+    def text(self) -> str:
+        """Returns the text underlying the token."""
+        return self.text_slice.get_text()
+
+    def position(self) -> tuple[int, int]:
+        """Returns the start and end positions of the token.
+
+        Returns:
+            tuple[int, int]: A tuple containing the start and end positions of the token within the source text.
+
+        """
+        return self.text_slice.start, self.text_slice.end
+
+    def __eq__(self, other: object) -> bool:
+        """Checks equality based on token type and text slice."""
+        if not isinstance(other, BaseToken):
+            return NotImplemented
+        return self.tok_type == other.tok_type and self.text_slice == other.text_slice
+
+    def __hash__(self) -> int:
+        """Returns a hash based on token type and text slice."""
+        return hash((self.tok_type, self.text_slice))

pbi_parsers-0.7.8/pbi_parsers/dax/__init__.py

@@ -0,0 +1,22 @@
+from . import exprs, utils
+from .exprs import Expression
+from .formatter import Formatter
+from .lexer import Lexer
+from .main import format_expression, to_ast
+from .parser import Parser
+from .tokens import Token, TokenType
+from .utils import highlight_section
+
+__all__ = [
+    "Expression",
+    "Formatter",
+    "Lexer",
+    "Parser",
+    "Token",
+    "TokenType",
+    "exprs",
+    "format_expression",
+    "highlight_section",
+    "to_ast",
+    "utils",
+]