pypbip 0.1.0__py3-none-any.whl

pypbip/__init__.py

@@ -0,0 +1,21 @@
+from .model_bim import ModelBim, SemanticModel, Table, DaxExpression, Column, Measure
+from .report import Report, Section, VisualContainer, ResourcePackage
+from .diagram import DiagramLayout, Diagram, DiagramNode
+
+__all__ = [
+    "ModelBim",
+    "SemanticModel",
+    "Table",
+    "DaxExpression",
+    "Column",
+    "Measure",
+    "Report",
+    "Section",
+    "VisualContainer",
+    "ResourcePackage",
+    "DiagramLayout",
+    "Diagram",
+    "DiagramNode",
+]
+
+

pypbip/diagram/__init__.py

@@ -0,0 +1,9 @@
+from .diagram_layout import DiagramLayout
+from .diagram import Diagram
+from .diagram_node import DiagramNode
+
+__all__ = [
+    "DiagramLayout",
+    "Diagram",
+    "DiagramNode",
+]

pypbip/diagram/diagram.py

@@ -0,0 +1,68 @@
+from typing import Dict, List
+
+from .diagram_node import DiagramNode
+from ..json_util import dump_json
+
+
+class Diagram:
+    def __init__(
+        self,
+        name: str,
+        ordinal: int,
+        nodes: List[DiagramNode],
+        scroll_position: Dict[str, float],
+        zoom_value: float,
+        pin_key_fields_to_top: bool,
+        show_extra_header_info: bool,
+        hide_key_fields_when_collapsed: bool,
+        tables_locked: bool,
+    ):
+        self.name = name
+        self.ordinal = ordinal
+        self.nodes = nodes
+        self.scroll_position = scroll_position
+        self.zoom_value = zoom_value
+        self.pin_key_fields_to_top = pin_key_fields_to_top
+        self.show_extra_header_info = show_extra_header_info
+        self.hide_key_fields_when_collapsed = hide_key_fields_when_collapsed
+        self.tables_locked = tables_locked
+        self.node_list = [n.name for n in self.nodes]
+        self.summary = {
+            "nodes": self.node_list,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "Diagram":
+        return cls(
+            name=data.get("name"),
+            ordinal=data.get("ordinal"),
+            nodes=[DiagramNode.from_dict(n) for n in data.get("nodes")],
+            scroll_position=data.get("scrollPosition"),
+            zoom_value=data.get("zoomValue"),
+            pin_key_fields_to_top=data.get("pinKeyFieldsToTop"),
+            show_extra_header_info=data.get("showExtraHeaderInfo"),
+            hide_key_fields_when_collapsed=data.get("hideKeyFieldsWhenCollapsed"),
+            tables_locked=data.get("tablesLocked"),
+        )
+
+    def to_dict(self) -> dict:
+        return {
+            "ordinal": self.ordinal,
+            "scrollPosition": self.scroll_position,
+            "nodes": [n.to_dict() for n in self.nodes],
+            "name": self.name,
+            "zoomValue": self.zoom_value,
+            "pinKeyFieldsToTop": self.pin_key_fields_to_top,
+            "showExtraHeaderInfo": self.show_extra_header_info,
+            "hideKeyFieldsWhenCollapsed": self.hide_key_fields_when_collapsed,
+            "tablesLocked": self.tables_locked,
+        }
+
+    def to_json(self, indent: int = 2) -> str:
+        return dump_json(self.to_dict(), indent=indent)
+
+
+if __name__ == "__main__":
+    diagram_dict = {"name": "ExampleDiagram", "ordinal": 0, "nodes": []}
+    diagram = Diagram.from_dict(diagram_dict)
+    print(diagram.to_json())

pypbip/diagram/diagram_layout.py

@@ -0,0 +1,115 @@
+import contextlib
+from ..json_util import read_json_file
+from typing import List, Dict
+from .diagram import Diagram
+from .diagram_node import DiagramNode
+from ..report import Section
+from ..model_bim import ModelBim, Table
+
+
+class DiagramLayout:
+    def __init__(
+        self,
+        version: str,
+        diagrams: List[Diagram],
+        selectedDiagram: str,
+        defaultDiagram: str,
+    ):
+        self.version = version
+        self.diagrams = diagrams
+        self.selectedDiagram = selectedDiagram
+        self.defaultDiagram = defaultDiagram
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "DiagramLayout":
+        return cls(
+            version=data.get("version"),
+            diagrams=[Diagram.from_dict(d) for d in data.get("diagrams", [])],
+            selectedDiagram=data.get("selectedDiagram"),
+            defaultDiagram=data.get("defaultDiagram"),
+        )
+
+    def to_dict(self) -> dict:
+        return {
+            "version": self.version,
+            "diagrams": [d.to_dict() for d in self.diagrams],
+            "selectedDiagram": self.selectedDiagram,
+            "defaultDiagram": self.defaultDiagram,
+        }
+
+    @classmethod
+    def from_file(cls, file_path: str) -> "DiagramLayout":
+        data = read_json_file(file_path)
+        return cls.from_dict(data)
+
+    def create_nodes(self, tables: List[str], mapping: Dict[str, Table]) -> List[DiagramNode]:
+        nodes = []
+        width = 300
+        height = 300
+        for idx, table in enumerate(tables):
+            with contextlib.suppress(Exception):
+                node_dict = {
+                    "location": {
+                        "x": 0 + idx * (width + 30),
+                        "y": 50 + 0 * height,
+                    },
+                    "nodeIndex": table,
+                    "nodeLineageTag": mapping[table].lineageTag,
+                    "size": {
+                        "height": height,
+                        "width": width
+                    },
+                    "zIndex": 0
+                }
+                node = DiagramNode.from_dict(node_dict)
+                nodes.append(node)
+        return nodes
+
+    def create_diagram(self, nodes: List[DiagramNode], name: str) -> Diagram:
+        diagram_dict = {
+            "ordinal": 99,
+            "scrollPosition": {
+                "x": 0,
+                "y": 0
+            },
+            "nodes": [node.to_dict() for node in nodes],
+            "name": f"@ {name}",
+            "zoomValue": 100,
+            "pinKeyFieldsToTop": True,
+            "showExtraHeaderInfo": False,
+            "hideKeyFieldsWhenCollapsed": False,
+            "tablesLocked": False
+        }
+        return Diagram.from_dict(diagram_dict)
+
+    def delete_existing_diagrams_with_section_names(self, section_names):
+        self.diagrams = [
+            diagram
+            for diagram in self.diagrams
+            if not diagram.name.startswith("@")
+        ]
+
+    def add_sections_diagrams(self, sections: List[Section], dataset: ModelBim):
+        section_names = []
+        sections_diagrams = []
+        for section in sections:
+            tables_from_fields = []
+            used_measures = [f.split(".")[-1] for f in section.used_fields if f.split(".")[-1] in dataset.model.measures.keys()]
+            for measure in used_measures:
+                field_used_tables = dataset.model.measures[measure].used_tables
+                tables_from_fields.extend(field_used_tables)
+            used_tables = section.used_tables + tables_from_fields
+            used_tables = list(set(used_tables))
+            nodes = self.create_nodes(used_tables, dataset.model.tables)
+            diagram = self.create_diagram(nodes, section.displayName)
+            sections_diagrams.append(diagram)
+            section_names.append(section.displayName)
+
+        self.delete_existing_diagrams_with_section_names(section_names)
+        self.diagrams.extend(sections_diagrams)
+
+
+if __name__ == "__main__":
+    diagramLayout_path = "BaseDataSet.Dataset/diagramLayout.json"
+    dl = DiagramLayout.from_file(diagramLayout_path)
+    print(dl.to_dict())

pypbip/diagram/diagram_node.py

@@ -0,0 +1,51 @@
+from typing import Optional, Dict
+
+
+class DiagramNode:
+    def __init__(
+        self,
+        name: str,
+        location: Dict[str, float],
+        node_lineage_tag: str,
+        size: Dict[str, float],
+        z_index: int,
+        expanded_height: Optional[float] = None,
+    ):
+        self.name = name
+        self.location = location
+        self.node_lineage_tag = node_lineage_tag
+        self.size = size
+        self.z_index = z_index
+        self.expanded_height = expanded_height
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "DiagramNode":
+        return cls(
+            name=data.get("nodeIndex"),
+            location=data.get("location"),
+            node_lineage_tag=data.get("nodeLineageTag"),
+            size=data.get("size"),
+            z_index=data.get("zIndex"),
+            expanded_height=data.get("expandedHeight"),
+        )
+
+    def to_dict(self) -> dict:
+        base_dict = {
+            "location": self.location,
+            "nodeIndex": self.name,
+            "nodeLineageTag": self.node_lineage_tag,
+            "size": self.size,
+            "zIndex": self.z_index,
+        }
+        if self.expanded_height is not None:
+            base_dict["expandedHeight"] = self.expanded_height
+
+        for k in list(base_dict.keys()):
+            if base_dict[k] in (None, "", {}):
+                del base_dict[k]
+        return base_dict
+
+
+if __name__ == "__main__":
+    node = DiagramNode.from_dict({"nodeIndex": "ExampleNode", "location": {"x": 0, "y": 0}, "nodeLineageTag": "example-tag", "size": {"height": 100, "width": 200}, "zIndex": 0})
+    print(node.to_json())

pypbip/json_util.py

@@ -0,0 +1,24 @@
+import json
+
+
+def write_json_file(filepath, content):
+    with open(filepath, "w", encoding="utf-8", newline="\n") as output_file:
+        json.dump(content, output_file, indent=2, ensure_ascii=False)
+
+
+def read_json_file(file_path) -> dict:
+    with open(file_path, "r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def dump_json(obj, indent=2) -> str:
+    return json.dumps(obj, indent=indent, ensure_ascii=False)
+
+
+def dump_json_inline(obj) -> str:
+    return json.dumps(obj)
+
+
+def parse_json(s: str):
+    return json.loads(s)
+

pypbip/model_bim/__init__.py

@@ -0,0 +1,17 @@
+from .model_bim import ModelBim
+from .semantic_model import SemanticModel
+from .table import Table
+from .dax_expression import DaxExpression
+from .column import Column
+from .measures import Measure
+
+__all__ = [
+    "ModelBim",
+    "SemanticModel",
+    "Table",
+    "DaxExpression",
+    "Column",
+    "Measure",
+]
+
+

pypbip/model_bim/column.py

@@ -0,0 +1,104 @@
+from ..json_util import read_json_file, dump_json
+from .dax_expression import DaxExpression
+
+
+class Column:
+    def __init__(
+            self,
+            sortByColumn,
+            expression,
+            isDataTypeInferred,
+            extendedProperties,
+            dataType,
+            description,
+            summarizeBy,
+            type,
+            isHidden,
+            name,
+            formatString,
+            changedProperties,
+            isNameInferred,
+            relatedColumnDetails,
+            sourceColumn,
+            lineageTag,
+            displayFolder,
+            annotations,
+        ):
+        self.sortByColumn = sortByColumn
+        self.expression = DaxExpression(expression)
+        self.isDataTypeInferred = isDataTypeInferred
+        self.extendedProperties = extendedProperties
+        self.dataType = dataType
+        self.description = description
+        self.summarizeBy = summarizeBy
+        self.type = type
+        self.isHidden = isHidden
+        self.name = name
+        self.formatString = formatString
+        self.changedProperties = changedProperties
+        self.isNameInferred = isNameInferred
+        self.relatedColumnDetails = relatedColumnDetails
+        self.sourceColumn = sourceColumn
+        self.lineageTag = lineageTag
+        self.displayFolder = displayFolder
+        self.annotations = annotations
+
+    @classmethod
+    def from_file(cls, file_path: str) -> "Column":
+        data: dict = read_json_file(file_path)
+        return Column.from_dict(data)
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "Column":
+        return cls(
+            sortByColumn=data.get("sortByColumn"),
+            expression=data.get("expression"),
+            isDataTypeInferred=data.get("isDataTypeInferred"),
+            extendedProperties=data.get("extendedProperties"),
+            dataType=data.get("dataType"),
+            description=data.get("description"),
+            summarizeBy=data.get("summarizeBy"),
+            type=data.get("type"),
+            isHidden=data.get("isHidden"),
+            name=data.get("name"),
+            formatString=data.get("formatString"),
+            changedProperties=data.get("changedProperties"),
+            isNameInferred=data.get("isNameInferred"),
+            relatedColumnDetails=data.get("relatedColumnDetails"),
+            sourceColumn=data.get("sourceColumn"),
+            lineageTag=data.get("lineageTag"),
+            displayFolder=data.get("displayFolder"),
+            annotations=data.get("annotations"),
+        )
+
+    def to_dict(self) -> dict:
+        base_dict = {
+            "name": self.name,
+            "annotations": self.annotations,
+            "changedProperties": self.changedProperties,
+            "dataType": self.dataType,
+            "description": self.description,
+            "displayFolder": self.displayFolder,
+            "expression": self.expression.to_dict(),
+            "extendedProperties": self.extendedProperties,
+            "formatString": self.formatString,
+            "isDataTypeInferred": self.isDataTypeInferred,
+            "isHidden": self.isHidden,
+            "isNameInferred": self.isNameInferred,
+            "lineageTag": self.lineageTag,
+            "relatedColumnDetails": self.relatedColumnDetails,
+            "sortByColumn": self.sortByColumn,
+            "sourceColumn": self.sourceColumn,
+            "summarizeBy": self.summarizeBy,
+            "type": self.type,
+        }
+        for k in list(base_dict.keys()):
+            if base_dict[k] in (None, "", []):
+                del base_dict[k]
+        return base_dict
+
+    def to_json(self, indent: int = 2) -> str:
+        return dump_json(self.to_dict(), indent=indent)
+
+    def clean_expression(self):
+        self.expression.clean_expression()

pypbip/model_bim/dax_expression.py

@@ -0,0 +1,122 @@
+from typing import List, Tuple
+from re import Match
+import re
+
+
+class DaxExpression:
+    def __init__(
+            self,
+            expression: str | List[str],
+        ):
+        self.expression: List[str] = expression if isinstance(expression, list) else [expression]
+
+    def to_dict(self) -> str | List[str]:
+        return self.expression[0] if len(self.expression) == 1 else self.expression
+
+    def as_str(self) -> str:
+        return "\n".join([x for x in self.expression if x])
+
+    def __remove_block_comments(self) -> None:
+        self.expression = re.sub(r'/\*.*?\*/', '', self.as_str(), flags=re.DOTALL).split("\n")
+
+    def __remove_hifen_comments(self) -> None:
+        self.expression = re.sub(r'--.*$', '', self.as_str(), flags=re.MULTILINE).split("\n")
+
+    def __remove_slash_comments(self) -> None:
+        pattern = r"""
+            ("(?:\\.|[^"\\])*"      # string entre aspas duplas (pode ter escapes)
+            |'(?:\\.|[^'\\])*'      # ou string entre aspas simples
+            )                       # <-- grupo 1: strings
+            |                       # ou
+            (//.*?$)                # <-- grupo 2: comentarios de linha
+        """
+        self.expression = re.sub(pattern, lambda m: m.group(1) or '', self.as_str(), flags=re.MULTILINE | re.VERBOSE).split("\n")
+
+    def remove_comments(self) -> None:
+        self.__remove_block_comments()
+        self.__remove_hifen_comments()
+        self.__remove_slash_comments()
+
+    def __remove_slash_r(self) -> None:
+        self.expression = [line.rstrip('\r') for line in self.expression]
+
+    def __replace_tab(self) -> None:
+        self.expression = [line.replace("\t", "    ") for line in self.expression]
+
+    def clean_expression(self) -> None:
+        if self.expression == [None]:
+            return
+        self.__remove_slash_r()
+        self.__replace_tab()
+
+    @staticmethod
+    def get_inner_calculate_boundates(measure: str, match: Match[str]) -> Tuple[int, int]:
+        inner_calculate_start_index = match.start(1)
+        inner_calculate_open_parenthesis = measure.find("(", match.end(1))
+        return inner_calculate_start_index, inner_calculate_open_parenthesis
+
+    @staticmethod
+    def check_if_nested_calculates_exist(measure: str) -> Tuple[bool, int, int]:
+        upper_measure = measure.upper()
+        pattern = re.compile(r"\bCALCULATE\s*\(\s*(CALCULATE)\s*\(", re.IGNORECASE)
+        match = pattern.search(upper_measure)
+        inner_calculate_start_index = None
+        inner_calculate_open_parenthesis = None
+        exists = False
+        if match:
+            exists = True
+            inner_calculate_start_index, inner_calculate_open_parenthesis = DaxExpression.get_inner_calculate_boundates(upper_measure, match)
+        return exists, inner_calculate_start_index, inner_calculate_open_parenthesis
+
+    @staticmethod
+    def check_if_inner_calculate_is_first_argument(measure: str, inner_calculate_close_parenthesis: int) -> bool:
+        after_close_parenthesis_content = measure[inner_calculate_close_parenthesis + 1:]
+        pattern = re.compile(r"^[ \t]*,", re.MULTILINE)
+        match = pattern.match(after_close_parenthesis_content)
+        return bool(match)
+
+    @staticmethod
+    def find_closing_parenthesis(text: str, open_index: int) -> int:
+        level = 0
+        for i in range(open_index - 1, len(text)):
+            if text[i] == "(":
+                level += 1
+            elif text[i] == ")":
+                level -= 1
+                if level == 0:
+                    return i
+        raise RuntimeError("Measure with parenthesis erros")
+
+    @staticmethod
+    def remove_inner_calculate(measure: str) -> str:
+        one_line_measure = " ".join([m.strip() for m in measure.split("\n")])
+
+        has_nested_calculate_exist, inner_calculate_start_index, inner_calculate_open_parenthesis = DaxExpression.check_if_nested_calculates_exist(one_line_measure)
+        if not has_nested_calculate_exist:
+            return measure
+
+        inner_calculate_close_parenthesis = DaxExpression.find_closing_parenthesis(one_line_measure, inner_calculate_open_parenthesis)
+        inner_calculate_boundaries = {
+            "inner_calculate_start_index": inner_calculate_start_index,
+            "inner_calculate_open_parenthesis": inner_calculate_open_parenthesis,
+            "inner_calculate_close_parenthesis": inner_calculate_close_parenthesis,
+        }
+
+        is_inner_calculate_the_first_argument = DaxExpression.check_if_inner_calculate_is_first_argument(one_line_measure, inner_calculate_boundaries["inner_calculate_close_parenthesis"])
+        if not is_inner_calculate_the_first_argument:
+            return measure
+
+        inner_calculate_start_index = inner_calculate_boundaries["inner_calculate_start_index"]
+        inner_calculate_open_parenthesis = inner_calculate_boundaries["inner_calculate_open_parenthesis"]
+        inner_calculate_close_parenthesis = inner_calculate_boundaries["inner_calculate_close_parenthesis"]
+
+        content_before_inner_calculate = one_line_measure[:inner_calculate_start_index]
+        inner_calculate_arguments = one_line_measure[inner_calculate_open_parenthesis + 1:inner_calculate_close_parenthesis]
+        content_after_inner_calculate = one_line_measure[inner_calculate_close_parenthesis + 1:]
+        improved_measure = f"{content_before_inner_calculate}{inner_calculate_arguments}{content_after_inner_calculate}"
+        return DaxExpression.remove_inner_calculate(improved_measure)
+
+    def simplify_expression(self):
+        self.expression = DaxExpression.remove_inner_calculate(self.as_str()).split("\n")
+
+

pypbip/model_bim/measures.py

@@ -0,0 +1,123 @@
+import re
+from ..json_util import read_json_file, dump_json
+from .dax_expression import DaxExpression
+from typing import Callable, Optional, List
+
+
+class Measure:
+    def __init__(
+            self,
+            name: str,
+            expression: str | List[str],
+            lineage_tag: str,
+            annotations: Optional[List[dict]] = None,
+            description: str | List[str] | None = None,
+            display_folder: Optional[str] = None,
+            format_string: Optional[str] = None,
+        ):
+        self.name = name
+        self.expression = DaxExpression(expression)
+        self.lineage_tag = lineage_tag
+        self.annotations = annotations
+        self.description = DaxExpression(description) if description else DaxExpression(expression)
+        self.display_folder = display_folder
+        self.format_string = format_string
+        self.used_tables: List[str] = self.get_used_tables()
+
+    def get_used_tables(self) -> list:
+        description = self.description.as_str()
+        if not description:
+            return []
+
+        tables = re.findall(r"[\'\"]?([A-Za-z0-9_]+)[\'\"]?\s*\[", description)
+
+        functions = re.findall(r"(?i)\b([A-Z_][A-Z0-9_]*)\s*\(", description)
+        for function in list(dict.fromkeys(functions)):
+            description = description.replace(function, "")
+
+        columns_or_measures = re.findall(r"(\[.*?\])", description)
+        for c_o_m in list(dict.fromkeys(columns_or_measures)):
+            description = description.replace(c_o_m, "")
+
+        matches = re.findall(r"\(([^()]*)\)", description)
+        cleaned = [re.sub(r"['\"\s]", "", m) for m in matches if m.strip()]
+        tables += cleaned
+        return list(set(tables))
+
+    @classmethod
+    def from_file(cls, file_path: str) -> "Measure":
+        data: dict = read_json_file(file_path)
+        return Measure.from_dict(data)
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "Measure":
+        return cls(
+            name=data.get("name"),
+            annotations=data.get("annotations"),
+            description=data.get("description"),
+            display_folder=data.get("displayFolder"),
+            expression=data.get("expression"),
+            format_string=data.get("formatString"),
+            lineage_tag=data.get("lineageTag"),
+        )
+
+    def to_dict(self) -> dict:
+        base_dict = {
+            "name": self.name,
+            "annotations": self.annotations,
+            "description": self.description.to_dict(),
+            "displayFolder": self.display_folder,
+            "expression": self.expression.to_dict(),
+            "formatString": self.format_string,
+            "lineageTag": self.lineage_tag,
+        }
+        for k in list(base_dict.keys()):
+            if base_dict[k] in (None, "", []):
+                del base_dict[k]
+        return base_dict
+
+    def to_json(self, indent: int = 2) -> str:
+        return dump_json(self.to_dict(), indent=indent)
+
+    def clean_expression(self):
+        self.expression.clean_expression()
+
+    def _sanitize(self) -> DaxExpression:
+        sanitized = DaxExpression(self.expression.expression)
+        sanitized.clean_expression()
+        sanitized.remove_comments()
+        return sanitized
+
+    def _apply_expanded(
+            self,
+            expanded_expression: str,
+            dax_formatter: Callable[[str], str] | None = None,
+        ) -> None:
+        desc = DaxExpression(expanded_expression)
+        desc.simplify_expression()
+        if dax_formatter:
+            formatted = dax_formatter(desc.as_str().replace("\n", " ")).split("\n")
+            desc = DaxExpression(formatted)
+        self.description = desc
+
+    def _is_original(self) -> bool:
+        expr = "".join(self.expression.expression).replace(" ", "").upper()
+        desc = "".join(self.description.expression).replace(" ", "").upper()
+        return expr == desc
+
+    def _augment_with_original(self) -> None:
+        augmented = self.expression.expression + [""] + self.description.expression
+        self.description = DaxExpression(augmented)
+
+    def expand_description(
+            self,
+            expand_expression: Callable[[str], str],
+            dax_formatter: Callable[[str], str] | None = None,
+        ) -> None:
+        sanitized = self._sanitize()
+        expanded = expand_expression(sanitized.as_str())
+        self._apply_expanded(expanded, dax_formatter)
+        if self._is_original():
+            self.description = DaxExpression(self.expression.expression)
+            return
+        self._augment_with_original()