semantic-link-labs 0.7.4__py3-none-any.whl → 0.8.1__py3-none-any.whl

sempy_labs/lakehouse/_lakehouse.py

@@ -1,9 +1,7 @@
 import sempy.fabric as fabric
 from tqdm.auto import tqdm
-from pyspark.sql import SparkSession
 from sempy_labs._helper_functions import resolve_lakehouse_name
 from typing import List, Optional, Union
-import sempy_labs._icons as icons
 from sempy._utils._log import log
 
 
@@ -16,6 +14,7 @@ def lakehouse_attached() -> bool:
     bool
         Returns True if a lakehouse is attached to the notebook.
     """
+    from pyspark.sql import SparkSession
 
     spark = SparkSession.builder.getOrCreate()
     lakeId = spark.conf.get("trident.lakehouse.id")
@@ -49,6 +48,7 @@ def optimize_lakehouse_tables(
         or if no lakehouse attached, resolves to the workspace of the notebook.
     """
 
+    from pyspark.sql import SparkSession
     from sempy_labs.lakehouse._get_lakehouse_tables import get_lakehouse_tables
     from delta import DeltaTable
 
@@ -107,6 +107,7 @@ def vacuum_lakehouse_tables(
         The default retention period is 168 hours (7 days) unless manually configured via table properties.
     """
 
+    from pyspark.sql import SparkSession
     from sempy_labs.lakehouse._get_lakehouse_tables import get_lakehouse_tables
     from delta import DeltaTable
 

sempy_labs/migration/_migrate_calctables_to_lakehouse.py

@@ -8,6 +8,7 @@ from sempy_labs._helper_functions import (
     resolve_lakehouse_id,
     create_abfss_path,
     retry,
+    generate_guid,
 )
 from sempy_labs.tom import connect_semantic_model
 from pyspark.sql import SparkSession
@@ -343,6 +344,8 @@ def migrate_field_parameters(
 
                 tbl = TOM.Table()
                 tbl.Name = tName
+                tbl.LineageTag = generate_guid()
+                tbl.SourceLineageTag = generate_guid()
                 tbl.Partitions.Add(par)
 
                 columns = ["Value1", "Value2", "Value3"]
@@ -352,6 +355,8 @@ def migrate_field_parameters(
                     col.Name = colName
                     col.SourceColumn = "[" + colName + "]"
                     col.DataType = TOM.DataType.String
+                    col.LineageTag = generate_guid()
+                    col.SourceLineageTag = generate_guid()
 
                     tbl.Columns.Add(col)
 

sempy_labs/report/__init__.py

@@ -1,3 +1,7 @@
+from sempy_labs.report._reportwrapper import (
+    ReportWrapper,
+)
+
 from sempy_labs.report._generate_report import (
     create_report_from_reportjson,
     get_report_definition,
@@ -10,15 +14,14 @@ from sempy_labs.report._report_functions import (
     export_report,
     clone_report,
     launch_report,
-    # list_report_pages,
-    # list_report_visuals,
-    # list_report_bookmarks,
     # translate_report_titles
 )
 from sempy_labs.report._report_rebind import (
     report_rebind,
     report_rebind_all,
 )
+from sempy_labs.report._report_bpa_rules import report_bpa_rules
+from sempy_labs.report._report_bpa import run_report_bpa
 
 __all__ = [
     "create_report_from_reportjson",
@@ -28,12 +31,12 @@ __all__ = [
     "export_report",
     "clone_report",
     "launch_report",
-    # list_report_pages,
-    # list_report_visuals,
-    # list_report_bookmarks,
     # translate_report_titles,
     "report_rebind",
     "report_rebind_all",
     "get_report_definition",
     "create_model_bpa_report",
+    "ReportWrapper",
+    "report_bpa_rules",
+    "run_report_bpa",
 ]

sempy_labs/report/_generate_report.py

@@ -313,7 +313,7 @@ def _create_report(
     dataset: str,
     dataset_workspace: Optional[str] = None,
     report_workspace: Optional[str] = None,
-    update_if_exists: Optional[bool] = False,
+    update_if_exists: bool = False,
 ):
 
     from sempy_labs.report import report_rebind

sempy_labs/report/_report_bpa.py

@@ -0,0 +1,369 @@
+import sempy.fabric as fabric
+from typing import Optional
+import pandas as pd
+import datetime
+from sempy._utils._log import log
+from sempy_labs.report import ReportWrapper, report_bpa_rules
+from sempy_labs._helper_functions import (
+    format_dax_object_name,
+    save_as_delta_table,
+    resolve_report_id,
+    resolve_lakehouse_name,
+    resolve_workspace_capacity,
+)
+from sempy_labs.lakehouse import get_lakehouse_tables, lakehouse_attached
+import sempy_labs._icons as icons
+from IPython.display import display, HTML
+import sempy_labs.report._report_helper as helper
+
+
+@log
+def run_report_bpa(
+    report: str,
+    rules: Optional[pd.DataFrame] = None,
+    workspace: Optional[str] = None,
+    # language: Optional[str] = None,
+    export: bool = False,
+    return_dataframe: bool = False,
+):
+    """
+    Displays an HTML visualization of the results of the Best Practice Analyzer scan for a report.
+
+    Parameters
+    ----------
+    report : str
+        Name of the report.
+    rules : pandas.DataFrame, default=None
+        A pandas dataframe containing rules to be evaluated.
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    export : bool, default=False
+        If True, exports the resulting dataframe to a delta table in the lakehouse attached to the notebook.
+    return_dataframe : bool, default=False
+        If True, returns a pandas dataframe instead of the visualization.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe in HTML format showing report objects which violated the best practice analyzer rules.
+    """
+
+    rpt = ReportWrapper(report=report, workspace=workspace)
+
+    dfCV = rpt.list_custom_visuals()
+    dfP = rpt.list_pages()
+    dfRF = rpt.list_report_filters()
+    dfRF["Filter Object"] = format_dax_object_name(
+        dfRF["Table Name"], dfRF["Object Name"]
+    )
+    dfPF = rpt.list_page_filters()
+    # Convert back to dataframe
+    # if isinstance(dfPF, pd.io.formats.style.Styler):
+    #    dfPF = dfPF.data
+    # if isinstance(dfP, pd.io.formats.style.Styler):
+    #    dfP = dfP.data
+
+    dfPF["Filter Object"] = (
+        dfPF["Page Display Name"]
+        + " : "
+        + format_dax_object_name(dfPF["Table Name"], dfPF["Object Name"])
+    )
+    dfVF = rpt.list_visual_filters()
+    dfVF["Filter Object"] = (
+        format_dax_object_name(dfVF["Page Display Name"], dfVF["Visual Name"])
+        + " : "
+        + format_dax_object_name(dfVF["Table Name"], dfVF["Object Name"])
+    )
+    dfRLM = rpt.list_report_level_measures()
+    dfV = rpt.list_visuals()
+    dfV["Visual Full Name"] = format_dax_object_name(
+        dfV["Page Display Name"], dfV["Visual Name"]
+    )
+    dfSMO = rpt.list_semantic_model_objects(extended=True)
+
+    # Translations
+    if rules is None:
+        rules = report_bpa_rules()
+
+    scope_to_dataframe = {
+        "Custom Visual": (dfCV, ["Custom Visual Display Name"]),
+        "Page": (dfP, ["Page Display Name"]),
+        "Visual": (dfV, ["Visual Full Name"]),
+        "Report Filter": (dfRF, ["Filter Object"]),
+        "Page Filter": (dfPF, ["Filter Object"]),
+        "Visual Filter": (dfVF, ["Filter Object"]),
+        "Report Level Measure": (dfRLM, ["Measure Name"]),
+        "Semantic Model": (dfSMO, ["Report Source Object"]),
+    }
+
+    def execute_rule(row):
+        scopes = row["Scope"]
+
+        # support both str and list as scope type
+        if isinstance(scopes, str):
+            scopes = [scopes]
+
+        # collect output dataframes
+        df_outputs = []
+
+        for scope in scopes:
+            # common fields for each scope
+            (df, violation_cols_or_func) = scope_to_dataframe[scope]
+
+            # execute rule and subset df
+            df_violations = df[row["Expression"](df)]
+
+            # subset the right output columns (e.g. Table Name & Column Name)
+            if isinstance(violation_cols_or_func, list):
+                violation_func = lambda violations: violations[violation_cols_or_func]
+            else:
+                violation_func = violation_cols_or_func
+
+            # build output data frame
+            df_output = violation_func(df_violations).copy()
+
+            df_output.columns = ["Object Name"]
+            df_output["Rule Name"] = row["Rule Name"]
+            df_output["Category"] = row["Category"]
+
+            # Handle 'Object Type' based on scope
+            if scope == "Semantic Model":
+                # Ensure 'Report Source Object' is unique in dfSMO
+                dfSMO_unique = dfSMO.drop_duplicates(subset="Report Source Object")
+                # Map 'Object Name' to the 'Report Source' column in dfSMO
+                df_output["Object Type"] = df_output["Object Name"].map(
+                    dfSMO_unique.set_index("Report Source Object")["Report Source"]
+                )
+            else:
+                df_output["Object Type"] = scope
+
+            df_output["Severity"] = row["Severity"]
+            df_output["Description"] = row["Description"]
+            df_output["URL"] = row["URL"]
+            df_output["Report URL"] = helper.get_web_url(
+                report=report, workspace=workspace
+            )
+
+            page_mapping_dict = dfP.set_index("Page Display Name")["Page URL"].to_dict()
+
+            if scope == "Page":
+                df_output["Report URL"] = df_output["Object Name"].map(
+                    page_mapping_dict
+                )
+            elif scope == "Page Filter":
+                df_output["Page Name"] = df_output["Object Name"].str.extract(
+                    r"(.*?) : '"
+                )
+                df_output["Report URL"] = df_output["Page Name"].map(page_mapping_dict)
+                df_output.drop(columns=["Page Name"], inplace=True)
+            elif scope in ["Visual", "Visual Filter"]:
+                df_output["Page Name"] = df_output["Object Name"].str.extract(
+                    r"'(.*?)'\[.*?\]"
+                )
+                df_output["Report URL"] = df_output["Page Name"].map(page_mapping_dict)
+                df_output.drop(columns=["Page Name"], inplace=True)
+
+            df_outputs.append(df_output)
+
+        return df_outputs
+
+    # flatten list of lists
+    flatten_dfs = [
+        df for dfs in rules.apply(execute_rule, axis=1).tolist() for df in dfs
+    ]
+
+    finalDF = pd.concat(flatten_dfs, ignore_index=True)
+    finalDF = finalDF[
+        [
+            "Category",
+            "Rule Name",
+            "Object Type",
+            "Object Name",
+            "Severity",
+            "Description",
+            "URL",
+            "Report URL",
+        ]
+    ]
+
+    if return_dataframe:
+        return finalDF
+
+    if export:
+        from pyspark.sql import SparkSession
+
+        if not lakehouse_attached():
+            raise ValueError(
+                f"{icons.red_dot} In order to export the BPA results, a lakehouse must be attached to the notebook."
+            )
+
+        now = datetime.datetime.now()
+        delta_table_name = "reportbparesults"
+        lakehouse_id = fabric.get_lakehouse_id()
+        lake_workspace = fabric.get_workspace_id()
+        lakehouse = resolve_lakehouse_name(
+            lakehouse_id=lakehouse_id, workspace=lake_workspace
+        )
+
+        lakeT = get_lakehouse_tables(lakehouse=lakehouse, workspace=lake_workspace)
+        lakeT_filt = lakeT[lakeT["Table Name"] == delta_table_name]
+
+        spark = SparkSession.builder.getOrCreate()
+        query = f"SELECT MAX(RunId) FROM {lakehouse}.{delta_table_name}"
+
+        if len(lakeT_filt) == 0:
+            runId = 1
+        else:
+            dfSpark = spark.sql(query)
+            maxRunId = dfSpark.collect()[0][0]
+            runId = maxRunId + 1
+
+        export_df = finalDF.copy()
+        capacity_id, capacity_name = resolve_workspace_capacity(workspace=workspace)
+        export_df["Capacity Name"] = capacity_name
+        export_df["Capacity Id"] = capacity_id
+        export_df["Workspace Name"] = workspace
+        export_df["Workspace Id"] = fabric.resolve_workspace_id(workspace)
+        export_df["Report Name"] = report
+        export_df["Report Id"] = resolve_report_id(report, workspace)
+        export_df["RunId"] = runId
+        export_df["Timestamp"] = now
+        export_df["RunId"] = export_df["RunId"].astype(int)
+
+        export_df = [
+            [
+                "Capacity Name",
+                "Capacity Id",
+                "Workspace Name",
+                "Workspace Id",
+                "Report Name",
+                "Report Id",
+                "Category",
+                "Rule Name",
+                "Object Type",
+                "Object Name",
+                "Severity",
+                "Description",
+                "URL",
+            ]
+        ]
+        save_as_delta_table(
+            dataframe=export_df,
+            delta_table_name=delta_table_name,
+            write_mode="append",
+            merge_schema=True,
+        )
+
+        return
+
+    finalDF.replace(
+        {"Warning": icons.warning, "Error": icons.error, "Info": icons.info},
+        inplace=True,
+    )
+
+    pd.set_option("display.max_colwidth", 100)
+
+    finalDF = (
+        finalDF[
+            [
+                "Category",
+                "Rule Name",
+                "Object Type",
+                "Object Name",
+                "Severity",
+                "Description",
+                "URL",
+                "Report URL",
+            ]
+        ]
+        .sort_values(["Category", "Rule Name", "Object Type", "Object Name"])
+        .set_index(["Category", "Rule Name"])
+    )
+
+    bpa2 = finalDF.reset_index()
+    bpa_dict = {
+        cat: bpa2[bpa2["Category"] == cat].drop("Category", axis=1)
+        for cat in bpa2["Category"].drop_duplicates().values
+    }
+
+    styles = """
+    <style>
+        .tab { overflow: hidden; border: 1px solid #ccc; background-color: #f1f1f1; }
+        .tab button { background-color: inherit; float: left; border: none; outline: none; cursor: pointer; padding: 14px 16px; transition: 0.3s; }
+        .tab button:hover { background-color: #ddd; }
+        .tab button.active { background-color: #ccc; }
+        .tabcontent { display: none; padding: 6px 12px; border: 1px solid #ccc; border-top: none; }
+        .tabcontent.active { display: block; }
+        .tooltip { position: relative; display: inline-block; }
+        .tooltip .tooltiptext { visibility: hidden; width: 300px; background-color: #555; color: #fff; text-align: center; border-radius: 6px; padding: 5px; position: absolute; z-index: 1; bottom: 125%; left: 50%; margin-left: -110px; opacity: 0; transition: opacity 0.3s; }
+        .tooltip:hover .tooltiptext { visibility: visible; opacity: 1; }
+    </style>
+    """
+
+    # JavaScript for tab functionality
+    script = """
+    <script>
+    function openTab(evt, tabName) {
+        var i, tabcontent, tablinks;
+        tabcontent = document.getElementsByClassName("tabcontent");
+        for (i = 0; i < tabcontent.length; i++) {
+            tabcontent[i].style.display = "none";
+        }
+        tablinks = document.getElementsByClassName("tablinks");
+        for (i = 0; i < tablinks.length; i++) {
+            tablinks[i].className = tablinks[i].className.replace(" active", "");
+        }
+        document.getElementById(tabName).style.display = "block";
+        evt.currentTarget.className += " active";
+    }
+    </script>
+    """
+
+    # HTML for tabs
+    tab_html = '<div class="tab">'
+    content_html = ""
+    for i, (title, df) in enumerate(bpa_dict.items()):
+        if df.shape[0] == 0:
+            continue
+
+        tab_id = f"tab{i}"
+        active_class = ""
+        if i == 0:
+            active_class = "active"
+
+        summary = " + ".join(
+            [f"{idx} ({v})" for idx, v in df["Severity"].value_counts().items()]
+        )
+        tab_html += f'<button class="tablinks {active_class}" onclick="openTab(event, \'{tab_id}\')"><b>{title}</b><br/>{summary}</button>'
+        content_html += f'<div id="{tab_id}" class="tabcontent {active_class}">'
+
+        # Adding tooltip for Rule Name using Description column
+        content_html += '<table border="1">'
+        content_html += "<tr><th>Rule Name</th><th>Object Type</th><th>Object Name</th><th>Severity</th></tr>"
+        for _, row in df.iterrows():
+            content_html += "<tr>"
+            if pd.notnull(row["URL"]):
+                content_html += f'<td class="tooltip" onmouseover="adjustTooltipPosition(event)"><a href="{row["URL"]}">{row["Rule Name"]}</a><span class="tooltiptext">{row["Description"]}</span></td>'
+            elif pd.notnull(row["Description"]):
+                content_html += f'<td class="tooltip" onmouseover="adjustTooltipPosition(event)">{row["Rule Name"]}<span class="tooltiptext">{row["Description"]}</span></td>'
+            else:
+                content_html += f'<td>{row["Rule Name"]}</td>'
+            content_html += f'<td>{row["Object Type"]}</td>'
+            if pd.notnull(row["Report URL"]):
+                content_html += (
+                    f'<td><a href="{row["Report URL"]}">{row["Object Name"]}</a></td>'
+                )
+            else:
+                content_html += f'<td>{row["Object Name"]}</td>'
+            # content_html += f'<td>{row["Object Name"]}</td>'
+            content_html += f'<td style="text-align: center;">{row["Severity"]}</td>'
+            content_html += "</tr>"
+        content_html += "</table>"
+
+        content_html += "</div>"
+    tab_html += "</div>"
+
+    # Display the tabs, tab contents, and run the script
+    return display(HTML(styles + tab_html + content_html + script))

sempy_labs/report/_report_bpa_rules.py

@@ -0,0 +1,113 @@
+import pandas as pd
+
+
+def report_bpa_rules() -> pd.DataFrame:
+    """
+    Shows the default rules for the report BPA used by the run_report_bpa function.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe containing the default rules for the run_report_bpa function.
+    """
+
+    rules = pd.DataFrame(
+        [
+            (
+                "Error Prevention",
+                "Semantic Model",
+                "Error",
+                "Fix report objects which reference invalid semantic model objects",
+                lambda df: df["Valid Semantic Model Object"] == False,
+                "This rule highlights visuals, report filters, page filters or visual filters which reference an invalid semantic model object (i.e Measure/Column/Hierarchy).",
+            ),
+            (
+                "Performance",
+                "Custom Visual",
+                "Warning",
+                "Remove custom visuals which are not used in the report",
+                lambda df: df["Used in Report"] == False,
+                "Removing unused custom visuals from a report may lead to faster report performance.",
+            ),
+            (
+                "Performance",
+                "Page",
+                "Warning",
+                "Reduce the number of visible visuals on the page",
+                lambda df: df["Visible Visual Count"] > 15,
+                'Reducing the number of visable visuals on a page will lead to faster report performance. This rule flags pages with over " + visVisuals + " visible visuals.',
+            ),
+            (
+                "Performance",
+                "Visual",
+                "Warning",
+                "Reduce the number of objects within visuals",
+                lambda df: df["Visual Object Count"] > 5,
+                "Reducing the number of objects (i.e. measures, columns) which are used in a visual will lead to faster report performance.",
+            ),
+            (
+                "Performance",
+                ["Report Filter", "Page Filter", "Visual Filter"],
+                "Warning",
+                "Reduce usage of filters on measures",
+                lambda df: df["Object Type"] == "Measure",
+                "Measure filters may cause performance degradation, especially against a large semantic model.",
+            ),
+            (
+                "Performance",
+                "Visual",
+                "Warning",
+                "Avoid setting 'Show items with no data' on columns",
+                lambda df: df["Show Items With No Data"],
+                "This setting will show all column values for all columns in the visual which may lead to performance degradation.",
+                "https://learn.microsoft.com/power-bi/create-reports/desktop-show-items-no-data",
+            ),
+            (
+                "Performance",
+                "Page",
+                "Warning",
+                "Avoid tall report pages with vertical scrolling",
+                lambda df: df["Height"] > 720,
+                "Report pages are designed to be in a single view and not scroll. Pages with scrolling is an indicator that the page has too many elements.",
+            ),
+            (
+                "Performance",
+                "Custom Visual",
+                "Info",
+                "Reduce usage of custom visuals",
+                lambda df: df["Custom Visual Name"] == df["Custom Visual Name"],
+                "Using custom visuals may lead to performance degradation.",
+            ),
+            (
+                "Maintenance",
+                "Report Level Measure",
+                "Info",
+                "Move report-level measures into the semantic model.",
+                lambda df: df["Measure Name"] == df["Measure Name"],
+                "It is a best practice to keep measures defined in the semantic model and not in the report.",
+            ),
+            (
+                "Performance",
+                ["Report Filter", "Page Filter", "Visual Filter"],
+                "Info",
+                "Reduce usage of TopN filtering within visuals",
+                lambda df: df["Type"] == "TopN",
+                "TopN filtering may cause performance degradation, especially against a high cardinality column.",
+            ),
+            # ('Performance', 'Custom Visual', 'Warning', "Set 'Edit Interactions' for non-data visuals to 'none'",
+            # lambda df: df['Custom Visual Name'] == df['Custom Visual Name'],
+            # "Setting 'Edit Interactions' to 'None' for non-data visuals may improve performance (since these visuals do not necessitate interactions between other visuals). 'Edit Interactions' may be found in the 'Format' tab of the ribbon in Power BI Desktop.",
+            # )
+        ],
+        columns=[
+            "Category",
+            "Scope",
+            "Severity",
+            "Rule Name",
+            "Expression",
+            "Description",
+            "URL",
+        ],
+    )
+
+    return rules