semantic-link-labs 0.8.3__py3-none-any.whl → 0.8.5__py3-none-any.whl

sempy_labs/_model_bpa.py

@@ -56,7 +56,7 @@ def run_model_bpa(
     extended : bool, default=False
         If True, runs the set_vertipaq_annotations function to collect Vertipaq Analyzer statistics to be used in the analysis of the semantic model.
     language : str, default=None
-        Specifying a language code (i.e. 'it-IT' for Italian) will auto-translate the Category, Rule Name and Description into the specified language.
+        Specifying a language name or code (i.e. 'it-IT' for Italian) will auto-translate the Category, Rule Name and Description into the specified language.
         Defaults to None which resolves to English.
 
     Returns
@@ -113,6 +113,7 @@ def run_model_bpa(
         )
 
     if extended:
+        icons.sll_tags.append("ModelBPAExtended")
         with connect_semantic_model(
             dataset=dataset, workspace=workspace, readonly=False
         ) as tom:
@@ -122,210 +123,239 @@ def run_model_bpa(
         dataset=dataset, workspace=workspace, readonly=True
     ) as tom:
 
-        dep = get_model_calc_dependencies(dataset=dataset, workspace=workspace)
-
-        def translate_using_po(rule_file):
-            current_dir = os.path.dirname(os.path.abspath(__file__))
-            translation_file = (
-                f"{current_dir}/_bpa_translation/_model/_translations_{language}.po"
+        # Do not run BPA for models with no tables
+        if tom.model.Tables.Count == 0:
+            print(
+                f"{icons.warning} The '{dataset}' semantic model within the '{workspace}' workspace has no tables and therefore there are no valid BPA results."
             )
-            for c in ["Category", "Description", "Rule Name"]:
-                po = polib.pofile(translation_file)
-                for entry in po:
-                    if entry.tcomment == c.lower().replace(" ", "_"):
-                        rule_file.loc[rule_file["Rule Name"] == entry.msgid, c] = (
-                            entry.msgstr
-                        )
+            finalDF = pd.DataFrame(
+                columns=[
+                    "Category",
+                    "Rule Name",
+                    "Severity",
+                    "Object Type",
+                    "Object Name",
+                    "Description",
+                    "URL",
+                ]
+            )
+        else:
+            dep = get_model_calc_dependencies(dataset=dataset, workspace=workspace)
 
-        translated = False
+            def translate_using_po(rule_file):
+                current_dir = os.path.dirname(os.path.abspath(__file__))
+                translation_file = (
+                    f"{current_dir}/_bpa_translation/_model/_translations_{language}.po"
+                )
+                for c in ["Category", "Description", "Rule Name"]:
+                    po = polib.pofile(translation_file)
+                    for entry in po:
+                        if entry.tcomment == c.lower().replace(" ", "_"):
+                            rule_file.loc[rule_file["Rule Name"] == entry.msgid, c] = (
+                                entry.msgstr
+                            )
 
-        # Translations
-        if language is not None and rules is None and language in language_list:
-            rules = model_bpa_rules(dependencies=dep)
-            translate_using_po(rules)
-            translated = True
-        if rules is None:
-            rules = model_bpa_rules(dependencies=dep)
-        if language is not None and not translated:
+            translated = False
 
-            def translate_using_spark(rule_file):
+            # Translations
+            if language is not None and rules is None and language in language_list:
+                rules = model_bpa_rules(dependencies=dep)
+                translate_using_po(rules)
+                translated = True
+            if rules is None:
+                rules = model_bpa_rules(dependencies=dep)
+            if language is not None and not translated:
 
-                from synapse.ml.services import Translate
-                from pyspark.sql import SparkSession
+                def translate_using_spark(rule_file):
 
-                rules_temp = rule_file.copy()
-                rules_temp = rules_temp.drop(["Expression", "URL", "Severity"], axis=1)
+                    from synapse.ml.services import Translate
+                    from pyspark.sql import SparkSession
 
-                schema = StructType(
-                    [
-                        StructField("Category", StringType(), True),
-                        StructField("Scope", StringType(), True),
-                        StructField("Rule Name", StringType(), True),
-                        StructField("Description", StringType(), True),
-                    ]
-                )
+                    rules_temp = rule_file.copy()
+                    rules_temp = rules_temp.drop(
+                        ["Expression", "URL", "Severity"], axis=1
+                    )
 
-                spark = SparkSession.builder.getOrCreate()
-                dfRules = spark.createDataFrame(rules_temp, schema)
-
-                columns = ["Category", "Rule Name", "Description"]
-                for clm in columns:
-                    translate = (
-                        Translate()
-                        .setTextCol(clm)
-                        .setToLanguage(language)
-                        .setOutputCol("translation")
-                        .setConcurrency(5)
+                    schema = StructType(
+                        [
+                            StructField("Category", StringType(), True),
+                            StructField("Scope", StringType(), True),
+                            StructField("Rule Name", StringType(), True),
+                            StructField("Description", StringType(), True),
+                        ]
                     )
 
-                    if clm == "Rule Name":
-                        transDF = (
-                            translate.transform(dfRules)
-                            .withColumn(
-                                "translation", flatten(col("translation.translations"))
-                            )
-                            .withColumn("translation", col("translation.text"))
-                            .select(clm, "translation")
+                    spark = SparkSession.builder.getOrCreate()
+                    dfRules = spark.createDataFrame(rules_temp, schema)
+
+                    columns = ["Category", "Rule Name", "Description"]
+                    for clm in columns:
+                        translate = (
+                            Translate()
+                            .setTextCol(clm)
+                            .setToLanguage(language)
+                            .setOutputCol("translation")
+                            .setConcurrency(5)
                         )
-                    else:
-                        transDF = (
-                            translate.transform(dfRules)
-                            .withColumn(
-                                "translation", flatten(col("translation.translations"))
+
+                        if clm == "Rule Name":
+                            transDF = (
+                                translate.transform(dfRules)
+                                .withColumn(
+                                    "translation",
+                                    flatten(col("translation.translations")),
+                                )
+                                .withColumn("translation", col("translation.text"))
+                                .select(clm, "translation")
+                            )
+                        else:
+                            transDF = (
+                                translate.transform(dfRules)
+                                .withColumn(
+                                    "translation",
+                                    flatten(col("translation.translations")),
+                                )
+                                .withColumn("translation", col("translation.text"))
+                                .select("Rule Name", clm, "translation")
                             )
-                            .withColumn("translation", col("translation.text"))
-                            .select("Rule Name", clm, "translation")
-                        )
 
-                    df_panda = transDF.toPandas()
-                    rule_file = pd.merge(
-                        rule_file,
-                        df_panda[["Rule Name", "translation"]],
-                        on="Rule Name",
-                        how="left",
-                    )
+                        df_panda = transDF.toPandas()
+                        rule_file = pd.merge(
+                            rule_file,
+                            df_panda[["Rule Name", "translation"]],
+                            on="Rule Name",
+                            how="left",
+                        )
 
-                    rule_file = rule_file.rename(
-                        columns={"translation": f"{clm}Translated"}
-                    )
-                    rule_file[f"{clm}Translated"] = rule_file[f"{clm}Translated"].apply(
-                        lambda x: x[0] if x is not None else None
-                    )
+                        rule_file = rule_file.rename(
+                            columns={"translation": f"{clm}Translated"}
+                        )
+                        rule_file[f"{clm}Translated"] = rule_file[
+                            f"{clm}Translated"
+                        ].apply(lambda x: x[0] if x is not None else None)
 
-                for clm in columns:
-                    rule_file = rule_file.drop([clm], axis=1)
-                    rule_file = rule_file.rename(columns={f"{clm}Translated": clm})
+                    for clm in columns:
+                        rule_file = rule_file.drop([clm], axis=1)
+                        rule_file = rule_file.rename(columns={f"{clm}Translated": clm})
 
-                return rule_file
+                    return rule_file
 
-            rules = translate_using_spark(rules)
+                rules = translate_using_spark(rules)
 
-        rules.loc[rules["Severity"] == "Warning", "Severity"] = icons.warning
-        rules.loc[rules["Severity"] == "Error", "Severity"] = icons.error
-        rules.loc[rules["Severity"] == "Info", "Severity"] = icons.info
+            rules.loc[rules["Severity"] == "Warning", "Severity"] = icons.warning
+            rules.loc[rules["Severity"] == "Error", "Severity"] = icons.error
+            rules.loc[rules["Severity"] == "Info", "Severity"] = icons.info
 
-        pd.set_option("display.max_colwidth", 1000)
+            pd.set_option("display.max_colwidth", 1000)
 
-        violations = pd.DataFrame(columns=["Object Name", "Scope", "Rule Name"])
+            violations = pd.DataFrame(columns=["Object Name", "Scope", "Rule Name"])
 
-        scope_to_dataframe = {
-            "Relationship": (
-                tom.model.Relationships,
-                lambda obj: create_relationship_name(
-                    obj.FromTable.Name,
-                    obj.FromColumn.Name,
-                    obj.ToTable.Name,
-                    obj.ToColumn.Name,
+            scope_to_dataframe = {
+                "Relationship": (
+                    tom.model.Relationships,
+                    lambda obj: create_relationship_name(
+                        obj.FromTable.Name,
+                        obj.FromColumn.Name,
+                        obj.ToTable.Name,
+                        obj.ToColumn.Name,
+                    ),
                 ),
-            ),
-            "Column": (
-                tom.all_columns(),
-                lambda obj: format_dax_object_name(obj.Parent.Name, obj.Name),
-            ),
-            "Measure": (tom.all_measures(), lambda obj: obj.Name),
-            "Hierarchy": (
-                tom.all_hierarchies(),
-                lambda obj: format_dax_object_name(obj.Parent.Name, obj.Name),
-            ),
-            "Table": (tom.model.Tables, lambda obj: obj.Name),
-            "Role": (tom.model.Roles, lambda obj: obj.Name),
-            "Model": (tom.model, lambda obj: obj.Model.Name),
-            "Calculation Item": (
-                tom.all_calculation_items(),
-                lambda obj: format_dax_object_name(obj.Parent.Table.Name, obj.Name),
-            ),
-            "Row Level Security": (
-                tom.all_rls(),
-                lambda obj: format_dax_object_name(obj.Parent.Name, obj.Name),
-            ),
-            "Partition": (
-                tom.all_partitions(),
-                lambda obj: format_dax_object_name(obj.Parent.Name, obj.Name),
-            ),
-        }
-
-        for i, r in rules.iterrows():
-            ruleName = r["Rule Name"]
-            expr = r["Expression"]
-            scopes = r["Scope"]
-
-            if isinstance(scopes, str):
-                scopes = [scopes]
-
-            for scope in scopes:
-                func = scope_to_dataframe[scope][0]
-                nm = scope_to_dataframe[scope][1]
-
-                if scope == "Model":
-                    x = []
-                    if expr(func, tom):
-                        x = ["Model"]
-                elif scope == "Measure":
-                    x = [nm(obj) for obj in tom.all_measures() if expr(obj, tom)]
-                elif scope == "Column":
-                    x = [nm(obj) for obj in tom.all_columns() if expr(obj, tom)]
-                elif scope == "Partition":
-                    x = [nm(obj) for obj in tom.all_partitions() if expr(obj, tom)]
-                elif scope == "Hierarchy":
-                    x = [nm(obj) for obj in tom.all_hierarchies() if expr(obj, tom)]
-                elif scope == "Table":
-                    x = [nm(obj) for obj in tom.model.Tables if expr(obj, tom)]
-                elif scope == "Relationship":
-                    x = [nm(obj) for obj in tom.model.Relationships if expr(obj, tom)]
-                elif scope == "Role":
-                    x = [nm(obj) for obj in tom.model.Roles if expr(obj, tom)]
-                elif scope == "Row Level Security":
-                    x = [nm(obj) for obj in tom.all_rls() if expr(obj, tom)]
-                elif scope == "Calculation Item":
-                    x = [
-                        nm(obj) for obj in tom.all_calculation_items() if expr(obj, tom)
-                    ]
-
-                if len(x) > 0:
-                    new_data = {"Object Name": x, "Scope": scope, "Rule Name": ruleName}
-                    violations = pd.concat(
-                        [violations, pd.DataFrame(new_data)], ignore_index=True
-                    )
+                "Column": (
+                    tom.all_columns(),
+                    lambda obj: format_dax_object_name(obj.Parent.Name, obj.Name),
+                ),
+                "Measure": (tom.all_measures(), lambda obj: obj.Name),
+                "Hierarchy": (
+                    tom.all_hierarchies(),
+                    lambda obj: format_dax_object_name(obj.Parent.Name, obj.Name),
+                ),
+                "Table": (tom.model.Tables, lambda obj: obj.Name),
+                "Role": (tom.model.Roles, lambda obj: obj.Name),
+                "Model": (tom.model, lambda obj: obj.Model.Name),
+                "Calculation Item": (
+                    tom.all_calculation_items(),
+                    lambda obj: format_dax_object_name(obj.Parent.Table.Name, obj.Name),
+                ),
+                "Row Level Security": (
+                    tom.all_rls(),
+                    lambda obj: format_dax_object_name(obj.Parent.Name, obj.Name),
+                ),
+                "Partition": (
+                    tom.all_partitions(),
+                    lambda obj: format_dax_object_name(obj.Parent.Name, obj.Name),
+                ),
+            }
+
+            for i, r in rules.iterrows():
+                ruleName = r["Rule Name"]
+                expr = r["Expression"]
+                scopes = r["Scope"]
+
+                if isinstance(scopes, str):
+                    scopes = [scopes]
+
+                for scope in scopes:
+                    func = scope_to_dataframe[scope][0]
+                    nm = scope_to_dataframe[scope][1]
+
+                    if scope == "Model":
+                        x = []
+                        if expr(func, tom):
+                            x = ["Model"]
+                    elif scope == "Measure":
+                        x = [nm(obj) for obj in tom.all_measures() if expr(obj, tom)]
+                    elif scope == "Column":
+                        x = [nm(obj) for obj in tom.all_columns() if expr(obj, tom)]
+                    elif scope == "Partition":
+                        x = [nm(obj) for obj in tom.all_partitions() if expr(obj, tom)]
+                    elif scope == "Hierarchy":
+                        x = [nm(obj) for obj in tom.all_hierarchies() if expr(obj, tom)]
+                    elif scope == "Table":
+                        x = [nm(obj) for obj in tom.model.Tables if expr(obj, tom)]
+                    elif scope == "Relationship":
+                        x = [
+                            nm(obj) for obj in tom.model.Relationships if expr(obj, tom)
+                        ]
+                    elif scope == "Role":
+                        x = [nm(obj) for obj in tom.model.Roles if expr(obj, tom)]
+                    elif scope == "Row Level Security":
+                        x = [nm(obj) for obj in tom.all_rls() if expr(obj, tom)]
+                    elif scope == "Calculation Item":
+                        x = [
+                            nm(obj)
+                            for obj in tom.all_calculation_items()
+                            if expr(obj, tom)
+                        ]
+
+                    if len(x) > 0:
+                        new_data = {
+                            "Object Name": x,
+                            "Scope": scope,
+                            "Rule Name": ruleName,
+                        }
+                        violations = pd.concat(
+                            [violations, pd.DataFrame(new_data)], ignore_index=True
+                        )
 
-        prepDF = pd.merge(
-            violations,
-            rules[["Rule Name", "Category", "Severity", "Description", "URL"]],
-            left_on="Rule Name",
-            right_on="Rule Name",
-            how="left",
-        )
-        prepDF.rename(columns={"Scope": "Object Type"}, inplace=True)
-        finalDF = prepDF[
-            [
-                "Category",
-                "Rule Name",
-                "Severity",
-                "Object Type",
-                "Object Name",
-                "Description",
-                "URL",
+            prepDF = pd.merge(
+                violations,
+                rules[["Rule Name", "Category", "Severity", "Description", "URL"]],
+                left_on="Rule Name",
+                right_on="Rule Name",
+                how="left",
+            )
+            prepDF.rename(columns={"Scope": "Object Type"}, inplace=True)
+            finalDF = prepDF[
+                [
+                    "Category",
+                    "Rule Name",
+                    "Severity",
+                    "Object Type",
+                    "Object Name",
+                    "Description",
+                    "URL",
+                ]
             ]
-        ]
 
     if export:
         if not lakehouse_attached():
@@ -387,10 +417,14 @@ def run_model_bpa(
         dfExport.insert(5, colName, dfExport.pop(colName))
 
         dfExport.columns = dfExport.columns.str.replace(" ", "_")
+        schema = {
+            key.replace(" ", "_"): value for key, value in icons.bpa_schema.items()
+        }
         save_as_delta_table(
             dataframe=dfExport,
             delta_table_name=delta_table_name,
             write_mode="append",
+            schema=schema,
             merge_schema=True,
         )
 

sempy_labs/_model_bpa_bulk.py

@@ -25,6 +25,7 @@ def run_model_bpa_bulk(
     language: Optional[str] = None,
     workspace: Optional[str | List[str]] = None,
     skip_models: Optional[str | List[str]] = ["ModelBPA", "Fabric Capacity Metrics"],
+    skip_models_in_workspace: Optional[dict] = None,
 ):
     """
     Runs the semantic model Best Practice Analyzer across all semantic models in a workspace (or all accessible workspaces).
@@ -33,8 +34,6 @@ def run_model_bpa_bulk(
 
     Parameters
     ----------
-    dataset : str
-        Name of the semantic model.
     rules : pandas.DataFrame, default=None
         A pandas dataframe containing rules to be evaluated. Based on the format of the dataframe produced by the model_bpa_rules function.
     extended : bool, default=False
@@ -47,6 +46,12 @@ def run_model_bpa_bulk(
         Defaults to None which scans all accessible workspaces.
     skip_models : str | List[str], default=['ModelBPA', 'Fabric Capacity Metrics']
         The semantic models to always skip when running this analysis.
+    skip_models_in_workspace : dict, default=None
+        A dictionary showing specific semantic models within specific workspaces to skip. See the example below:
+        {
+            "Workspace A": ["Dataset1", "Dataset2"],
+            "Workspace B": ["Dataset5", "Dataset 8"],
+        }
     """
 
     if not lakehouse_attached():
@@ -59,24 +64,6 @@ def run_model_bpa_bulk(
 
     skip_models.extend(["ModelBPA", "Fabric Capacity Metrics"])
 
-    cols = [
-        "Capacity Name",
-        "Capacity Id",
-        "Workspace Name",
-        "Workspace Id",
-        "Dataset Name",
-        "Dataset Id",
-        "Configured By",
-        "Rule Name",
-        "Category",
-        "Severity",
-        "Object Type",
-        "Object Name",
-        "Description",
-        "URL",
-        "RunId",
-        "Timestamp",
-    ]
     now = datetime.datetime.now()
     output_table = "modelbparesults"
     lakehouse_workspace = fabric.resolve_workspace_name()
@@ -86,7 +73,6 @@ def run_model_bpa_bulk(
     )
     lakeT = get_lakehouse_tables(lakehouse=lakehouse, workspace=lakehouse_workspace)
     lakeT_filt = lakeT[lakeT["Table Name"] == output_table]
-    # query = f"SELECT MAX(RunId) FROM {lakehouse}.{output_table}"
     if len(lakeT_filt) == 0:
         runId = 1
     else:
@@ -102,13 +88,22 @@ def run_model_bpa_bulk(
     else:
         dfW_filt = dfW[dfW["Name"].isin(workspace)]
 
+    if len(dfW_filt) == 0:
+        raise ValueError(
+            f"{icons.red_dot} There are no valid workspaces to assess. This is likely due to not having proper permissions to the workspace(s) entered in the 'workspace' parameter."
+        )
+
     for i, r in dfW_filt.iterrows():
         wksp = r["Name"]
         wksp_id = r["Id"]
         capacity_id, capacity_name = resolve_workspace_capacity(workspace=wksp)
-        df = pd.DataFrame(columns=cols)
+        df = pd.DataFrame(columns=list(icons.bpa_schema.keys()))
         dfD = fabric.list_datasets(workspace=wksp, mode="rest")
 
+        # Skip models in workspace
+        skip_models_wkspc = skip_models_in_workspace.get(wksp)
+        dfD = dfD[~dfD["Dataset Name"].isin(skip_models_wkspc)]
+
         # Exclude default semantic models
         if len(dfD) > 0:
             dfI = fabric.list_items(workspace=wksp)
@@ -137,8 +132,8 @@ def run_model_bpa_bulk(
                             rules=rules,
                             extended=extended,
                         )
-                        bpa_df["Capacity Id"] = capacity_id
                         bpa_df["Capacity Name"] = capacity_name
+                        bpa_df["Capacity Id"] = capacity_id
                         bpa_df["Workspace Name"] = wksp
                         bpa_df["Workspace Id"] = wksp_id
                         bpa_df["Dataset Name"] = dataset_name
@@ -146,7 +141,7 @@ def run_model_bpa_bulk(
                         bpa_df["Configured By"] = config_by
                         bpa_df["Timestamp"] = now
                         bpa_df["RunId"] = runId
-                        bpa_df = bpa_df[cols]
+                        bpa_df = bpa_df[list(icons.bpa_schema.keys())]
 
                         bpa_df["RunId"] = bpa_df["RunId"].astype("int")
 
@@ -160,21 +155,33 @@ def run_model_bpa_bulk(
                         )
                         print(e)
 
-                df["Severity"].replace(icons.severity_mapping)
+                if len(df) == 0:
+                    print(
+                        f"{icons.yellow_dot} No BPA results to save for the '{wksp}' workspace."
+                    )
+                else:
+                    df["Severity"].replace(icons.severity_mapping)
 
-                # Append save results individually for each workspace (so as not to create a giant dataframe)
-                print(
-                    f"{icons.in_progress} Saving the Model BPA results of the '{wksp}' workspace to the '{output_table}' within the '{lakehouse}' lakehouse within the '{lakehouse_workspace}' workspace..."
-                )
-                save_as_delta_table(
-                    dataframe=df,
-                    delta_table_name=output_table,
-                    write_mode="append",
-                    merge_schema=True,
-                )
-                print(
-                    f"{icons.green_dot} Saved BPA results to the '{output_table}' delta table."
-                )
+                    # Append save results individually for each workspace (so as not to create a giant dataframe)
+                    print(
+                        f"{icons.in_progress} Saving the Model BPA results of the '{wksp}' workspace to the '{output_table}' within the '{lakehouse}' lakehouse within the '{lakehouse_workspace}' workspace..."
+                    )
+
+                    schema = {
+                        key.replace(" ", "_"): value
+                        for key, value in icons.bpa_schema.items()
+                    }
+
+                    save_as_delta_table(
+                        dataframe=df,
+                        delta_table_name=output_table,
+                        write_mode="append",
+                        schema=schema,
+                        merge_schema=True,
+                    )
+                    print(
+                        f"{icons.green_dot} Saved BPA results to the '{output_table}' delta table."
+                    )
 
     print(f"{icons.green_dot} Bulk BPA scan complete.")
 
@@ -203,9 +210,6 @@ def create_model_bpa_semantic_model(
         The workspace in which the lakehouse resides.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
-
-    Returns
-    -------
     """
 
     from sempy_labs._helper_functions import resolve_lakehouse_name
@@ -244,7 +248,7 @@ def create_model_bpa_semantic_model(
             tom.model
 
     dyn_connect()
-
+    icons.sll_tags.append("ModelBPABulk")
     table_exists = False
     with connect_semantic_model(
         dataset=dataset, readonly=False, workspace=lakehouse_workspace