semantic-link-labs 0.8.10__py3-none-any.whl → 0.9.0__py3-none-any.whl

sempy_labs/_model_bpa.py

@@ -12,7 +12,7 @@ from sempy_labs._helper_functions import (
     resolve_workspace_capacity,
     resolve_dataset_name_and_id,
     get_language_codes,
-    _get_max_run_id,
+    _get_column_aggregate,
     resolve_workspace_name_and_id,
 )
 from sempy_labs.lakehouse import get_lakehouse_tables, lakehouse_attached
@@ -36,6 +36,7 @@ def run_model_bpa(
     return_dataframe: bool = False,
     extended: bool = False,
     language: Optional[str] = None,
+    check_dependencies: bool = True,
     **kwargs,
 ):
     """
@@ -43,12 +44,12 @@ def run_model_bpa(
 
     Parameters
     ----------
-    dataset : str | UUID
+    dataset : str | uuid.UUID
         Name or ID of the semantic model.
     rules : pandas.DataFrame, default=None
         A pandas dataframe containing rules to be evaluated.
-    workspace : str, default=None
-        The Fabric workspace name.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
     export : bool, default=False
@@ -60,6 +61,8 @@ def run_model_bpa(
     language : str, default=None
         Specifying a language name or code (i.e. 'it-IT' for Italian) will auto-translate the Category, Rule Name and Description into the specified language.
         Defaults to None which resolves to English.
+    check_dependencies : bool, default=True
+        If True, leverages the model dependencies from get_model_calc_dependencies to evaluate the rules. Set this parameter to False if running the rules against a semantic model in a shared capacity.
 
     Returns
     -------
@@ -129,236 +132,236 @@ def run_model_bpa(
             print(
                 f"{icons.warning} The '{dataset_name}' semantic model within the '{workspace_name}' workspace has no tables and therefore there are no valid BPA results."
             )
-            finalDF = pd.DataFrame(
+            return
+
+        if check_dependencies:
+            dep = get_model_calc_dependencies(
+                dataset=dataset_id, workspace=workspace_id
+            )
+        else:
+            dep = pd.DataFrame(
                 columns=[
-                    "Category",
-                    "Rule Name",
-                    "Severity",
-                    "Object Type",
+                    "Table Name",
                     "Object Name",
-                    "Description",
-                    "URL",
+                    "Object Type",
+                    "Expression",
+                    "Referenced Table",
+                    "Referenced Object",
+                    "Referenced Object Type",
+                    "Full Object Name",
+                    "Referenced Full Object Name",
+                    "Parent Node",
                 ]
             )
-        else:
-            dep = get_model_calc_dependencies(
-                dataset=dataset_id, workspace=workspace_id
+
+        def translate_using_po(rule_file):
+            current_dir = os.path.dirname(os.path.abspath(__file__))
+            translation_file = (
+                f"{current_dir}/_bpa_translation/_model/_translations_{language}.po"
             )
+            for c in ["Category", "Description", "Rule Name"]:
+                po = polib.pofile(translation_file)
+                for entry in po:
+                    if entry.tcomment == c.lower().replace(" ", "_"):
+                        rule_file.loc[rule_file["Rule Name"] == entry.msgid, c] = (
+                            entry.msgstr
+                        )
 
-            def translate_using_po(rule_file):
-                current_dir = os.path.dirname(os.path.abspath(__file__))
-                translation_file = (
-                    f"{current_dir}/_bpa_translation/_model/_translations_{language}.po"
-                )
-                for c in ["Category", "Description", "Rule Name"]:
-                    po = polib.pofile(translation_file)
-                    for entry in po:
-                        if entry.tcomment == c.lower().replace(" ", "_"):
-                            rule_file.loc[rule_file["Rule Name"] == entry.msgid, c] = (
-                                entry.msgstr
-                            )
+        translated = False
 
-            translated = False
+        # Translations
+        if language is not None and rules is None and language in language_list:
+            rules = model_bpa_rules(dependencies=dep)
+            translate_using_po(rules)
+            translated = True
+        if rules is None:
+            rules = model_bpa_rules(dependencies=dep)
+        if language is not None and not translated:
 
-            # Translations
-            if language is not None and rules is None and language in language_list:
-                rules = model_bpa_rules(dependencies=dep)
-                translate_using_po(rules)
-                translated = True
-            if rules is None:
-                rules = model_bpa_rules(dependencies=dep)
-            if language is not None and not translated:
+            def translate_using_spark(rule_file):
 
-                def translate_using_spark(rule_file):
+                from synapse.ml.services import Translate
+                from pyspark.sql import SparkSession
 
-                    from synapse.ml.services import Translate
-                    from pyspark.sql import SparkSession
+                rules_temp = rule_file.copy()
+                rules_temp = rules_temp.drop(["Expression", "URL", "Severity"], axis=1)
 
-                    rules_temp = rule_file.copy()
-                    rules_temp = rules_temp.drop(
-                        ["Expression", "URL", "Severity"], axis=1
-                    )
+                schema = StructType(
+                    [
+                        StructField("Category", StringType(), True),
+                        StructField("Scope", StringType(), True),
+                        StructField("Rule Name", StringType(), True),
+                        StructField("Description", StringType(), True),
+                    ]
+                )
 
-                    schema = StructType(
-                        [
-                            StructField("Category", StringType(), True),
-                            StructField("Scope", StringType(), True),
-                            StructField("Rule Name", StringType(), True),
-                            StructField("Description", StringType(), True),
-                        ]
+                spark = SparkSession.builder.getOrCreate()
+                dfRules = spark.createDataFrame(rules_temp, schema)
+
+                columns = ["Category", "Rule Name", "Description"]
+                for clm in columns:
+                    translate = (
+                        Translate()
+                        .setTextCol(clm)
+                        .setToLanguage(language)
+                        .setOutputCol("translation")
+                        .setConcurrency(5)
                     )
 
-                    spark = SparkSession.builder.getOrCreate()
-                    dfRules = spark.createDataFrame(rules_temp, schema)
-
-                    columns = ["Category", "Rule Name", "Description"]
-                    for clm in columns:
-                        translate = (
-                            Translate()
-                            .setTextCol(clm)
-                            .setToLanguage(language)
-                            .setOutputCol("translation")
-                            .setConcurrency(5)
-                        )
-
-                        if clm == "Rule Name":
-                            transDF = (
-                                translate.transform(dfRules)
-                                .withColumn(
-                                    "translation",
-                                    flatten(col("translation.translations")),
-                                )
-                                .withColumn("translation", col("translation.text"))
-                                .select(clm, "translation")
+                    if clm == "Rule Name":
+                        transDF = (
+                            translate.transform(dfRules)
+                            .withColumn(
+                                "translation",
+                                flatten(col("translation.translations")),
                             )
-                        else:
-                            transDF = (
-                                translate.transform(dfRules)
-                                .withColumn(
-                                    "translation",
-                                    flatten(col("translation.translations")),
-                                )
-                                .withColumn("translation", col("translation.text"))
-                                .select("Rule Name", clm, "translation")
+                            .withColumn("translation", col("translation.text"))
+                            .select(clm, "translation")
+                        )
+                    else:
+                        transDF = (
+                            translate.transform(dfRules)
+                            .withColumn(
+                                "translation",
+                                flatten(col("translation.translations")),
                             )
-
-                        df_panda = transDF.toPandas()
-                        rule_file = pd.merge(
-                            rule_file,
-                            df_panda[["Rule Name", "translation"]],
-                            on="Rule Name",
-                            how="left",
+                            .withColumn("translation", col("translation.text"))
+                            .select("Rule Name", clm, "translation")
                         )
 
-                        rule_file = rule_file.rename(
-                            columns={"translation": f"{clm}Translated"}
-                        )
-                        rule_file[f"{clm}Translated"] = rule_file[
-                            f"{clm}Translated"
-                        ].apply(lambda x: x[0] if x is not None else None)
+                    df_panda = transDF.toPandas()
+                    rule_file = pd.merge(
+                        rule_file,
+                        df_panda[["Rule Name", "translation"]],
+                        on="Rule Name",
+                        how="left",
+                    )
+
+                    rule_file = rule_file.rename(
+                        columns={"translation": f"{clm}Translated"}
+                    )
+                    rule_file[f"{clm}Translated"] = rule_file[f"{clm}Translated"].apply(
+                        lambda x: x[0] if x is not None else None
+                    )
 
-                    for clm in columns:
-                        rule_file = rule_file.drop([clm], axis=1)
-                        rule_file = rule_file.rename(columns={f"{clm}Translated": clm})
+                for clm in columns:
+                    rule_file = rule_file.drop([clm], axis=1)
+                    rule_file = rule_file.rename(columns={f"{clm}Translated": clm})
 
-                    return rule_file
+                return rule_file
 
-                rules = translate_using_spark(rules)
+            rules = translate_using_spark(rules)
 
-            rules.loc[rules["Severity"] == "Warning", "Severity"] = icons.warning
-            rules.loc[rules["Severity"] == "Error", "Severity"] = icons.error
-            rules.loc[rules["Severity"] == "Info", "Severity"] = icons.info
+        rules.loc[rules["Severity"] == "Warning", "Severity"] = icons.warning
+        rules.loc[rules["Severity"] == "Error", "Severity"] = icons.error
+        rules.loc[rules["Severity"] == "Info", "Severity"] = icons.info
 
-            pd.set_option("display.max_colwidth", 1000)
+        pd.set_option("display.max_colwidth", 1000)
 
-            violations = pd.DataFrame(columns=["Object Name", "Scope", "Rule Name"])
+        violations = pd.DataFrame(columns=["Object Name", "Scope", "Rule Name"])
 
-            scope_to_dataframe = {
-                "Relationship": (
-                    tom.model.Relationships,
-                    lambda obj: create_relationship_name(
-                        obj.FromTable.Name,
-                        obj.FromColumn.Name,
-                        obj.ToTable.Name,
-                        obj.ToColumn.Name,
-                    ),
-                ),
-                "Column": (
-                    tom.all_columns(),
-                    lambda obj: format_dax_object_name(obj.Parent.Name, obj.Name),
-                ),
-                "Measure": (tom.all_measures(), lambda obj: obj.Name),
-                "Hierarchy": (
-                    tom.all_hierarchies(),
-                    lambda obj: format_dax_object_name(obj.Parent.Name, obj.Name),
-                ),
-                "Table": (tom.model.Tables, lambda obj: obj.Name),
-                "Role": (tom.model.Roles, lambda obj: obj.Name),
-                "Model": (tom.model, lambda obj: obj.Model.Name),
-                "Calculation Item": (
-                    tom.all_calculation_items(),
-                    lambda obj: format_dax_object_name(obj.Parent.Table.Name, obj.Name),
-                ),
-                "Row Level Security": (
-                    tom.all_rls(),
-                    lambda obj: format_dax_object_name(obj.Parent.Name, obj.Name),
-                ),
-                "Partition": (
-                    tom.all_partitions(),
-                    lambda obj: format_dax_object_name(obj.Parent.Name, obj.Name),
+        scope_to_dataframe = {
+            "Relationship": (
+                tom.model.Relationships,
+                lambda obj: create_relationship_name(
+                    obj.FromTable.Name,
+                    obj.FromColumn.Name,
+                    obj.ToTable.Name,
+                    obj.ToColumn.Name,
                 ),
-            }
-
-            for i, r in rules.iterrows():
-                ruleName = r["Rule Name"]
-                expr = r["Expression"]
-                scopes = r["Scope"]
-
-                if isinstance(scopes, str):
-                    scopes = [scopes]
-
-                for scope in scopes:
-                    func = scope_to_dataframe[scope][0]
-                    nm = scope_to_dataframe[scope][1]
-
-                    if scope == "Model":
-                        x = []
-                        if expr(func, tom):
-                            x = ["Model"]
-                    elif scope == "Measure":
-                        x = [nm(obj) for obj in tom.all_measures() if expr(obj, tom)]
-                    elif scope == "Column":
-                        x = [nm(obj) for obj in tom.all_columns() if expr(obj, tom)]
-                    elif scope == "Partition":
-                        x = [nm(obj) for obj in tom.all_partitions() if expr(obj, tom)]
-                    elif scope == "Hierarchy":
-                        x = [nm(obj) for obj in tom.all_hierarchies() if expr(obj, tom)]
-                    elif scope == "Table":
-                        x = [nm(obj) for obj in tom.model.Tables if expr(obj, tom)]
-                    elif scope == "Relationship":
-                        x = [
-                            nm(obj) for obj in tom.model.Relationships if expr(obj, tom)
-                        ]
-                    elif scope == "Role":
-                        x = [nm(obj) for obj in tom.model.Roles if expr(obj, tom)]
-                    elif scope == "Row Level Security":
-                        x = [nm(obj) for obj in tom.all_rls() if expr(obj, tom)]
-                    elif scope == "Calculation Item":
-                        x = [
-                            nm(obj)
-                            for obj in tom.all_calculation_items()
-                            if expr(obj, tom)
-                        ]
-
-                    if len(x) > 0:
-                        new_data = {
-                            "Object Name": x,
-                            "Scope": scope,
-                            "Rule Name": ruleName,
-                        }
-                        violations = pd.concat(
-                            [violations, pd.DataFrame(new_data)], ignore_index=True
-                        )
+            ),
+            "Column": (
+                tom.all_columns(),
+                lambda obj: format_dax_object_name(obj.Parent.Name, obj.Name),
+            ),
+            "Measure": (tom.all_measures(), lambda obj: obj.Name),
+            "Hierarchy": (
+                tom.all_hierarchies(),
+                lambda obj: format_dax_object_name(obj.Parent.Name, obj.Name),
+            ),
+            "Table": (tom.model.Tables, lambda obj: obj.Name),
+            "Role": (tom.model.Roles, lambda obj: obj.Name),
+            "Model": (tom.model, lambda obj: obj.Model.Name),
+            "Calculation Item": (
+                tom.all_calculation_items(),
+                lambda obj: format_dax_object_name(obj.Parent.Table.Name, obj.Name),
+            ),
+            "Row Level Security": (
+                tom.all_rls(),
+                lambda obj: format_dax_object_name(obj.Parent.Name, obj.Name),
+            ),
+            "Partition": (
+                tom.all_partitions(),
+                lambda obj: format_dax_object_name(obj.Parent.Name, obj.Name),
+            ),
+        }
 
-            prepDF = pd.merge(
-                violations,
-                rules[["Rule Name", "Category", "Severity", "Description", "URL"]],
-                left_on="Rule Name",
-                right_on="Rule Name",
-                how="left",
-            )
-            prepDF.rename(columns={"Scope": "Object Type"}, inplace=True)
-            finalDF = prepDF[
-                [
-                    "Category",
-                    "Rule Name",
-                    "Severity",
-                    "Object Type",
-                    "Object Name",
-                    "Description",
-                    "URL",
-                ]
+        for i, r in rules.iterrows():
+            ruleName = r["Rule Name"]
+            expr = r["Expression"]
+            scopes = r["Scope"]
+
+            if isinstance(scopes, str):
+                scopes = [scopes]
+
+            for scope in scopes:
+                func = scope_to_dataframe[scope][0]
+                nm = scope_to_dataframe[scope][1]
+
+                if scope == "Model":
+                    x = []
+                    if expr(func, tom):
+                        x = ["Model"]
+                elif scope == "Measure":
+                    x = [nm(obj) for obj in tom.all_measures() if expr(obj, tom)]
+                elif scope == "Column":
+                    x = [nm(obj) for obj in tom.all_columns() if expr(obj, tom)]
+                elif scope == "Partition":
+                    x = [nm(obj) for obj in tom.all_partitions() if expr(obj, tom)]
+                elif scope == "Hierarchy":
+                    x = [nm(obj) for obj in tom.all_hierarchies() if expr(obj, tom)]
+                elif scope == "Table":
+                    x = [nm(obj) for obj in tom.model.Tables if expr(obj, tom)]
+                elif scope == "Relationship":
+                    x = [nm(obj) for obj in tom.model.Relationships if expr(obj, tom)]
+                elif scope == "Role":
+                    x = [nm(obj) for obj in tom.model.Roles if expr(obj, tom)]
+                elif scope == "Row Level Security":
+                    x = [nm(obj) for obj in tom.all_rls() if expr(obj, tom)]
+                elif scope == "Calculation Item":
+                    x = [
+                        nm(obj) for obj in tom.all_calculation_items() if expr(obj, tom)
+                    ]
+
+                if len(x) > 0:
+                    new_data = {
+                        "Object Name": x,
+                        "Scope": scope,
+                        "Rule Name": ruleName,
+                    }
+                    violations = pd.concat(
+                        [violations, pd.DataFrame(new_data)], ignore_index=True
+                    )
+
+        prepDF = pd.merge(
+            violations,
+            rules[["Rule Name", "Category", "Severity", "Description", "URL"]],
+            left_on="Rule Name",
+            right_on="Rule Name",
+            how="left",
+        )
+        prepDF.rename(columns={"Scope": "Object Type"}, inplace=True)
+        finalDF = prepDF[
+            [
+                "Category",
+                "Rule Name",
+                "Severity",
+                "Object Type",
+                "Object Name",
+                "Description",
+                "URL",
             ]
+        ]
 
     if export:
         if not lakehouse_attached():
@@ -383,7 +386,7 @@ def run_model_bpa(
         if len(lakeT_filt) == 0:
             runId = 1
         else:
-            max_run_id = _get_max_run_id(
+            max_run_id = _get_column_aggregate(
                 lakehouse=lakehouse, table_name=delta_table_name
             )
             runId = max_run_id + 1

sempy_labs/_model_bpa_bulk.py

@@ -6,7 +6,7 @@ from sempy_labs._helper_functions import (
     save_as_delta_table,
     resolve_workspace_capacity,
     retry,
-    _get_max_run_id,
+    _get_column_aggregate,
 )
 from sempy_labs.lakehouse import (
     get_lakehouse_tables,
@@ -76,7 +76,7 @@ def run_model_bpa_bulk(
     if len(lakeT_filt) == 0:
         runId = 1
     else:
-        max_run_id = _get_max_run_id(lakehouse=lakehouse, table_name=output_table)
+        max_run_id = _get_column_aggregate(lakehouse=lakehouse, table_name=output_table)
         runId = max_run_id + 1
 
     if isinstance(workspace, str):

sempy_labs/_model_bpa_rules.py

@@ -64,7 +64,7 @@ def model_bpa_rules(
                     obj.FromCardinality == TOM.RelationshipEndCardinality.Many
                     and obj.ToCardinality == TOM.RelationshipEndCardinality.Many
                 )
-                or str(obj.CrossFilteringBehavior) == "BothDirections"
+                or str(obj.CrossFilteringBehavior) == "BothDirections",
                 "Bi-directional and many-to-many relationships may cause performance degradation or even have unintended consequences. Make sure to check these specific relationships to ensure they are working as designed and are actually necessary.",
                 "https://www.sqlbi.com/articles/bidirectional-relationships-and-ambiguity-in-dax",
             ),
@@ -402,8 +402,8 @@ def model_bpa_rules(
                 lambda obj, tom: tom.is_direct_lake() is False
                 and obj.IsAvailableInMDX is False
                 and (
-                    tom.used_in_sort_by(column=obj)
-                    or tom.used_in_hierarchies(column=obj)
+                    any(tom.used_in_sort_by(column=obj))
+                    or any(tom.used_in_hierarchies(column=obj))
                     or obj.SortByColumn is not None
                 ),
                 "In order to avoid errors, ensure that attribute hierarchies are enabled if a column is used for sorting another column, used in a hierarchy, used in variations, or is sorted by another column. The IsAvailableInMdx property is not relevant for Direct Lake models.",