semantic-link-labs 0.7.2__py3-none-any.whl → 0.7.3__py3-none-any.whl

sempy_labs/_list_functions.py

@@ -1,7 +1,6 @@
 import sempy.fabric as fabric
 from sempy_labs._helper_functions import (
     resolve_workspace_name_and_id,
-    resolve_lakehouse_name,
     create_relationship_name,
     resolve_lakehouse_id,
     resolve_dataset_id,
@@ -86,7 +85,9 @@ def get_object_level_security(
         return df
 
 
-def list_tables(dataset: str, workspace: Optional[str] = None) -> pd.DataFrame:
+def list_tables(
+    dataset: str, workspace: Optional[str] = None, extended: Optional[bool] = False
+) -> pd.DataFrame:
     """
     Shows a semantic model's tables and their properties.
 
@@ -98,6 +99,8 @@ def list_tables(dataset: str, workspace: Optional[str] = None) -> pd.DataFrame:
         The Fabric workspace name.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
+    extended : bool, default=False
+        Adds additional columns including Vertipaq statistics.
 
     Returns
     -------
@@ -105,18 +108,136 @@ def list_tables(dataset: str, workspace: Optional[str] = None) -> pd.DataFrame:
         A pandas dataframe showing the semantic model's tables and their properties.
     """
 
+    from sempy_labs.tom import connect_semantic_model
+
     workspace = fabric.resolve_workspace_name(workspace)
 
-    df = fabric.list_tables(
-        dataset=dataset,
-        workspace=workspace,
-        additional_xmla_properties=["RefreshPolicy", "RefreshPolicy.SourceExpression"],
+    df = pd.DataFrame(
+        columns=[
+            "Name",
+            "Description",
+            "Hidden",
+            "Data Category",
+            "Type",
+            "Refresh Policy",
+            "Source Expression",
+        ]
     )
 
-    df["Refresh Policy"] = df["Refresh Policy"].notna()
-    df.rename(
-        columns={"Refresh Policy Source Expression": "Source Expression"}, inplace=True
-    )
+    with connect_semantic_model(
+        dataset=dataset, workspace=workspace, readonly=True
+    ) as tom:
+        if extended:
+            dict_df = fabric.evaluate_dax(
+                dataset=dataset,
+                workspace=workspace,
+                dax_string="""
+                EVALUATE SELECTCOLUMNS(FILTER(INFO.STORAGETABLECOLUMNS(), [COLUMN_TYPE] = "BASIC_DATA"),[DIMENSION_NAME],[DICTIONARY_SIZE])
+                """,
+            )
+            dict_sum = dict_df.groupby("[DIMENSION_NAME]")["[DICTIONARY_SIZE]"].sum()
+            data = fabric.evaluate_dax(
+                dataset=dataset,
+                workspace=workspace,
+                dax_string="""EVALUATE SELECTCOLUMNS(INFO.STORAGETABLECOLUMNSEGMENTS(),[TABLE_ID],[DIMENSION_NAME],[USED_SIZE])""",
+            )
+            data_sum = (
+                data[
+                    ~data["[TABLE_ID]"].str.startswith("R$")
+                    & ~data["[TABLE_ID]"].str.startswith("U$")
+                    & ~data["[TABLE_ID]"].str.startswith("H$")
+                ]
+                .groupby("[DIMENSION_NAME]")["[USED_SIZE]"]
+                .sum()
+            )
+            hier_sum = (
+                data[data["[TABLE_ID]"].str.startswith("H$")]
+                .groupby("[DIMENSION_NAME]")["[USED_SIZE]"]
+                .sum()
+            )
+            rel_sum = (
+                data[data["[TABLE_ID]"].str.startswith("R$")]
+                .groupby("[DIMENSION_NAME]")["[USED_SIZE]"]
+                .sum()
+            )
+            uh_sum = (
+                data[data["[TABLE_ID]"].str.startswith("U$")]
+                .groupby("[DIMENSION_NAME]")["[USED_SIZE]"]
+                .sum()
+            )
+            rc = fabric.evaluate_dax(
+                dataset=dataset,
+                workspace=workspace,
+                dax_string="""
+                SELECT [DIMENSION_NAME],[DIMENSION_CARDINALITY] FROM $SYSTEM.MDSCHEMA_DIMENSIONS
+            """,
+            )
+
+        rows = []
+        for t in tom.model.Tables:
+            t_name = t.Name
+            t_type = (
+                "Calculation Group"
+                if t.CalculationGroup
+                else (
+                    "Calculated Table"
+                    if tom.is_calculated_table(table_name=t.Name)
+                    else "Table"
+                )
+            )
+            ref = bool(t.RefreshPolicy)
+            ref_se = t.RefreshPolicy.SourceExpression if ref else None
+
+            new_data = {
+                "Name": t_name,
+                "Description": t.Description,
+                "Hidden": t.IsHidden,
+                "Data Category": t.DataCategory,
+                "Type": t_type,
+                "Refresh Policy": ref,
+                "Source Expression": ref_se,
+            }
+
+            if extended:
+                dict_size = dict_sum.get(t_name, 0)
+                data_size = data_sum.get(t_name, 0)
+                h_size = hier_sum.get(t_name, 0)
+                r_size = rel_sum.get(t_name, 0)
+                u_size = uh_sum.get(t_name, 0)
+                total_size = data_size + dict_size + h_size + r_size + u_size
+
+                new_data.update(
+                    {
+                        "Row Count": (
+                            rc[rc["DIMENSION_NAME"] == t_name][
+                                "DIMENSION_CARDINALITY"
+                            ].iloc[0]
+                            if not rc.empty
+                            else 0
+                        ),
+                        "Total Size": total_size,
+                        "Dictionary Size": dict_size,
+                        "Data Size": data_size,
+                        "Hierarchy Size": h_size,
+                        "Relationship Size": r_size,
+                        "User Hierarchy Size": u_size,
+                    }
+                )
+
+            rows.append(new_data)
+
+        int_cols = [
+            "Row Count",
+            "Total Size",
+            "Dictionary Size",
+            "Data Size",
+            "Hierarchy Size",
+            "Relationship Size",
+            "User Hierarchy Size",
+        ]
+        df[int_cols] = df[int_cols].astype(int)
+
+        df = pd.DataFrame(rows)
 
     return df
 
@@ -880,14 +1001,10 @@ def list_eventstreams(workspace: Optional[str] = None) -> pd.DataFrame:
 
     for r in responses:
         for v in r.get("value", []):
-            model_id = v.get("id")
-            modelName = v.get("displayName")
-            desc = v.get("description")
-
             new_data = {
-                "Eventstream Name": modelName,
-                "Eventstream ID": model_id,
-                "Description": desc,
+                "Eventstream Name": v.get("displayName"),
+                "Eventstream ID": v.get("id"),
+                "Description": v.get("description"),
             }
             df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
 
@@ -1030,10 +1147,6 @@ def create_warehouse(
         The Fabric workspace name.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
-
-    Returns
-    -------
-
     """
 
     (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
@@ -1045,11 +1158,11 @@ def create_warehouse(
 
     client = fabric.FabricRestClient()
     response = client.post(
-        f"/v1/workspaces/{workspace_id}/warehouses/", json=request_body, lro_wait=True
+        f"/v1/workspaces/{workspace_id}/warehouses/", json=request_body
     )
 
-    if response.status_code != 200:
-        raise FabricHTTPException(response)
+    lro(client, response, status_codes=[201, 202])
+
     print(
         f"{icons.green_dot} The '{warehouse}' warehouse has been created within the '{workspace}' workspace."
     )
@@ -1216,44 +1329,6 @@ def list_relationships(
     return dfR
 
 
-def list_dataflow_storage_accounts() -> pd.DataFrame:
-    """
-    Shows the accessible dataflow storage accounts.
-
-    Parameters
-    ----------
-
-    Returns
-    -------
-    pandas.DataFrame
-        A pandas dataframe showing the accessible dataflow storage accounts.
-    """
-
-    df = pd.DataFrame(
-        columns=[
-            "Dataflow Storage Account ID",
-            "Dataflow Storage Account Name",
-            "Enabled",
-        ]
-    )
-    client = fabric.PowerBIRestClient()
-    response = client.get("/v1.0/myorg/dataflowStorageAccounts")
-    if response.status_code != 200:
-        raise FabricHTTPException(response)
-
-    for v in response.json().get("value", []):
-        new_data = {
-            "Dataflow Storage Account ID": v.get("id"),
-            "Dataflow Storage Account Name": v.get("name"),
-            "Enabled": v.get("isEnabled"),
-        }
-        df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
-
-    df["Enabled"] = df["Enabled"].astype(bool)
-
-    return df
-
-
 def list_kpis(dataset: str, workspace: Optional[str] = None) -> pd.DataFrame:
     """
     Shows a semantic model's KPIs and their properties.
@@ -1749,9 +1824,6 @@ def create_custom_pool(
         The name of the Fabric workspace.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
-
-    Returns
-    -------
     """
 
     # https://learn.microsoft.com/en-us/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool
@@ -1775,10 +1847,10 @@ def create_custom_pool(
 
     client = fabric.FabricRestClient()
     response = client.post(
-        f"/v1/workspaces/{workspace_id}/spark/pools", json=request_body, lro_wait=True
+        f"/v1/workspaces/{workspace_id}/spark/pools", json=request_body
     )
 
-    if response.status_code != 200:
+    if response.status_code != 201:
         raise FabricHTTPException(response)
     print(
         f"{icons.green_dot} The '{pool_name}' spark pool has been created within the '{workspace}' workspace."
@@ -1832,9 +1904,6 @@ def update_custom_pool(
         The name of the Fabric workspace.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
-
-    Returns
-    -------
     """
 
     # https://learn.microsoft.com/en-us/rest/api/fabric/spark/custom-pools/update-workspace-custom-pool?tabs=HTTP
@@ -1907,9 +1976,6 @@ def delete_custom_pool(pool_name: str, workspace: Optional[str] = None):
         The name of the Fabric workspace.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
-
-    Returns
-    -------
     """
 
     (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
@@ -1945,15 +2011,16 @@ def assign_workspace_to_capacity(capacity_name: str, workspace: Optional[str] =
         The name of the Fabric workspace.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
-
-    Returns
-    -------
     """
 
     (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
 
     dfC = fabric.list_capacities()
     dfC_filt = dfC[dfC["Display Name"] == capacity_name]
+
+    if len(dfC_filt) == 0:
+        raise ValueError(f"{icons.red_dot} The '{capacity_name}' capacity does not exist.")
+
     capacity_id = dfC_filt["Id"].iloc[0]
 
     request_body = {"capacityId": capacity_id}
@@ -1962,7 +2029,6 @@ def assign_workspace_to_capacity(capacity_name: str, workspace: Optional[str] =
     response = client.post(
         f"/v1/workspaces/{workspace_id}/assignToCapacity",
         json=request_body,
-        lro_wait=True,
     )
 
     if response.status_code not in [200, 202]:
@@ -1982,9 +2048,6 @@ def unassign_workspace_from_capacity(workspace: Optional[str] = None):
         The name of the Fabric workspace.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
-
-    Returns
-    -------
     """
 
     # https://learn.microsoft.com/en-us/rest/api/fabric/core/workspaces/unassign-from-capacity?tabs=HTTP
@@ -1992,7 +2055,7 @@ def unassign_workspace_from_capacity(workspace: Optional[str] = None):
 
     client = fabric.FabricRestClient()
     response = client.post(
-        f"/v1/workspaces/{workspace_id}/unassignFromCapacity", lro_wait=True
+        f"/v1/workspaces/{workspace_id}/unassignFromCapacity"
     )
 
     if response.status_code not in [200, 202]:
@@ -2357,46 +2420,6 @@ def list_workspace_users(workspace: Optional[str] = None) -> pd.DataFrame:
     return df
 
 
-def assign_workspace_to_dataflow_storage(
-    dataflow_storage_account: str, workspace: Optional[str] = None
-):
-    """
-    Assigns a dataflow storage account to a workspace.
-
-    Parameters
-    ----------
-    dataflow_storage_account : str
-        The name of the dataflow storage account.
-    workspace : str, default=None
-        The name of the workspace.
-        Defaults to None which resolves to the workspace of the attached lakehouse
-        or if no lakehouse attached, resolves to the workspace of the notebook.
-
-    Returns
-    -------
-    """
-
-    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
-
-    df = list_dataflow_storage_accounts()
-    df_filt = df[df["Dataflow Storage Account Name"] == dataflow_storage_account]
-    dataflow_storage_id = df_filt["Dataflow Storage Account ID"].iloc[0]
-
-    client = fabric.PowerBIRestClient()
-
-    request_body = {"dataflowStorageId": dataflow_storage_id}
-
-    response = client.post(
-        f"/v1.0/myorg/groups/{workspace_id}/AssignToDataflowStorage", json=request_body
-    )
-
-    if response.status_code != 200:
-        raise FabricHTTPException(response)
-    print(
-        f"{icons.green_dot} The '{dataflow_storage_account}' dataflow storage account has been assigned to the '{workspace}' workspacce."
-    )
-
-
 def list_capacities() -> pd.DataFrame:
     """
     Shows the capacities and their properties.

sempy_labs/_model_bpa.py

@@ -21,7 +21,6 @@ from sempy._utils._log import log
 import sempy_labs._icons as icons
 from pyspark.sql.functions import col, flatten
 from pyspark.sql.types import StructType, StructField, StringType
-import polib
 import os
 
 
@@ -64,8 +63,8 @@ def run_model_bpa(
     pandas.DataFrame
         A pandas dataframe in HTML format showing semantic model objects which violated the best practice analyzer rules.
     """
-
-    from synapse.ml.services import Translate
+    
+    import polib
 
     if "extend" in kwargs:
         print(
@@ -108,6 +107,7 @@ def run_model_bpa(
         "zu-ZA",
         "am-ET",
         "ar-AE",
+        "sv-SE",
     ]
 
     # Map languages to the closest language (first 2 letters matching)
@@ -162,86 +162,86 @@ def run_model_bpa(
                             entry.msgstr
                         )
 
-        def translate_using_spark(rule_file):
-            rules_temp = rule_file.copy()
-            rules_temp = rules_temp.drop(["Expression", "URL", "Severity"], axis=1)
-
-            schema = StructType(
-                [
-                    StructField("Category", StringType(), True),
-                    StructField("Scope", StringType(), True),
-                    StructField("Rule Name", StringType(), True),
-                    StructField("Description", StringType(), True),
-                ]
-            )
+        translated = False
 
-            spark = SparkSession.builder.getOrCreate()
-            dfRules = spark.createDataFrame(rules_temp, schema)
-
-            columns = ["Category", "Rule Name", "Description"]
-            for clm in columns:
-                translate = (
-                    Translate()
-                    .setTextCol(clm)
-                    .setToLanguage(language)
-                    .setOutputCol("translation")
-                    .setConcurrency(5)
+        # Translations
+        if language is not None and rules is None and language in language_list:
+            rules = model_bpa_rules(dependencies=dep)
+            translate_using_po(rules)
+            translated = True
+        if rules is None:
+            rules = model_bpa_rules(dependencies=dep)
+        if language is not None and not translated:
+
+            def translate_using_spark(rule_file):
+
+                from synapse.ml.services import Translate
+
+                rules_temp = rule_file.copy()
+                rules_temp = rules_temp.drop(["Expression", "URL", "Severity"], axis=1)
+
+                schema = StructType(
+                    [
+                        StructField("Category", StringType(), True),
+                        StructField("Scope", StringType(), True),
+                        StructField("Rule Name", StringType(), True),
+                        StructField("Description", StringType(), True),
+                    ]
                 )
 
-                if clm == "Rule Name":
-                    transDF = (
-                        translate.transform(dfRules)
-                        .withColumn(
-                            "translation", flatten(col("translation.translations"))
-                        )
-                        .withColumn("translation", col("translation.text"))
-                        .select(clm, "translation")
-                    )
-                else:
-                    transDF = (
-                        translate.transform(dfRules)
-                        .withColumn(
-                            "translation", flatten(col("translation.translations"))
-                        )
-                        .withColumn("translation", col("translation.text"))
-                        .select("Rule Name", clm, "translation")
+                spark = SparkSession.builder.getOrCreate()
+                dfRules = spark.createDataFrame(rules_temp, schema)
+
+                columns = ["Category", "Rule Name", "Description"]
+                for clm in columns:
+                    translate = (
+                        Translate()
+                        .setTextCol(clm)
+                        .setToLanguage(language)
+                        .setOutputCol("translation")
+                        .setConcurrency(5)
                     )
 
-                df_panda = transDF.toPandas()
-                rule_file = pd.merge(
-                    rule_file,
-                    df_panda[["Rule Name", "translation"]],
-                    on="Rule Name",
-                    how="left",
-                )
+                    if clm == "Rule Name":
+                        transDF = (
+                            translate.transform(dfRules)
+                            .withColumn(
+                                "translation", flatten(col("translation.translations"))
+                            )
+                            .withColumn("translation", col("translation.text"))
+                            .select(clm, "translation")
+                        )
+                    else:
+                        transDF = (
+                            translate.transform(dfRules)
+                            .withColumn(
+                                "translation", flatten(col("translation.translations"))
+                            )
+                            .withColumn("translation", col("translation.text"))
+                            .select("Rule Name", clm, "translation")
+                        )
 
-                rule_file = rule_file.rename(
-                    columns={"translation": f"{clm}Translated"}
-                )
-                rule_file[f"{clm}Translated"] = rule_file[f"{clm}Translated"].apply(
-                    lambda x: x[0] if x is not None else None
-                )
+                    df_panda = transDF.toPandas()
+                    rule_file = pd.merge(
+                        rule_file,
+                        df_panda[["Rule Name", "translation"]],
+                        on="Rule Name",
+                        how="left",
+                    )
 
-            for clm in columns:
-                rule_file = rule_file.drop([clm], axis=1)
-                rule_file = rule_file.rename(columns={f"{clm}Translated": clm})
+                    rule_file = rule_file.rename(
+                        columns={"translation": f"{clm}Translated"}
+                    )
+                    rule_file[f"{clm}Translated"] = rule_file[f"{clm}Translated"].apply(
+                        lambda x: x[0] if x is not None else None
+                    )
 
-            return rule_file
+                for clm in columns:
+                    rule_file = rule_file.drop([clm], axis=1)
+                    rule_file = rule_file.rename(columns={f"{clm}Translated": clm})
 
-        translated = False
+                return rule_file
 
-        # Translations
-        if language is not None and rules is None and language in language_list:
-            rules = model_bpa_rules(
-                dataset=dataset, workspace=workspace, dependencies=dep
-            )
-            translate_using_po(rules)
-            translated = True
-        if rules is None:
-            rules = model_bpa_rules(
-                dataset=dataset, workspace=workspace, dependencies=dep
-            )
-        if language is not None and not translated:
             rules = translate_using_spark(rules)
 
         rules["Severity"].replace("Warning", icons.warning, inplace=True)
@@ -302,26 +302,28 @@ def run_model_bpa(
 
                 if scope == "Model":
                     x = []
-                    if expr(func):
+                    if expr(func, tom):
                         x = ["Model"]
                 elif scope == "Measure":
-                    x = [nm(obj) for obj in tom.all_measures() if expr(obj)]
+                    x = [nm(obj) for obj in tom.all_measures() if expr(obj, tom)]
                 elif scope == "Column":
-                    x = [nm(obj) for obj in tom.all_columns() if expr(obj)]
+                    x = [nm(obj) for obj in tom.all_columns() if expr(obj, tom)]
                 elif scope == "Partition":
-                    x = [nm(obj) for obj in tom.all_partitions() if expr(obj)]
+                    x = [nm(obj) for obj in tom.all_partitions() if expr(obj, tom)]
                 elif scope == "Hierarchy":
-                    x = [nm(obj) for obj in tom.all_hierarchies() if expr(obj)]
+                    x = [nm(obj) for obj in tom.all_hierarchies() if expr(obj, tom)]
                 elif scope == "Table":
-                    x = [nm(obj) for obj in tom.model.Tables if expr(obj)]
+                    x = [nm(obj) for obj in tom.model.Tables if expr(obj, tom)]
                 elif scope == "Relationship":
-                    x = [nm(obj) for obj in tom.model.Relationships if expr(obj)]
+                    x = [nm(obj) for obj in tom.model.Relationships if expr(obj, tom)]
                 elif scope == "Role":
-                    x = [nm(obj) for obj in tom.model.Roles if expr(obj)]
+                    x = [nm(obj) for obj in tom.model.Roles if expr(obj, tom)]
                 elif scope == "Row Level Security":
-                    x = [nm(obj) for obj in tom.all_rls() if expr(obj)]
+                    x = [nm(obj) for obj in tom.all_rls() if expr(obj, tom)]
                 elif scope == "Calculation Item":
-                    x = [nm(obj) for obj in tom.all_calculation_items() if expr(obj)]
+                    x = [
+                        nm(obj) for obj in tom.all_calculation_items() if expr(obj, tom)
+                    ]
 
                 if len(x) > 0:
                     new_data = {"Object Name": x, "Scope": scope, "Rule Name": ruleName}

sempy_labs/_model_bpa_bulk.py

@@ -229,7 +229,9 @@ def create_model_bpa_semantic_model(
     expr = get_shared_expression(lakehouse=lakehouse, workspace=lakehouse_workspace)
 
     # Create blank model
-    create_blank_semantic_model(dataset=dataset, workspace=lakehouse_workspace)
+    create_blank_semantic_model(
+        dataset=dataset, workspace=lakehouse_workspace, overwrite=True
+    )
 
     @retry(
         sleep_time=1,