semantic-link-labs 0.9.1__py3-none-any.whl → 0.9.3__py3-none-any.whl

sempy_labs/_list_functions.py

@@ -3,16 +3,21 @@ from sempy_labs._helper_functions import (
     resolve_workspace_name_and_id,
     create_relationship_name,
     resolve_lakehouse_id,
-    pagination,
     resolve_item_type,
     format_dax_object_name,
     resolve_dataset_name_and_id,
+    _update_dataframe_datatypes,
+    _base_api,
+    _create_dataframe,
+    _run_spark_sql_query,
 )
+from sempy._utils._log import log
 import pandas as pd
 from typing import Optional
 import sempy_labs._icons as icons
-from sempy.fabric.exceptions import FabricHTTPException
 from uuid import UUID
+import json
+from collections import defaultdict
 
 
 def get_object_level_security(
@@ -41,7 +46,13 @@ def get_object_level_security(
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
     (dataset_name, dataset_id) = resolve_dataset_name_and_id(dataset, workspace_id)
 
-    df = pd.DataFrame(columns=["Role Name", "Object Type", "Table Name", "Object Name"])
+    columns = {
+        "Role Name": "string",
+        "Object Type": "string",
+        "Table Name": "string",
+        "Object Name": "string",
+    }
+    df = _create_dataframe(columns=columns)
 
     with connect_semantic_model(
         dataset=dataset_id, readonly=True, workspace=workspace_id
@@ -111,17 +122,17 @@ def list_tables(
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
     (dataset_name, dataset_id) = resolve_dataset_name_and_id(dataset, workspace_id)
 
-    df = pd.DataFrame(
-        columns=[
-            "Name",
-            "Description",
-            "Hidden",
-            "Data Category",
-            "Type",
-            "Refresh Policy",
-            "Source Expression",
-        ]
-    )
+    columns = {
+        "Name": "string",
+        "Description": "string",
+        "Hidden": "bool",
+        "Data Category": "string",
+        "Type": "string",
+        "Refresh Policy": "bool",
+        "Source Expression": "string",
+    }
+
+    df = _create_dataframe(columns=columns)
 
     with connect_semantic_model(
         dataset=dataset_id, workspace=workspace_id, readonly=True
@@ -240,19 +251,20 @@ def list_tables(
         df = pd.DataFrame(rows)
 
         if extended:
-            int_cols = [
-                "Row Count",
-                "Total Size",
-                "Dictionary Size",
-                "Data Size",
-                "Hierarchy Size",
-                "Relationship Size",
-                "User Hierarchy Size",
-                "Partitions",
-                "Columns",
-            ]
-            df[int_cols] = df[int_cols].astype(int)
-            df["% DB"] = df["% DB"].astype(float)
+            column_map = {
+                "Row Count": "int",
+                "Total Size": "int",
+                "Dictionary Size": "int",
+                "Data Size": "int",
+                "Hierarchy Size": "int",
+                "Relationship Size": "int",
+                "User Hierarchy Size": "int",
+                "Partitions": "int",
+                "Columns": "int",
+                "% DB": "float",
+            }
+
+            _update_dataframe_datatypes(dataframe=df, column_map=column_map)
 
     return df
 
@@ -283,15 +295,14 @@ def list_annotations(
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
     (dataset_name, dataset_id) = resolve_dataset_name_and_id(dataset, workspace_id)
 
-    df = pd.DataFrame(
-        columns=[
-            "Object Name",
-            "Parent Object Name",
-            "Object Type",
-            "Annotation Name",
-            "Annotation Value",
-        ]
-    )
+    columns = {
+        "Object Name": "string",
+        "Parent Object Name": "string",
+        "Object Type": "string",
+        "Annotation Name": "string",
+        "Annotation Value": "string",
+    }
+    df = _create_dataframe(columns=columns)
 
     with connect_semantic_model(
         dataset=dataset_id, readonly=True, workspace=workspace_id
@@ -528,6 +539,8 @@ def list_columns(
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
     (dataset_name, dataset_id) = resolve_dataset_name_and_id(dataset, workspace_id)
 
+    fabric.refresh_tom_cache(workspace=workspace)
+
     dfP = fabric.list_partitions(dataset=dataset_id, workspace=workspace_id)
 
     isDirectLake = any(r["Mode"] == "DirectLake" for i, r in dfP.iterrows())
@@ -572,14 +585,12 @@ def list_columns(
             query = f"{query} FROM {lakehouse}.{lakeTName}"
             sql_statements.append((table_name, query))
 
-            spark = SparkSession.builder.getOrCreate()
-
         for o in sql_statements:
             tName = o[0]
             query = o[1]
 
             # Run the query
-            df = spark.sql(query)
+            df = _run_spark_sql_query(query)
 
             for column in df.columns:
                 x = df.collect()[0][column]
@@ -610,25 +621,21 @@ def list_dashboards(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
         A pandas dataframe showing the dashboards within a workspace.
     """
 
-    df = pd.DataFrame(
-        columns=[
-            "Dashboard ID",
-            "Dashboard Name",
-            "Read Only",
-            "Web URL",
-            "Embed URL",
-            "Data Classification",
-            "Users",
-            "Subscriptions",
-        ]
-    )
+    columns = {
+        "Dashboard ID": "string",
+        "Dashboard Name": "string",
+        "Read Only": "bool",
+        "Web URL": "string",
+        "Embed URL": "string",
+        "Data Classification": "string",
+        "Users": "string",
+        "Subscriptions": "string",
+    }
+    df = _create_dataframe(columns=columns)
 
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
 
-    client = fabric.PowerBIRestClient()
-    response = client.get(f"/v1.0/myorg/groups/{workspace_id}/dashboards")
-    if response.status_code != 200:
-        raise FabricHTTPException(response)
+    response = _base_api(request=f"/v1.0/myorg/groups/{workspace_id}/dashboards")
 
     for v in response.json().get("value", []):
         new_data = {
@@ -643,7 +650,7 @@ def list_dashboards(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
         }
         df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
 
-    df["Read Only"] = df["Read Only"].astype(bool)
+    _update_dataframe_datatypes(dataframe=df, column_map=columns)
 
     return df
 
@@ -665,28 +672,23 @@ def list_lakehouses(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
         A pandas dataframe showing the lakehouses within a workspace.
     """
 
-    df = pd.DataFrame(
-        columns=[
-            "Lakehouse Name",
-            "Lakehouse ID",
-            "Description",
-            "OneLake Tables Path",
-            "OneLake Files Path",
-            "SQL Endpoint Connection String",
-            "SQL Endpoint ID",
-            "SQL Endpoint Provisioning Status",
-        ]
-    )
+    columns = {
+        "Lakehouse Name": "string",
+        "Lakehouse ID": "string",
+        "Description": "string",
+        "OneLake Tables Path": "string",
+        "OneLake Files Path": "string",
+        "SQL Endpoint Connection String": "string",
+        "SQL Endpoint ID": "string",
+        "SQL Endpoint Provisioning Status": "string",
+    }
+    df = _create_dataframe(columns=columns)
 
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
 
-    client = fabric.FabricRestClient()
-    response = client.get(f"/v1/workspaces/{workspace_id}/lakehouses")
-
-    if response.status_code != 200:
-        raise FabricHTTPException(response)
-
-    responses = pagination(client, response)
+    responses = _base_api(
+        request=f"/v1/workspaces/{workspace_id}/lakehouses", uses_pagination=True
+    )
 
     for r in responses:
         for v in r.get("value", []):
@@ -725,16 +727,18 @@ def list_sql_endpoints(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
         A pandas dataframe showing the SQL endpoints within a workspace.
     """
 
-    df = pd.DataFrame(columns=["SQL Endpoint Id", "SQL Endpoint Name", "Description"])
+    columns = {
+        "SQL Endpoint Id": "string",
+        "SQL Endpoint Name": "string",
+        "Description": "string",
+    }
+    df = _create_dataframe(columns=columns)
 
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
 
-    client = fabric.FabricRestClient()
-    response = client.get(f"/v1/workspaces/{workspace_id}/sqlEndpoints")
-    if response.status_code != 200:
-        raise FabricHTTPException(response)
-
-    responses = pagination(client, response)
+    responses = _base_api(
+        request=f"/v1/workspaces/{workspace_id}/sqlEndpoints", uses_pagination=True
+    )
 
     for r in responses:
         for v in r.get("value", []):
@@ -766,19 +770,21 @@ def list_datamarts(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
         A pandas dataframe showing the datamarts within a workspace.
     """
 
-    df = pd.DataFrame(columns=["Datamart Name", "Datamart ID", "Description"])
+    columns = {
+        "Datamart Name": "string",
+        "Datamart ID": "string",
+        "Description": "string",
+    }
+    df = _create_dataframe(columns=columns)
 
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
 
-    client = fabric.FabricRestClient()
-    response = client.get(f"/v1/workspaces/{workspace_id}/datamarts")
-    if response.status_code != 200:
-        raise FabricHTTPException(response)
-
-    responses = pagination(client, response)
+    responses = _base_api(
+        request=f"/v1/workspaces/{workspace_id}/datamarts", uses_pagination=True
+    )
 
     for r in responses:
-        for v in response.get("value", []):
+        for v in r.get("value", []):
             new_data = {
                 "Datamart Name": v.get("displayName"),
                 "Datamart ID": v.get("id"),
@@ -835,17 +841,15 @@ def update_item(
 
     itemId = dfI_filt["Id"].iloc[0]
 
-    request_body = {"displayName": new_name}
+    payload = {"displayName": new_name}
     if description:
-        request_body["description"] = description
+        payload["description"] = description
 
-    client = fabric.FabricRestClient()
-    response = client.patch(
-        f"/v1/workspaces/{workspace_id}/{itemType}/{itemId}", json=request_body
+    _base_api(
+        request=f"/v1/workspaces/{workspace_id}/{itemType}/{itemId}",
+        payload=payload,
+        method="patch",
     )
-
-    if response.status_code != 200:
-        raise FabricHTTPException(response)
     if description is None:
         print(
             f"{icons.green_dot} The '{current_name}' {item_type} within the '{workspace_name}' workspace has been updated to be named '{new_name}'"
@@ -882,6 +886,8 @@ def list_relationships(
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
     (dataset_name, dataset_id) = resolve_dataset_name_and_id(dataset, workspace_id)
 
+    fabric.refresh_tom_cache(workspace=workspace)
+
     dfR = fabric.list_relationships(dataset=dataset_id, workspace=workspace_id)
     dfR["From Object"] = format_dax_object_name(dfR["From Table"], dfR["From Column"])
     dfR["To Object"] = format_dax_object_name(dfR["To Table"], dfR["To Column"])
@@ -935,7 +941,11 @@ def list_relationships(
             sumval = filtered_cs["USED_SIZE"].sum()
             dfR.at[i, "Used Size"] = sumval
 
-        dfR["Used Size"] = dfR["Used Size"].astype("int")
+        column_map = {
+            "Used Size": "int",
+        }
+
+        _update_dataframe_datatypes(dataframe=dfR, column_map=column_map)
 
     return dfR
 
@@ -966,26 +976,25 @@ def list_kpis(
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
     (dataset_name, dataset_id) = resolve_dataset_name_and_id(dataset, workspace_id)
 
+    columns = {
+        "Table Name": "string",
+        "Measure Name": "string",
+        "Target Expression": "string",
+        "Target Format String": "string",
+        "Target Description": "string",
+        "Status Expression": "string",
+        "Status Graphic": "string",
+        "Status Description": "string",
+        "Trend Expression": "string",
+        "Trend Graphic": "string",
+        "Trend Description": "string",
+    }
+    df = _create_dataframe(columns=columns)
+
     with connect_semantic_model(
         dataset=dataset_id, workspace=workspace_id, readonly=True
     ) as tom:
 
-        df = pd.DataFrame(
-            columns=[
-                "Table Name",
-                "Measure Name",
-                "Target Expression",
-                "Target Format String",
-                "Target Description",
-                "Status Expression",
-                "Status Graphic",
-                "Status Description",
-                "Trend Expression",
-                "Trend Graphic",
-                "Trend Description",
-            ]
-        )
-
         for t in tom.model.Tables:
             for m in t.Measures:
                 if m.KPI is not None:
@@ -1032,7 +1041,13 @@ def list_semantic_model_objects(
     """
     from sempy_labs.tom import connect_semantic_model
 
-    df = pd.DataFrame(columns=["Parent Name", "Object Name", "Object Type"])
+    columns = {
+        "Parent Name": "string",
+        "Object Name": "string",
+        "Object Type": "string",
+    }
+    df = _create_dataframe(columns=columns)
+
     with connect_semantic_model(
         dataset=dataset, workspace=workspace, readonly=True
     ) as tom:
@@ -1202,35 +1217,28 @@ def list_shortcuts(
     else:
         lakehouse_id = resolve_lakehouse_id(lakehouse, workspace_id)
 
-    client = fabric.FabricRestClient()
-
-    df = pd.DataFrame(
-        columns=[
-            "Shortcut Name",
-            "Shortcut Path",
-            "Source Type",
-            "Source Workspace Id",
-            "Source Workspace Name",
-            "Source Item Id",
-            "Source Item Name",
-            "Source Item Type",
-            "OneLake Path",
-            "Connection Id",
-            "Location",
-            "Bucket",
-            "SubPath",
-        ]
+    columns = {
+        "Shortcut Name": "string",
+        "Shortcut Path": "string",
+        "Source Type": "string",
+        "Source Workspace Id": "string",
+        "Source Workspace Name": "string",
+        "Source Item Id": "string",
+        "Source Item Name": "string",
+        "Source Item Type": "string",
+        "OneLake Path": "string",
+        "Connection Id": "string",
+        "Location": "string",
+        "Bucket": "string",
+        "SubPath": "string",
+    }
+    df = _create_dataframe(columns=columns)
+
+    responses = _base_api(
+        request=f"/v1/workspaces/{workspace_id}/items/{lakehouse_id}/shortcuts",
+        uses_pagination=True,
     )
 
-    response = client.get(
-        f"/v1/workspaces/{workspace_id}/items/{lakehouse_id}/shortcuts"
-    )
-
-    if response.status_code != 200:
-        raise FabricHTTPException(response)
-
-    responses = pagination(client, response)
-
     for r in responses:
         for i in r.get("value", []):
             tgt = i.get("target", {})
@@ -1293,14 +1301,17 @@ def list_capacities() -> pd.DataFrame:
         A pandas dataframe showing the capacities and their properties
     """
 
-    df = pd.DataFrame(
-        columns=["Id", "Display Name", "Sku", "Region", "State", "Admins"]
-    )
+    columns = {
+        "Id": "string",
+        "Display Name": "string",
+        "Sku": "string",
+        "Region": "string",
+        "State": "string",
+        "Admins": "string",
+    }
+    df = _create_dataframe(columns=columns)
 
-    client = fabric.PowerBIRestClient()
-    response = client.get("/v1.0/myorg/capacities")
-    if response.status_code != 200:
-        raise FabricHTTPException(response)
+    response = _base_api(request="/v1.0/myorg/capacities")
 
     for i in response.json().get("value", []):
         new_data = {
@@ -1360,10 +1371,7 @@ def list_reports_using_semantic_model(
 
     return dfR_filt
 
-    # client = fabric.PowerBIRestClient()
-    # response = client.get(
-    #    f"metadata/relations/downstream/dataset/{dataset_id}?apiVersion=3"
-    # )
+    # response = _base_api(request=f"metadata/relations/downstream/dataset/{dataset_id}?apiVersion=3")
 
     # response_json = response.json()
 
@@ -1414,17 +1422,16 @@ def list_report_semantic_model_objects(
     from sempy_labs.report import ReportWrapper
     from sempy_labs.tom import connect_semantic_model
 
-    dfRO = pd.DataFrame(
-        columns=[
-            "Report Name",
-            "Report Workspace Name",
-            "Table Name",
-            "Object Name",
-            "Object Type",
-            "Report Source",
-            "Report Source Object",
-        ]
-    )
+    columns = {
+        "Report Name": "string",
+        "Report Workspace Name": "string",
+        "Table Name": "string",
+        "Object Name": "string",
+        "Object Type": "string",
+        "Report Source": "string",
+        "Report Source Object": "string",
+    }
+    dfRO = _create_dataframe(columns=columns)
 
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
     (dataset_name, dataset_id) = resolve_dataset_name_and_id(dataset, workspace_id)
@@ -1515,6 +1522,8 @@ def list_semantic_model_object_report_usage(
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
     (dataset_name, dataset_id) = resolve_dataset_name_and_id(dataset, workspace_id)
 
+    fabric.refresh_tom_cache(workspace=workspace)
+
     dfR = list_report_semantic_model_objects(dataset=dataset_id, workspace=workspace_id)
     usage_column_name = "Report Usage Count"
 
@@ -1636,8 +1645,12 @@ def list_server_properties(workspace: Optional[str | UUID] = None) -> pd.DataFra
     tom_server.Dispose()
     df = pd.DataFrame(rows)
 
-    bool_cols = ["Is Read Only", "Requires Restart"]
-    df[bool_cols] = df[bool_cols].astype(bool)
+    column_map = {
+        "Is Read Only": "bool",
+        "Requires Restart": "bool",
+    }
+
+    _update_dataframe_datatypes(dataframe=df, column_map=column_map)
 
     return df
 
@@ -1744,3 +1757,87 @@ def list_semantic_model_errors(
                     )
 
     return pd.DataFrame(error_rows)
+
+
+@log
+def list_synonyms(dataset: str | UUID, workspace: Optional[str] = None):
+
+    from sempy_labs.tom import connect_semantic_model
+
+    columns = {
+        "Culture Name": "string",
+        "Table Name": "string",
+        "Object Name": "string",
+        "Object Type": "string",
+        "Synonym": "string",
+        "Type": "string",
+        "State": "string",
+        "Source": "string",
+        "Weight": "float_fillna",
+        "Last Modified": "datetime",
+    }
+
+    df = _create_dataframe(columns=columns)
+
+    rows = []
+    with connect_semantic_model(
+        dataset=dataset, workspace=workspace, readonly=True
+    ) as tom:
+        for c in tom.model.Cultures:
+            if c.LinguisticMetadata is not None:
+                lm = json.loads(c.LinguisticMetadata.Content)
+                if "Entities" in lm:
+                    for _, v in lm.get("Entities", []).items():
+                        binding = v.get("Definition", {}).get("Binding", {})
+
+                        t_name = binding.get("ConceptualEntity")
+                        object_name = binding.get("ConceptualProperty")
+
+                        if object_name is None:
+                            object_type = "Table"
+                            object_name = t_name
+                        elif any(
+                            m.Name == object_name and m.Parent.Name == t_name
+                            for m in tom.all_measures()
+                        ):
+                            object_type = "Measure"
+                        elif any(
+                            m.Name == object_name and m.Parent.Name == t_name
+                            for m in tom.all_columns()
+                        ):
+                            object_type = "Column"
+                        elif any(
+                            m.Name == object_name and m.Parent.Name == t_name
+                            for m in tom.all_hierarchies()
+                        ):
+                            object_type = "Hierarchy"
+
+                        merged_terms = defaultdict(dict)
+                        for t in v.get("Terms", []):
+                            for term, properties in t.items():
+                                normalized_term = term.lower()
+                                merged_terms[normalized_term].update(properties)
+
+                        for term, props in merged_terms.items():
+                            new_data = {
+                                "Culture Name": lm.get("Language"),
+                                "Table Name": t_name,
+                                "Object Name": object_name,
+                                "Object Type": object_type,
+                                "Synonym": term,
+                                "Type": props.get("Type"),
+                                "State": props.get("State"),
+                                "Source": props.get("Source", {}).get("Agent"),
+                                "Weight": props.get("Weight"),
+                                "Last Modified": props.get("LastModified"),
+                            }
+
+                            # Skip concatenation if new_data is empty or invalid
+                            if any(new_data.values()):
+                                rows.append(new_data)
+
+    if rows:
+        df = pd.DataFrame(rows)
+        _update_dataframe_datatypes(dataframe=df, column_map=columns)
+
+    return df