semantic-link-labs 0.7.4__py3-none-any.whl → 0.8.1__py3-none-any.whl

sempy_labs/_list_functions.py

@@ -5,7 +5,6 @@ from sempy_labs._helper_functions import (
     resolve_lakehouse_id,
     resolve_dataset_id,
     pagination,
-    lro,
     resolve_item_type,
     format_dax_object_name,
 )
@@ -83,7 +82,7 @@ def get_object_level_security(
 
 
 def list_tables(
-    dataset: str, workspace: Optional[str] = None, extended: Optional[bool] = False
+    dataset: str, workspace: Optional[str] = None, extended: bool = False
 ) -> pd.DataFrame:
     """
     Shows a semantic model's tables and their properties.
@@ -706,9 +705,9 @@ def list_lakehouses(workspace: Optional[str] = None) -> pd.DataFrame:
     return df
 
 
-def list_warehouses(workspace: Optional[str] = None) -> pd.DataFrame:
+def list_sql_endpoints(workspace: Optional[str] = None) -> pd.DataFrame:
     """
-    Shows the warehouses within a workspace.
+    Shows the SQL endpoints within a workspace.
 
     Parameters
     ----------
@@ -720,64 +719,10 @@ def list_warehouses(workspace: Optional[str] = None) -> pd.DataFrame:
     Returns
     -------
     pandas.DataFrame
-        A pandas dataframe showing the warehouses within a workspace.
+        A pandas dataframe showing the SQL endpoints within a workspace.
     """
 
-    df = pd.DataFrame(
-        columns=[
-            "Warehouse Name",
-            "Warehouse ID",
-            "Description",
-            "Connection Info",
-            "Created Date",
-            "Last Updated Time",
-        ]
-    )
-
-    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
-
-    client = fabric.FabricRestClient()
-    response = client.get(f"/v1/workspaces/{workspace_id}/warehouses")
-    if response.status_code != 200:
-        raise FabricHTTPException(response)
-
-    responses = pagination(client, response)
-
-    for r in responses:
-        for v in r.get("value", []):
-            prop = v.get("properties", {})
-
-            new_data = {
-                "Warehouse Name": v.get("displayName"),
-                "Warehouse ID": v.get("id"),
-                "Description": v.get("description"),
-                "Connection Info": prop.get("connectionInfo"),
-                "Created Date": prop.get("createdDate"),
-                "Last Updated Time": prop.get("lastUpdatedTime"),
-            }
-            df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
-
-    return df
-
-
-def list_sqlendpoints(workspace: Optional[str] = None) -> pd.DataFrame:
-    """
-    Shows the SQL Endpoints within a workspace.
-
-    Parameters
-    ----------
-    workspace : str, default=None
-        The Fabric workspace name.
-        Defaults to None which resolves to the workspace of the attached lakehouse
-        or if no lakehouse attached, resolves to the workspace of the notebook.
-
-    Returns
-    -------
-    pandas.DataFrame
-        A pandas dataframe showing the SQL Endpoints within a workspace.
-    """
-
-    df = pd.DataFrame(columns=["SQL Endpoint ID", "SQL Endpoint Name", "Description"])
+    df = pd.DataFrame(columns=["SQL Endpoint Id", "SQL Endpoint Name", "Description"])
 
     (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
 
@@ -792,7 +737,7 @@ def list_sqlendpoints(workspace: Optional[str] = None) -> pd.DataFrame:
         for v in r.get("value", []):
 
             new_data = {
-                "SQL Endpoint ID": v.get("id"),
+                "SQL Endpoint Id": v.get("id"),
                 "SQL Endpoint Name": v.get("displayName"),
                 "Description": v.get("description"),
             }
@@ -801,310 +746,6 @@ def list_sqlendpoints(workspace: Optional[str] = None) -> pd.DataFrame:
     return df
 
 
-def list_mirroredwarehouses(workspace: Optional[str] = None) -> pd.DataFrame:
-    """
-    Shows the mirrored warehouses within a workspace.
-
-    Parameters
-    ----------
-    workspace : str, default=None
-        The Fabric workspace name.
-        Defaults to None which resolves to the workspace of the attached lakehouse
-        or if no lakehouse attached, resolves to the workspace of the notebook.
-
-    Returns
-    -------
-    pandas.DataFrame
-        A pandas dataframe showing the mirrored warehouses within a workspace.
-    """
-
-    df = pd.DataFrame(
-        columns=["Mirrored Warehouse", "Mirrored Warehouse ID", "Description"]
-    )
-
-    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
-
-    client = fabric.FabricRestClient()
-    response = client.get(f"/v1/workspaces/{workspace_id}/mirroredWarehouses")
-    if response.status_code != 200:
-        raise FabricHTTPException(response)
-
-    responses = pagination(client, response)
-
-    for r in responses:
-        for v in r.get("value", []):
-
-            new_data = {
-                "Mirrored Warehouse": v.get("displayName"),
-                "Mirrored Warehouse ID": v.get("id"),
-                "Description": v.get("description"),
-            }
-            df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
-
-    return df
-
-
-def list_kqldatabases(workspace: Optional[str] = None) -> pd.DataFrame:
-    """
-    Shows the KQL databases within a workspace.
-
-    Parameters
-    ----------
-    workspace : str, default=None
-        The Fabric workspace name.
-        Defaults to None which resolves to the workspace of the attached lakehouse
-        or if no lakehouse attached, resolves to the workspace of the notebook.
-
-    Returns
-    -------
-    pandas.DataFrame
-        A pandas dataframe showing the KQL Databases within a workspace.
-    """
-
-    df = pd.DataFrame(
-        columns=[
-            "KQL Database Name",
-            "KQL Database ID",
-            "Description",
-            "Parent Eventhouse Item ID",
-            "Query Service URI",
-            "Ingestion Service URI",
-            "Kusto Database Type",
-        ]
-    )
-
-    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
-
-    client = fabric.FabricRestClient()
-    response = client.get(f"/v1/workspaces/{workspace_id}/kqlDatabases")
-    if response.status_code != 200:
-        raise FabricHTTPException(response)
-
-    responses = pagination(client, response)
-
-    for r in responses:
-        for v in r.get("value", []):
-            prop = v.get("properties", {})
-
-            new_data = {
-                "KQL Database Name": v.get("displayName"),
-                "KQL Database ID": v.get("id"),
-                "Description": v.get("description"),
-                "Parent Eventhouse Item ID": prop.get("parentEventhouseItemId"),
-                "Query Service URI": prop.get("queryServiceUri"),
-                "Ingestion Service URI": prop.get("ingestionServiceUri"),
-                "Kusto Database Type": prop.get("kustoDatabaseType"),
-            }
-            df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
-
-    return df
-
-
-def list_kqlquerysets(workspace: Optional[str] = None) -> pd.DataFrame:
-    """
-    Shows the KQL Querysets within a workspace.
-
-    Parameters
-    ----------
-    workspace : str, default=None
-        The Fabric workspace name.
-        Defaults to None which resolves to the workspace of the attached lakehouse
-        or if no lakehouse attached, resolves to the workspace of the notebook.
-
-    Returns
-    -------
-    pandas.DataFrame
-        A pandas dataframe showing the KQL Querysets within a workspace.
-    """
-
-    df = pd.DataFrame(columns=["KQL Queryset Name", "KQL Queryset ID", "Description"])
-
-    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
-
-    client = fabric.FabricRestClient()
-    response = client.get(f"/v1/workspaces/{workspace_id}/kqlQuerysets")
-    if response.status_code != 200:
-        raise FabricHTTPException(response)
-
-    responses = pagination(client, response)
-
-    for r in responses:
-        for v in r.get("value", []):
-
-            new_data = {
-                "KQL Queryset Name": v.get("displayName"),
-                "KQL Queryset ID": v.get("id"),
-                "Description": v.get("description"),
-            }
-            df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
-
-    return df
-
-
-def list_mlmodels(workspace: Optional[str] = None) -> pd.DataFrame:
-    """
-    Shows the ML models within a workspace.
-
-    Parameters
-    ----------
-    workspace : str, default=None
-        The Fabric workspace name.
-        Defaults to None which resolves to the workspace of the attached lakehouse
-        or if no lakehouse attached, resolves to the workspace of the notebook.
-
-    Returns
-    -------
-    pandas.DataFrame
-        A pandas dataframe showing the ML models within a workspace.
-    """
-
-    df = pd.DataFrame(columns=["ML Model Name", "ML Model ID", "Description"])
-
-    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
-
-    client = fabric.FabricRestClient()
-    response = client.get(f"/v1/workspaces/{workspace_id}/mlModels")
-    if response.status_code != 200:
-        raise FabricHTTPException(response)
-
-    responses = pagination(client, response)
-
-    for r in responses:
-        for v in r.get("value", []):
-            model_id = v.get("id")
-            modelName = v.get("displayName")
-            desc = v.get("description")
-
-            new_data = {
-                "ML Model Name": modelName,
-                "ML Model ID": model_id,
-                "Description": desc,
-            }
-            df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
-
-    return df
-
-
-def list_eventstreams(workspace: Optional[str] = None) -> pd.DataFrame:
-    """
-    Shows the eventstreams within a workspace.
-
-    Parameters
-    ----------
-    workspace : str, default=None
-        The Fabric workspace name.
-        Defaults to None which resolves to the workspace of the attached lakehouse
-        or if no lakehouse attached, resolves to the workspace of the notebook.
-
-    Returns
-    -------
-    pandas.DataFrame
-        A pandas dataframe showing the eventstreams within a workspace.
-    """
-
-    df = pd.DataFrame(columns=["Eventstream Name", "Eventstream ID", "Description"])
-
-    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
-
-    client = fabric.FabricRestClient()
-    response = client.get(f"/v1/workspaces/{workspace_id}/eventstreams")
-    if response.status_code != 200:
-        raise FabricHTTPException(response)
-
-    responses = pagination(client, response)
-
-    for r in responses:
-        for v in r.get("value", []):
-            new_data = {
-                "Eventstream Name": v.get("displayName"),
-                "Eventstream ID": v.get("id"),
-                "Description": v.get("description"),
-            }
-            df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
-
-    return df
-
-
-def list_datapipelines(workspace: Optional[str] = None) -> pd.DataFrame:
-    """
-    Shows the data pipelines within a workspace.
-
-    Parameters
-    ----------
-    workspace : str, default=None
-        The Fabric workspace name.
-        Defaults to None which resolves to the workspace of the attached lakehouse
-        or if no lakehouse attached, resolves to the workspace of the notebook.
-
-    Returns
-    -------
-    pandas.DataFrame
-        A pandas dataframe showing the data pipelines within a workspace.
-    """
-
-    df = pd.DataFrame(columns=["Data Pipeline Name", "Data Pipeline ID", "Description"])
-
-    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
-
-    client = fabric.FabricRestClient()
-    response = client.get(f"/v1/workspaces/{workspace_id}/dataPipelines")
-    if response.status_code != 200:
-        raise FabricHTTPException(response)
-
-    responses = pagination(client, response)
-
-    for r in responses:
-        for v in r.get("value", []):
-            new_data = {
-                "Data Pipeline Name": v.get("displayName"),
-                "Data Pipeline ID": v.get("id"),
-                "Description": v.get("description"),
-            }
-            df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
-
-    return df
-
-
-def list_mlexperiments(workspace: Optional[str] = None) -> pd.DataFrame:
-    """
-    Shows the ML experiments within a workspace.
-
-    Parameters
-    ----------
-    workspace : str, default=None
-        The Fabric workspace name.
-        Defaults to None which resolves to the workspace of the attached lakehouse
-        or if no lakehouse attached, resolves to the workspace of the notebook.
-
-    Returns
-    -------
-    pandas.DataFrame
-        A pandas dataframe showing the ML experiments within a workspace.
-    """
-
-    df = pd.DataFrame(columns=["ML Experiment Name", "ML Experiment ID", "Description"])
-
-    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
-
-    client = fabric.FabricRestClient()
-    response = client.get(f"/v1/workspaces/{workspace_id}/mlExperiments")
-    if response.status_code != 200:
-        raise FabricHTTPException(response)
-
-    responses = pagination(client, response)
-
-    for r in responses:
-        for v in r.get("value", []):
-            new_data = {
-                "ML Experiment Name": v.get("displayName"),
-                "ML Experiment ID": v.get("id"),
-                "Description": v.get("description"),
-            }
-            df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
-
-    return df
-
-
 def list_datamarts(workspace: Optional[str] = None) -> pd.DataFrame:
     """
     Shows the datamarts within a workspace.
@@ -1145,43 +786,6 @@ def list_datamarts(workspace: Optional[str] = None) -> pd.DataFrame:
     return df
 
 
-def create_warehouse(
-    warehouse: str, description: Optional[str] = None, workspace: Optional[str] = None
-):
-    """
-    Creates a Fabric warehouse.
-
-    Parameters
-    ----------
-    warehouse: str
-        Name of the warehouse.
-    description : str, default=None
-        A description of the warehouse.
-    workspace : str, default=None
-        The Fabric workspace name.
-        Defaults to None which resolves to the workspace of the attached lakehouse
-        or if no lakehouse attached, resolves to the workspace of the notebook.
-    """
-
-    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
-
-    request_body = {"displayName": warehouse}
-
-    if description:
-        request_body["description"] = description
-
-    client = fabric.FabricRestClient()
-    response = client.post(
-        f"/v1/workspaces/{workspace_id}/warehouses/", json=request_body
-    )
-
-    lro(client, response, status_codes=[201, 202])
-
-    print(
-        f"{icons.green_dot} The '{warehouse}' warehouse has been created within the '{workspace}' workspace."
-    )
-
-
 def update_item(
     item_type: str,
     current_name: str,
@@ -1209,27 +813,14 @@ def update_item(
     """
 
     (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
-
-    itemTypes = {
-        "DataPipeline": "dataPipelines",
-        "Eventstream": "eventstreams",
-        "KQLDatabase": "kqlDatabases",
-        "KQLQueryset": "kqlQuerysets",
-        "Lakehouse": "lakehouses",
-        "MLExperiment": "mlExperiments",
-        "MLModel": "mlModels",
-        "Notebook": "notebooks",
-        "Warehouse": "warehouses",
-    }
-
     item_type = item_type.replace(" ", "").capitalize()
 
-    if item_type not in itemTypes.keys():
+    if item_type not in icons.itemTypes.keys():
         raise ValueError(
             f"{icons.red_dot} The '{item_type}' is not a valid item type. "
         )
 
-    itemType = itemTypes[item_type]
+    itemType = icons.itemTypes[item_type]
 
     dfI = fabric.list_items(workspace=workspace, type=item_type)
     dfI_filt = dfI[(dfI["Display Name"] == current_name)]
@@ -1263,7 +854,7 @@ def update_item(
 
 
 def list_relationships(
-    dataset: str, workspace: Optional[str] = None, extended: Optional[bool] = False
+    dataset: str, workspace: Optional[str] = None, extended: bool = False
 ) -> pd.DataFrame:
     """
     Shows a semantic model's relationships and their properties.
@@ -1687,9 +1278,6 @@ def list_capacities() -> pd.DataFrame:
     """
     Shows the capacities and their properties.
 
-    Parameters
-    ----------
-
     Returns
     -------
     pandas.DataFrame
@@ -1774,3 +1362,212 @@ def list_reports_using_semantic_model(
             df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
 
     return df
+
+
+def list_report_semantic_model_objects(
+    dataset: str, workspace: Optional[str] = None, extended: bool = False
+) -> pd.DataFrame:
+    """
+    Shows a list of semantic model objects (i.e. columns, measures, hierarchies) used in all reports which feed data from
+    a given semantic model.
+
+    Requirement: Reports must be in the PBIR format.
+
+    Parameters
+    ----------
+    dataset : str
+        Name of the semantic model.
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    extended: bool, default=False
+        If True, adds an extra column called 'Valid Semantic Model Object' which identifies whether the semantic model object used
+        in the report exists in the semantic model which feeds data to the report.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing a list of semantic model objects (i.e. columns, measures, hierarchies) used in all reports which feed data from
+    a given semantic model.
+    """
+
+    from sempy_labs.report import ReportWrapper
+    from sempy_labs.tom import connect_semantic_model
+
+    dfRO = pd.DataFrame(
+        columns=[
+            "Report Name",
+            "Report Workspace Name",
+            "Table Name",
+            "Object Name",
+            "Object Type",
+            "Report Source",
+            "Report Source Object",
+        ]
+    )
+
+    # Collect all reports which use the semantic model
+    dfR = list_reports_using_semantic_model(dataset=dataset, workspace=workspace)
+
+    if len(dfR) == 0:
+        return dfRO
+
+    for _, r in dfR.iterrows():
+        report_name = r["Report Name"]
+        report_workspace = r["Report Workspace Name"]
+
+        rpt = ReportWrapper(report=report_name, workspace=report_workspace)
+        # Collect all semantic model objects used in the report
+        dfRSO = rpt.list_semantic_model_objects()
+        dfRSO["Report Name"] = report_name
+        dfRSO["Report Workspace Name"] = report_workspace
+        colName = "Report Name"
+        dfRSO.insert(0, colName, dfRSO.pop(colName))
+        colName = "Report Workspace Name"
+        dfRSO.insert(1, colName, dfRSO.pop(colName))
+
+        dfRO = pd.concat([dfRO, dfRSO], ignore_index=True)
+
+    # Collect all semantic model objects
+    if extended:
+        with connect_semantic_model(
+            dataset=dataset, readonly=True, workspace=workspace
+        ) as tom:
+            for index, row in dfRO.iterrows():
+                object_type = row["Object Type"]
+                if object_type == "Measure":
+                    dfRO.at[index, "Valid Semantic Model Object"] = any(
+                        o.Name == row["Object Name"] for o in tom.all_measures()
+                    )
+                elif object_type == "Column":
+                    dfRO.at[index, "Valid Semantic Model Object"] = any(
+                        format_dax_object_name(c.Parent.Name, c.Name)
+                        == format_dax_object_name(row["Table Name"], row["Object Name"])
+                        for c in tom.all_columns()
+                    )
+                elif object_type == "Hierarchy":
+                    dfRO.at[index, "Valid Semantic Model Object"] = any(
+                        format_dax_object_name(h.Parent.Name, h.Name)
+                        == format_dax_object_name(row["Table Name"], row["Object Name"])
+                        for h in tom.all_hierarchies()
+                    )
+
+    return dfRO
+
+
+def list_semantic_model_object_report_usage(
+    dataset: str,
+    workspace: Optional[str] = None,
+    include_dependencies: bool = False,
+    extended: bool = False,
+) -> pd.DataFrame:
+    """
+    Shows a list of semantic model objects and how many times they are referenced in all reports which rely on this semantic model.
+
+    Requirement: Reports must be in the PBIR format.
+
+    Parameters
+    ----------
+    dataset : str
+        Name of the semantic model.
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    include_dependencies : bool, default=False
+        If True, includes measure dependencies.
+    extended: bool, default=False
+        If True, adds columns 'Total Size', 'Data Size', 'Dictionary Size', 'Hierarchy Size' based on Vertipaq statistics.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing a list of semantic model objects and how many times they are referenced in all reports which rely on this semantic model. By default, the dataframe
+        is sorted descending by 'Report Usage Count'.
+    """
+
+    from sempy_labs._model_dependencies import get_measure_dependencies
+    from sempy_labs._helper_functions import format_dax_object_name
+
+    workspace = fabric.resolve_workspace_name(workspace)
+
+    dfR = list_report_semantic_model_objects(dataset=dataset, workspace=workspace)
+    usage_column_name = "Report Usage Count"
+
+    if not include_dependencies:
+        final_df = (
+            dfR.groupby(["Table Name", "Object Name", "Object Type"])
+            .size()
+            .reset_index(name=usage_column_name)
+        )
+    else:
+        df = pd.DataFrame(columns=["Table Name", "Object Name", "Object Type"])
+        dep = get_measure_dependencies(dataset=dataset, workspace=workspace)
+
+        for i, r in dfR.iterrows():
+            object_type = r["Object Type"]
+            table_name = r["Table Name"]
+            object_name = r["Object Name"]
+            new_data = {
+                "Table Name": table_name,
+                "Object Name": object_name,
+                "Object Type": object_type,
+            }
+            df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
+            if object_type == "Measure":
+                df_filt = dep[dep["Object Name"] == object_name][
+                    ["Referenced Table", "Referenced Object", "Referenced Object Type"]
+                ]
+                df_filt.rename(
+                    columns={
+                        "Referenced Table": "Table Name",
+                        "Referenced Object": "Object Name",
+                        "Referenced Object Type": "Object Type",
+                    },
+                    inplace=True,
+                )
+
+                df = pd.concat([df, df_filt], ignore_index=True)
+
+        final_df = (
+            df.groupby(["Table Name", "Object Name", "Object Type"])
+            .size()
+            .reset_index(name=usage_column_name)
+        )
+
+    if extended:
+        final_df["Object"] = format_dax_object_name(
+            final_df["Table Name"], final_df["Object Name"]
+        )
+        dfC = fabric.list_columns(dataset=dataset, workspace=workspace, extended=True)
+        dfC["Object"] = format_dax_object_name(dfC["Table Name"], dfC["Column Name"])
+        final_df = pd.merge(
+            final_df,
+            dfC[
+                [
+                    "Object",
+                    "Total Size",
+                    "Data Size",
+                    "Dictionary Size",
+                    "Hierarchy Size",
+                ]
+            ],
+            on="Object",
+            how="left",
+        )
+
+        ext_int_cols = ["Total Size", "Data Size", "Dictionary Size", "Hierarchy Size"]
+        final_df[ext_int_cols] = final_df[ext_int_cols].fillna(0).astype(int)
+        final_df.drop("Object", axis=1, inplace=True)
+
+    int_cols = [usage_column_name]
+    final_df[int_cols] = final_df[int_cols].astype(int)
+
+    final_df = final_df[final_df["Object Type"] != "Table"].sort_values(
+        by=usage_column_name, ascending=False
+    )
+
+    final_df.reset_index(drop=True, inplace=True)
+
+    return final_df