semantic-link-labs 0.7.4__py3-none-any.whl → 0.8.0__py3-none-any.whl

sempy_labs/_connections.py

@@ -2,7 +2,6 @@ import sempy.fabric as fabric
 import pandas as pd
 from sempy.fabric.exceptions import FabricHTTPException
 from typing import Optional
-import sempy_labs._icons as icons
 from sempy_labs._helper_functions import pagination
 
 
@@ -77,8 +76,9 @@ def list_connections() -> pd.DataFrame:
     return df
 
 
-def list_item_connections(item_name: str, item_type: str, workspace: Optional[str] = None) -> pd.DataFrame:
-
+def list_item_connections(
+    item_name: str, item_type: str, workspace: Optional[str] = None
+) -> pd.DataFrame:
     """
     Shows the list of connections that the specified item is connected to.
 
@@ -104,7 +104,9 @@ def list_item_connections(item_name: str, item_type: str, workspace: Optional[st
     workspace = fabric.resolve_workspace_name(workspace)
     workspace_id = fabric.resolve_workspace_id(workspace)
     item_type = item_type[0].upper() + item_type[1:]
-    item_id = fabric.resolve_item_id(item_name=item_name, type=item_type, workspace=workspace)
+    item_id = fabric.resolve_item_id(
+        item_name=item_name, type=item_type, workspace=workspace
+    )
 
     client = fabric.FabricRestClient()
     response = client.post(f"/v1/workspaces/{workspace_id}/items/{item_id}/connections")
@@ -123,17 +125,17 @@ def list_item_connections(item_name: str, item_type: str, workspace: Optional[st
     if response.status_code != 200:
         raise FabricHTTPException(response)
 
-    respnoses = pagination(client, response)
+    responses = pagination(client, response)
 
-    for r in respnoses:
-        for v in r.get('value', []):
+    for r in responses:
+        for v in r.get("value", []):
             new_data = {
-                "Connection Name": v.get('displayName'),
-                "Connection Id": v.get('id'),
-                "Connectivity Type": v.get('connectivityType'),
-                "Connection Type": v.get('connectionDetails', {}).get('type'),
-                "Connection Path": v.get('connectionDetails', {}).get('path'),
-                "Gateway Id": v.get('gatewayId'),
+                "Connection Name": v.get("displayName"),
+                "Connection Id": v.get("id"),
+                "Connectivity Type": v.get("connectivityType"),
+                "Connection Type": v.get("connectionDetails", {}).get("type"),
+                "Connection Path": v.get("connectionDetails", {}).get("path"),
+                "Gateway Id": v.get("gatewayId"),
             }
 
             df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)

sempy_labs/_git.py

@@ -132,19 +132,18 @@ def get_git_status(workspace: Optional[str] = None) -> pd.DataFrame:
     client = fabric.FabricRestClient()
     response = client.get(f"/v1/workspaces/{workspace_id}/git/status")
 
-    if response not in [200, 202]:
+    if response.status_code not in [200, 202]:
         raise FabricHTTPException(response)
 
     result = lro(client, response).json()
 
-    for v in result.get("value", []):
-        changes = v.get("changes", [])
+    for changes in result.get("changes", []):
         item_metadata = changes.get("itemMetadata", {})
         item_identifier = item_metadata.get("itemIdentifier", {})
 
         new_data = {
-            "Workspace Head": v.get("workspaceHead"),
-            "Remote Commit Hash": v.get("remoteCommitHash"),
+            "Workspace Head": result.get("workspaceHead"),
+            "Remote Commit Hash": result.get("remoteCommitHash"),
             "Object ID": item_identifier.get("objectId"),
             "Logical ID": item_identifier.get("logicalId"),
             "Item Type": item_metadata.get("itemType"),
@@ -199,21 +198,21 @@ def get_git_connection(workspace: Optional[str] = None) -> pd.DataFrame:
     if response.status_code != 200:
         raise FabricHTTPException(response)
 
-    for v in response.json().get("value", []):
-        provider_details = v.get("gitProviderDetails", {})
-        sync_details = v.get("gitSyncDetails", {})
-        new_data = {
-            "Organization Name": provider_details.get("organizationName"),
-            "Project Name": provider_details.get("projectName"),
-            "Git Provider Type": provider_details.get("gitProviderType"),
-            "Repository Name": provider_details.get("repositoryName"),
-            "Branch Name": provider_details.get("branchName"),
-            "Directory Name": provider_details.get("directoryName"),
-            "Workspace Head": sync_details.get("head"),
-            "Last Sync Time": sync_details.get("lastSyncTime"),
-            "Git Conneciton State": v.get("gitConnectionState"),
-        }
-        df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
+    r = response.json()
+    provider_details = r.get("gitProviderDetails", {})
+    sync_details = r.get("gitSyncDetails", {})
+    new_data = {
+        "Organization Name": provider_details.get("organizationName"),
+        "Project Name": provider_details.get("projectName"),
+        "Git Provider Type": provider_details.get("gitProviderType"),
+        "Repository Name": provider_details.get("repositoryName"),
+        "Branch Name": provider_details.get("branchName"),
+        "Directory Name": provider_details.get("directoryName"),
+        "Workspace Head": sync_details.get("head"),
+        "Last Sync Time": sync_details.get("lastSyncTime"),
+        "Git Connection State": r.get("gitConnectionState"),
+    }
+    df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
 
     return df
 
@@ -237,7 +236,7 @@ def initialize_git_connection(workspace: Optional[str] = None):
     client = fabric.FabricRestClient()
     response = client.post(f"/v1/workspaces/{workspace_id}/git/initializeConnection")
 
-    if response not in [200, 202]:
+    if response.status_code not in [200, 202]:
         raise FabricHTTPException(response)
 
     lro(client, response)

sempy_labs/_helper_functions.py

@@ -2,14 +2,14 @@ import sempy.fabric as fabric
 import re
 import json
 import base64
+import time
+from sempy.fabric.exceptions import FabricHTTPException
 import pandas as pd
 from functools import wraps
 import datetime
-import time
 from typing import Optional, Tuple, List
 from uuid import UUID
 import sempy_labs._icons as icons
-from sempy.fabric.exceptions import FabricHTTPException
 import urllib.parse
 from azure.core.credentials import TokenCredential, AccessToken
 
@@ -992,34 +992,6 @@ def get_language_codes(languages: str | List[str]):
     return languages
 
 
-def resolve_environment_id(environment: str, workspace: Optional[str] = None) -> UUID:
-    """
-    Obtains the environment Id for a given environment.
-
-    Parameters
-    ----------
-    environment: str
-        Name of the environment.
-    workspace : str, default=None
-        The Fabric workspace name.
-        Defaults to None which resolves to the workspace of the attached lakehouse
-        or if no lakehouse attached, resolves to the workspace of the notebook.
-    """
-
-    from sempy_labs._environments import list_environments
-
-    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
-
-    dfE = list_environments(workspace=workspace)
-    dfE_filt = dfE[dfE["Environment Name"] == environment]
-    if len(dfE_filt) == 0:
-        raise ValueError(
-            f"{icons.red_dot} The '{environment}' environment does not exist within the '{workspace}' workspace."
-        )
-
-    return dfE_filt["Environment Id"].iloc[0]
-
-
 def get_azure_token_credentials(
     key_vault_uri: str,
     key_vault_tenant_id: str,
@@ -1048,3 +1020,34 @@ def get_azure_token_credentials(
     }
 
     return token, credential, headers
+
+
+def convert_to_alphanumeric_lowercase(input_string):
+
+    cleaned_string = re.sub(r"[^a-zA-Z0-9]", "", input_string)
+    cleaned_string = cleaned_string.lower()
+
+    return cleaned_string
+
+
+def resolve_environment_id(environment: str, workspace: Optional[str] = None) -> UUID:
+    """
+    Obtains the environment Id for a given environment.
+
+    Parameters
+    ----------
+    environment: str
+        Name of the environment.
+    """
+    from sempy_labs._environments import list_environments
+
+    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
+
+    dfE = list_environments(workspace=workspace)
+    dfE_filt = dfE[dfE["Environment Name"] == environment]
+    if len(dfE_filt) == 0:
+        raise ValueError(
+            f"{icons.red_dot} The '{environment}' environment does not exist within the '{workspace}' workspace."
+        )
+
+    return dfE_filt["Environment Id"].iloc[0]

sempy_labs/_icons.py

@@ -74,3 +74,22 @@ language_map = {
 }
 workspace_roles = ["Admin", "Member", "Viewer", "Contributor"]
 principal_types = ["App", "Group", "None", "User"]
+azure_api_version = "2023-11-01"
+migrate_capacity_suffix = "fsku"
+sku_mapping = {
+    "A1": "F8",
+    "EM1": "F8",
+    "A2": "F16",
+    "EM2": "F16",
+    "A3": "F32",
+    "EM3": "F32",
+    "A4": "F64",
+    "P1": "F64",
+    "A5": "F128",
+    "P2": "F128",
+    "A6": "F256",
+    "P3": "F256",
+    "A7": "F512",
+    "P4": "F512",
+    "P5": "F1024",
+}

sempy_labs/_list_functions.py

@@ -8,6 +8,7 @@ from sempy_labs._helper_functions import (
     lro,
     resolve_item_type,
     format_dax_object_name,
+    pagination,
 )
 import pandas as pd
 from typing import Optional
@@ -1774,3 +1775,212 @@ def list_reports_using_semantic_model(
             df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
 
     return df
+
+
+def list_report_semantic_model_objects(
+    dataset: str, workspace: Optional[str] = None, extended: Optional[bool] = False
+) -> pd.DataFrame:
+    """
+    Shows a list of semantic model objects (i.e. columns, measures, hierarchies) used in all reports which feed data from
+    a given semantic model.
+
+    Requirement: Reports must be in the PBIR format.
+
+    Parameters
+    ----------
+    dataset : str
+        Name of the semantic model.
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    extended: bool, default=False
+        If True, adds an extra column called 'Valid Semantic Model Object' which identifies whether the semantic model object used
+        in the report exists in the semantic model which feeds data to the report.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing a list of semantic model objects (i.e. columns, measures, hierarchies) used in all reports which feed data from
+    a given semantic model.
+    """
+
+    from sempy_labs.report import ReportWrapper
+    from sempy_labs.tom import connect_semantic_model
+
+    dfRO = pd.DataFrame(
+        columns=[
+            "Report Name",
+            "Report Workspace Name",
+            "Table Name",
+            "Object Name",
+            "Object Type",
+            "Report Source",
+            "Report Source Object",
+        ]
+    )
+
+    # Collect all reports which use the semantic model
+    dfR = list_reports_using_semantic_model(dataset=dataset, workspace=workspace)
+
+    if len(dfR) == 0:
+        return dfRO
+
+    for _, r in dfR.iterrows():
+        report_name = r["Report Name"]
+        report_workspace = r["Report Workspace Name"]
+
+        rpt = ReportWrapper(report=report_name, workspace=report_workspace)
+        # Collect all semantic model objects used in the report
+        dfRSO = rpt.list_semantic_model_objects()
+        dfRSO["Report Name"] = report_name
+        dfRSO["Report Workspace Name"] = report_workspace
+        colName = "Report Name"
+        dfRSO.insert(0, colName, dfRSO.pop(colName))
+        colName = "Report Workspace Name"
+        dfRSO.insert(1, colName, dfRSO.pop(colName))
+
+        dfRO = pd.concat([dfRO, dfRSO], ignore_index=True)
+
+    # Collect all semantic model objects
+    if extended:
+        with connect_semantic_model(
+            dataset=dataset, readonly=True, workspace=workspace
+        ) as tom:
+            for index, row in dfRO.iterrows():
+                object_type = row["Object Type"]
+                if object_type == "Measure":
+                    dfRO.at[index, "Valid Semantic Model Object"] = any(
+                        o.Name == row["Object Name"] for o in tom.all_measures()
+                    )
+                elif object_type == "Column":
+                    dfRO.at[index, "Valid Semantic Model Object"] = any(
+                        format_dax_object_name(c.Parent.Name, c.Name)
+                        == format_dax_object_name(row["Table Name"], row["Object Name"])
+                        for c in tom.all_columns()
+                    )
+                elif object_type == "Hierarchy":
+                    dfRO.at[index, "Valid Semantic Model Object"] = any(
+                        format_dax_object_name(h.Parent.Name, h.Name)
+                        == format_dax_object_name(row["Table Name"], row["Object Name"])
+                        for h in tom.all_hierarchies()
+                    )
+
+    return dfRO
+
+
+def list_semantic_model_object_report_usage(
+    dataset: str,
+    workspace: Optional[str] = None,
+    include_dependencies: Optional[bool] = False,
+    extended: Optional[bool] = False,
+) -> pd.DataFrame:
+    """
+    Shows a list of semantic model objects and how many times they are referenced in all reports which rely on this semantic model.
+
+    Requirement: Reports must be in the PBIR format.
+
+    Parameters
+    ----------
+    dataset : str
+        Name of the semantic model.
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    include_dependencies : bool, default=False
+        If True, includes measure dependencies.
+    extended: bool, default=False
+        If True, adds columns 'Total Size', 'Data Size', 'Dictionary Size', 'Hierarchy Size' based on Vertipaq statistics.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing a list of semantic model objects and how many times they are referenced in all reports which rely on this semantic model. By default, the dataframe
+        is sorted descending by 'Report Usage Count'.
+    """
+
+    from sempy_labs._model_dependencies import get_measure_dependencies
+    from sempy_labs._helper_functions import format_dax_object_name
+
+    workspace = fabric.resolve_workspace_name(workspace)
+
+    dfR = list_report_semantic_model_objects(dataset=dataset, workspace=workspace)
+    usage_column_name = "Report Usage Count"
+
+    if not include_dependencies:
+        final_df = (
+            dfR.groupby(["Table Name", "Object Name", "Object Type"])
+            .size()
+            .reset_index(name=usage_column_name)
+        )
+    else:
+        df = pd.DataFrame(columns=["Table Name", "Object Name", "Object Type"])
+        dep = get_measure_dependencies(dataset=dataset, workspace=workspace)
+
+        for i, r in dfR.iterrows():
+            object_type = r["Object Type"]
+            table_name = r["Table Name"]
+            object_name = r["Object Name"]
+            new_data = {
+                "Table Name": table_name,
+                "Object Name": object_name,
+                "Object Type": object_type,
+            }
+            df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
+            if object_type == "Measure":
+                df_filt = dep[dep["Object Name"] == object_name][
+                    ["Referenced Table", "Referenced Object", "Referenced Object Type"]
+                ]
+                df_filt.rename(
+                    columns={
+                        "Referenced Table": "Table Name",
+                        "Referenced Object": "Object Name",
+                        "Referenced Object Type": "Object Type",
+                    },
+                    inplace=True,
+                )
+
+                df = pd.concat([df, df_filt], ignore_index=True)
+
+        final_df = (
+            df.groupby(["Table Name", "Object Name", "Object Type"])
+            .size()
+            .reset_index(name=usage_column_name)
+        )
+
+    if extended:
+        final_df["Object"] = format_dax_object_name(
+            final_df["Table Name"], final_df["Object Name"]
+        )
+        dfC = fabric.list_columns(dataset=dataset, workspace=workspace, extended=True)
+        dfC["Object"] = format_dax_object_name(dfC["Table Name"], dfC["Column Name"])
+        final_df = pd.merge(
+            final_df,
+            dfC[
+                [
+                    "Object",
+                    "Total Size",
+                    "Data Size",
+                    "Dictionary Size",
+                    "Hierarchy Size",
+                ]
+            ],
+            on="Object",
+            how="left",
+        )
+
+        ext_int_cols = ["Total Size", "Data Size", "Dictionary Size", "Hierarchy Size"]
+        final_df[ext_int_cols] = final_df[ext_int_cols].fillna(0).astype(int)
+        final_df.drop("Object", axis=1, inplace=True)
+
+    int_cols = [usage_column_name]
+    final_df[int_cols] = final_df[int_cols].astype(int)
+
+    final_df = final_df[final_df["Object Type"] != "Table"].sort_values(
+        by=usage_column_name, ascending=False
+    )
+
+    final_df.reset_index(drop=True, inplace=True)
+
+    return final_df

sempy_labs/_model_bpa.py

@@ -506,7 +506,7 @@ def run_model_bpa(
             content_html += f'<td>{row["Object Type"]}</td>'
             content_html += f'<td>{row["Object Name"]}</td>'
             content_html += f'<td style="text-align: center;">{row["Severity"]}</td>'
-            #content_html += f'<td>{row["Severity"]}</td>'
+            # content_html += f'<td>{row["Severity"]}</td>'
             content_html += "</tr>"
         content_html += "</table>"
 

sempy_labs/_query_scale_out.py

@@ -339,7 +339,6 @@ def list_qso_settings(
     if dataset is not None:
         dataset_id = resolve_dataset_id(dataset, workspace)
 
-    workspace_id = fabric.get_workspace_id()
     df = pd.DataFrame(
         columns=[
             "Dataset Id",
@@ -413,10 +412,12 @@ def set_workspace_default_storage_format(
     dfW = fabric.list_workspaces(filter=f"name eq '{workspace}'")
     if len(dfW) == 0:
         raise ValueError()
-    current_storage_format = dfW['Default Dataset Storage Format'].iloc[0]
+    current_storage_format = dfW["Default Dataset Storage Format"].iloc[0]
 
     if current_storage_format == storage_format:
-        print(f"{icons.info} The '{workspace}' is already set to a default storage format of '{current_storage_format}'.")
+        print(
+            f"{icons.info} The '{workspace}' is already set to a default storage format of '{current_storage_format}'."
+        )
         return
 
     request_body = {

sempy_labs/_spark.py

@@ -298,7 +298,9 @@ def delete_custom_pool(pool_name: str, workspace: Optional[str] = None):
     )
 
 
-def get_spark_settings(workspace: Optional[str] = None) -> pd.DataFrame:
+def get_spark_settings(
+    workspace: Optional[str] = None, return_dataframe: Optional[bool] = True
+) -> pd.DataFrame | dict:
     """
     Shows the spark settings for a workspace.
 
@@ -308,10 +310,12 @@ def get_spark_settings(workspace: Optional[str] = None) -> pd.DataFrame:
         The name of the Fabric workspace.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
+    return_dataframe : bool, default=True
+        If True, returns a pandas dataframe. If False, returns a json dictionary.
 
     Returns
     -------
-    pandas.DataFrame
+    pandas.DataFrame | dict
         A pandas dataframe showing the spark settings for a workspace.
     """
 
@@ -363,12 +367,15 @@ def get_spark_settings(workspace: Optional[str] = None) -> pd.DataFrame:
         "High Concurrency Enabled",
         "Customize Compute Enabled",
     ]
-    int_cols = ["Max Node Count", "Max Executors"]
+    # int_cols = ["Max Node Count", "Max Executors"]
 
     df[bool_cols] = df[bool_cols].astype(bool)
-    df[int_cols] = df[int_cols].astype(int)
+    # df[int_cols] = df[int_cols].astype(int)
 
-    return df
+    if return_dataframe:
+        return df
+    else:
+        return response.json()
 
 
 def update_spark_settings(
@@ -420,38 +427,26 @@ def update_spark_settings(
     # https://learn.microsoft.com/en-us/rest/api/fabric/spark/workspace-settings/update-spark-settings?tabs=HTTP
     (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
 
-    dfS = get_spark_settings(workspace=workspace)
-
-    if automatic_log_enabled is None:
-        automatic_log_enabled = bool(dfS["Automatic Log Enabled"].iloc[0])
-    if high_concurrency_enabled is None:
-        high_concurrency_enabled = bool(dfS["High Concurrency Enabled"].iloc[0])
-    if customize_compute_enabled is None:
-        customize_compute_enabled = bool(dfS["Customize Compute Enabled"].iloc[0])
-    if default_pool_name is None:
-        default_pool_name = dfS["Default Pool Name"].iloc[0]
-    if max_node_count is None:
-        max_node_count = int(dfS["Max Node Count"].iloc[0])
-    if max_executors is None:
-        max_executors = int(dfS["Max Executors"].iloc[0])
-    if environment_name is None:
-        environment_name = dfS["Environment Name"].iloc[0]
-    if runtime_version is None:
-        runtime_version = dfS["Runtime Version"].iloc[0]
+    request_body = get_spark_settings(workspace=workspace, return_dataframe=False)
 
-    request_body = {
-        "automaticLog": {"enabled": automatic_log_enabled},
-        "highConcurrency": {"notebookInteractiveRunEnabled": high_concurrency_enabled},
-        "pool": {
-            "customizeComputeEnabled": customize_compute_enabled,
-            "defaultPool": {"name": default_pool_name, "type": "Workspace"},
-            "starterPool": {
-                "maxNodeCount": max_node_count,
-                "maxExecutors": max_executors,
-            },
-        },
-        "environment": {"name": environment_name, "runtimeVersion": runtime_version},
-    }
+    if automatic_log_enabled is not None:
+        request_body["automaticLog"]["enabled"] = automatic_log_enabled
+    if high_concurrency_enabled is not None:
+        request_body["highConcurrency"][
+            "notebookInteractiveRunEnabled"
+        ] = high_concurrency_enabled
+    if customize_compute_enabled is not None:
+        request_body["pool"]["customizeComputeEnabled"] = customize_compute_enabled
+    if default_pool_name is not None:
+        request_body["pool"]["defaultPool"]["name"] = default_pool_name
+    if max_node_count is not None:
+        request_body["pool"]["starterPool"]["maxNodeCount"] = max_node_count
+    if max_executors is not None:
+        request_body["pool"]["starterPool"]["maxExecutors"] = max_executors
+    if environment_name is not None:
+        request_body["environment"]["name"] = environment_name
+    if runtime_version is not None:
+        request_body["environment"]["runtimeVersion"] = runtime_version
 
     client = fabric.FabricRestClient()
     response = client.patch(