semantic-link-labs 0.8.2__py3-none-any.whl → 0.8.4__py3-none-any.whl

sempy_labs/_helper_functions.py

@@ -13,6 +13,8 @@ from uuid import UUID
 import sempy_labs._icons as icons
 import urllib.parse
 from azure.core.credentials import TokenCredential, AccessToken
+import numpy as np
+from IPython.display import display, HTML
 
 
 def create_abfss_path(
@@ -961,15 +963,15 @@ class FabricTokenCredential(TokenCredential):
         **kwargs: any,
     ) -> AccessToken:
 
-        from notebookutils import mssparkutils
+        import notebookutils
 
-        token = mssparkutils.credentials.getToken(scopes)
+        token = notebookutils.credentials.getToken(scopes)
         access_token = AccessToken(token, 0)
 
         return access_token
 
 
-def get_adls_client(account_name):
+def _get_adls_client(account_name):
 
     from azure.storage.filedatalake import DataLakeServiceClient
 
@@ -1017,19 +1019,22 @@ def get_language_codes(languages: str | List[str]):
     return languages
 
 
-def get_azure_token_credentials(
+def _get_azure_token_credentials(
     key_vault_uri: str,
     key_vault_tenant_id: str,
     key_vault_client_id: str,
     key_vault_client_secret: str,
+    audience: str = "https://management.azure.com/.default",
 ) -> Tuple[str, str, dict]:
 
-    from notebookutils import mssparkutils
+    import notebookutils
     from azure.identity import ClientSecretCredential
 
-    tenant_id = mssparkutils.credentials.getSecret(key_vault_uri, key_vault_tenant_id)
-    client_id = mssparkutils.credentials.getSecret(key_vault_uri, key_vault_client_id)
-    client_secret = mssparkutils.credentials.getSecret(
+    # "https://analysis.windows.net/powerbi/api/.default"
+
+    tenant_id = notebookutils.credentials.getSecret(key_vault_uri, key_vault_tenant_id)
+    client_id = notebookutils.credentials.getSecret(key_vault_uri, key_vault_client_id)
+    client_secret = notebookutils.credentials.getSecret(
         key_vault_uri, key_vault_client_secret
     )
 
@@ -1037,7 +1042,7 @@ def get_azure_token_credentials(
         tenant_id=tenant_id, client_id=client_id, client_secret=client_secret
     )
 
-    token = credential.get_token("https://management.azure.com/.default").token
+    token = credential.get_token(audience).token
 
     headers = {
         "Authorization": f"Bearer {token}",
@@ -1077,7 +1082,7 @@ def resolve_environment_id(environment: str, workspace: Optional[str] = None) ->
     )
 
 
-def make_clickable(val):
+def _make_clickable(val):
 
     return f'<a target="_blank" href="{val}">{val}</a>'
 
@@ -1132,7 +1137,7 @@ def generate_guid():
     return str(uuid.uuid4())
 
 
-def get_max_run_id(lakehouse: str, table_name: str) -> int:
+def _get_max_run_id(lakehouse: str, table_name: str) -> int:
 
     from pyspark.sql import SparkSession
 
@@ -1142,3 +1147,173 @@ def get_max_run_id(lakehouse: str, table_name: str) -> int:
     max_run_id = dfSpark.collect()[0][0]
 
     return max_run_id
+
+
+def _make_list_unique(my_list):
+
+    return list(set(my_list))
+
+
+def _get_partition_map(dataset: str, workspace: Optional[str] = None) -> pd.DataFrame:
+
+    if workspace is None:
+        workspace = fabric.resolve_workspace_name()
+
+    partitions = fabric.evaluate_dax(
+        dataset=dataset,
+        workspace=workspace,
+        dax_string="""
+    select [ID] AS [PartitionID], [TableID], [Name] AS [PartitionName] from $system.tmschema_partitions
+    """,
+    )
+
+    tables = fabric.evaluate_dax(
+        dataset=dataset,
+        workspace=workspace,
+        dax_string="""
+    select [ID] AS [TableID], [Name] AS [TableName] from $system.tmschema_tables
+    """,
+    )
+
+    partition_map = pd.merge(partitions, tables, on="TableID", how="left")
+    partition_map["PartitionID"] = partition_map["PartitionID"].astype(str)
+    partition_counts = partition_map.groupby("TableID")["PartitionID"].transform(
+        "count"
+    )
+    partition_map["Object Name"] = partition_map.apply(
+        lambda row: (
+            f"'{row['TableName']}'[{row['PartitionName']}]"
+            if partition_counts[row.name] > 1
+            else row["TableName"]
+        ),
+        axis=1,
+    )
+    return partition_map
+
+
+def _show_chart(spec, title):
+
+    h = f"""
+    <!DOCTYPE html>
+    <html>
+        <head>
+            <script src="https://cdn.jsdelivr.net/npm/vega@5"></script>
+            <script src="https://cdn.jsdelivr.net/npm/vega-lite@5"></script>
+            <script src="https://cdn.jsdelivr.net/npm/vega-embed@6"></script>
+            <style>
+                table, th, td {{
+                border: 10px solid #e7e9eb;
+                border-collapse: collapse;
+                }}
+            </style>
+        </head>
+        <body>
+            <table>
+                <tr>
+                    <td style="text-align: center;">
+                        <h1>{title}</h1>
+                    </td>
+                </tr>
+                <tr>
+                    <td>
+                        <div id="vis"></div>
+                    </td>
+                </tr>
+            </table>    
+            <script type="text/javascript">
+                var spec = {spec};
+                var opt = {{"renderer": "canvas", "actions": false}};
+                vegaEmbed("#vis", spec, opt);
+            </script>
+        </body>
+    </html>"""
+
+    display(HTML(h))
+
+
+def _process_and_display_chart(df, title, widget):
+
+    # Convert time columns to milliseconds
+    df["Start"] = df["Start Time"].astype(np.int64) / int(1e6)
+    df["End"] = df["End Time"].astype(np.int64) / int(1e6)
+
+    # Calculate the time offset for proper Gantt chart rendering
+    Offset = min(df["Start"])
+    df["Start"] = df["Start"] - Offset
+    df["End"] = df["End"] - Offset
+
+    # Vega-Lite spec for Gantt chart
+    spec = (
+        """{
+        "$schema": "https://vega.github.io/schema/vega-lite/v5.json",
+        "description": "A simple bar chart with ranged data (aka Gantt Chart).",
+        "data": { "values": """
+        + df.to_json(orient="records")
+        + """ },
+        "width": 700,
+        "height": 400,
+        "mark": "bar",
+        "encoding": {
+            "y": {
+                "field": "Object Name",
+                "type": "ordinal",
+                "axis": {
+                    "labelFontSize": 15,
+                    "titleFontSize": 20,
+                    "title": "Object"
+                }
+            },
+            "x": {
+                "field": "Start", 
+                "type": "quantitative", 
+                "title": "milliseconds",
+                "axis": {
+                    "titleFontSize": 20
+                }
+            },
+            "x2": {"field": "End"},
+            "color": {
+                "field": "Event Subclass",
+                "scale": {
+                    "domain": ["Process", "ExecuteSql"],
+                    "range": ["#FFC000","#0070C0"]
+                },
+                "legend": {
+                    "labelFontSize": 20,
+                    "titleFontSize": 20,
+                    "title": "Event Type"
+                }
+            },
+            "tooltip": [
+                {"field": "Duration", "type": "quantitative", "format": ","},
+                {"field": "Cpu Time", "type": "quantitative", "format": ","},
+                {"field": "Event Subclass", "type": "nominal"}
+            ]
+        }
+    }"""
+    )
+
+    with widget:
+        widget.clear_output(wait=True)
+    _show_chart(spec, title=title)
+
+
+def _convert_data_type(input_data_type: str) -> str:
+
+    data_type_mapping = {
+        "string": "String",
+        "int": "Int64",
+        "tinyint": "Int64",
+        "smallint": "Int64",
+        "bigint": "Int64",
+        "boolean": "Boolean",
+        "timestamp": "DateTime",
+        "date": "DateTime",
+        "double": "Double",
+        "float": "Double",
+    }
+
+    if "decimal" in input_data_type:
+        return "Decimal"
+    else:
+        return data_type_mapping.get(input_data_type)

sempy_labs/_icons.py

@@ -10,20 +10,6 @@ bullet = "\u2022"
 warning = "⚠️"
 error = "\u274C"
 info = "ℹ️"
-data_type_mapping = {
-    "string": "String",
-    "int": "Int64",
-    "tinyint": "Int64",
-    "smallint": "Int64",
-    "bigint": "Int64",
-    "boolean": "Boolean",
-    "timestamp": "DateTime",
-    "date": "DateTime",
-    "decimal(38,18)": "Decimal",
-    "decimal(19,4)": "Decimal",
-    "double": "Double",
-    "float": "Double",
-}
 measure_icon = "\u2211"
 table_icon = "\u229E"
 column_icon = "\u229F"
@@ -93,14 +79,15 @@ sku_mapping = {
     "P4": "F512",
     "P5": "F1024",
 }
-refreshTypes = [
-    "full",
-    "automatic",
-    "dataOnly",
-    "calculate",
-    "clearValues",
-    "defragment",
-]
+
+refresh_type_mapping = {
+    "full": "full",
+    "auto": "automatic",
+    "data": "dataOnly",
+    "calc": "calculate",
+    "clear": "clearValues",
+    "defrag": "defragment",
+}
 
 itemTypes = {
     "DataPipeline": "dataPipelines",
@@ -114,3 +101,49 @@ itemTypes = {
     "Warehouse": "warehouses",
 }
 default_schema = "dbo"
+
+data_type_string = "string"
+data_type_long = "long"
+data_type_timestamp = "timestamp"
+data_type_double = "double"
+data_type_bool = "bool"
+int_format = "int"
+pct_format = "pct"
+no_format = ""
+
+bpa_schema = {
+    "Capacity Name": data_type_string,
+    "Capacity Id": data_type_string,
+    "Workspace Name": data_type_string,
+    "Workspace Id": data_type_string,
+    "Dataset Name": data_type_string,
+    "Dataset Id": data_type_string,
+    "Configured By": data_type_string,
+    "Rule Name": data_type_string,
+    "Category": data_type_string,
+    "Severity": data_type_string,
+    "Object Type": data_type_string,
+    "Object Name": data_type_string,
+    "Description": data_type_string,
+    "URL": data_type_string,
+    "RunId": data_type_long,
+    "Timestamp": data_type_timestamp,
+}
+
+sll_ann_name = "PBI_ProTooling"
+sll_prefix = "SLL_"
+sll_tags = []
+base_cols = ["EventClass", "EventSubclass", "CurrentTime", "TextData"]
+end_cols = base_cols + [
+    "StartTime",
+    "EndTime",
+    "Duration",
+    "CpuTime",
+    "Success",
+    "IntegerData",
+    "ObjectID",
+]
+refresh_event_schema = {
+    "JobGraph": base_cols,
+    "ProgressReportEnd": end_cols,
+}

sempy_labs/_kql_databases.py

@@ -14,6 +14,8 @@ def list_kql_databases(workspace: Optional[str] = None) -> pd.DataFrame:
     """
     Shows the KQL databases within a workspace.
 
+    This is a wrapper function for the following API: `Items - List KQL Databases <https://learn.microsoft.com/rest/api/fabric/kqldatabase/items/list-kql-databases`_.
+
     Parameters
     ----------
     workspace : str, default=None
@@ -72,6 +74,8 @@ def create_kql_database(
     """
     Creates a KQL database.
 
+    This is a wrapper function for the following API: `Items - Create KQL Database <https://learn.microsoft.com/rest/api/fabric/kqldatabase/items/create-kql-database`_.
+
     Parameters
     ----------
     name: str
@@ -107,6 +111,8 @@ def delete_kql_database(name: str, workspace: Optional[str] = None):
     """
     Deletes a KQL database.
 
+    This is a wrapper function for the following API: `Items - Delete KQL Database <https://learn.microsoft.com/rest/api/fabric/kqldatabase/items/delete-kql-database`_.
+
     Parameters
     ----------
     name: str

sempy_labs/_kql_querysets.py

@@ -14,6 +14,8 @@ def list_kql_querysets(workspace: Optional[str] = None) -> pd.DataFrame:
     """
     Shows the KQL querysets within a workspace.
 
+    This is a wrapper function for the following API: `Items - List KQL Querysets <https://learn.microsoft.com/rest/api/fabric/kqlqueryset/items/list-kql-querysets`_.
+
     Parameters
     ----------
     workspace : str, default=None
@@ -62,6 +64,8 @@ def create_kql_queryset(
     """
     Creates a KQL queryset.
 
+    This is a wrapper function for the following API: `Items - Create KQL Queryset <https://learn.microsoft.com/rest/api/fabric/kqlqueryset/items/create-kql-queryset`_.
+
     Parameters
     ----------
     name: str
@@ -97,6 +101,8 @@ def delete_kql_queryset(name: str, workspace: Optional[str] = None):
     """
     Deletes a KQL queryset.
 
+    This is a wrapper function for the following API: `Items - Delete KQL Queryset <https://learn.microsoft.com/rest/api/fabric/kqlqueryset/items/delete-kql-queryset`_.
+
     Parameters
     ----------
     name: str

sempy_labs/_list_functions.py

@@ -1487,7 +1487,7 @@ def list_semantic_model_object_report_usage(
         is sorted descending by 'Report Usage Count'.
     """
 
-    from sempy_labs._model_dependencies import get_measure_dependencies
+    from sempy_labs._model_dependencies import get_model_calc_dependencies
     from sempy_labs._helper_functions import format_dax_object_name
 
     workspace = fabric.resolve_workspace_name(workspace)
@@ -1503,7 +1503,7 @@ def list_semantic_model_object_report_usage(
         )
     else:
         df = pd.DataFrame(columns=["Table Name", "Object Name", "Object Type"])
-        dep = get_measure_dependencies(dataset=dataset, workspace=workspace)
+        dep = get_model_calc_dependencies(dataset=dataset, workspace=workspace)
 
         for i, r in dfR.iterrows():
             object_type = r["Object Type"]
@@ -1515,7 +1515,10 @@ def list_semantic_model_object_report_usage(
                 "Object Type": object_type,
             }
             df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
-            if object_type == "Measure":
+            df["Object Type"] = df["Object Type"].replace(
+                "Attribute Hierarchy", "Column"
+            )
+            if object_type in ["Measure", "Calc Column", "Calc Table", "Hierarchy"]:
                 df_filt = dep[dep["Object Name"] == object_name][
                     ["Referenced Table", "Referenced Object", "Referenced Object Type"]
                 ]

sempy_labs/_managed_private_endpoints.py

@@ -0,0 +1,166 @@
+import sempy.fabric as fabric
+import pandas as pd
+import sempy_labs._icons as icons
+from typing import Optional
+from sempy_labs._helper_functions import (
+    resolve_workspace_name_and_id,
+    lro,
+    pagination,
+)
+from sempy.fabric.exceptions import FabricHTTPException
+from uuid import UUID
+
+
+def create_managed_private_endpoint(
+    name: str,
+    target_private_link_resource_id: UUID,
+    target_subresource_type: str,
+    request_message: Optional[str] = None,
+    workspace: Optional[str] = None,
+):
+    """
+    Creates a managed private endpoint.
+
+    This is a wrapper function for the following API: `Managed Private Endpoints - Create Workspace Managed Private Endpoint <https://learn.microsoft.com/rest/api/fabric/core/managed-private-endpoints/create-workspace-managed-private-endpoint>`.
+
+    Parameters
+    ----------
+    name: str
+        Name of the managed private endpoint.
+    target_private_link_resource_id: UUID
+        Resource Id of data source for which private endpoint needs to be created.
+    target_subresource_type : str
+        Sub-resource pointing to Private-link resoure.
+    request_message : str, default=None
+        Message to approve private endpoint request. Should not be more than 140 characters.
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
+
+    request_body = {
+        "name": name,
+        "targetPrivateLinkResourceId": target_private_link_resource_id,
+        "targetSubresourceType": target_subresource_type,
+    }
+
+    if request_message is not None:
+        if len(request_message) > 140:
+            raise ValueError(
+                f"{icons.red_dot} The request message cannot be more than 140 characters."
+            )
+        request_body["requestMessage"] = request_message
+
+    client = fabric.FabricRestClient()
+    response = client.post(
+        f"/v1/workspaces/{workspace_id}/managedPrivateEndpoints", json=request_body
+    )
+
+    lro(client, response, status_codes=[201, 202])
+
+    print(
+        f"{icons.green_dot} The '{name}' managed private endpoint has been created within the '{workspace}' workspace."
+    )
+
+
+def list_managed_private_endpoints(workspace: Optional[str] = None) -> pd.DataFrame:
+    """
+    Shows the managed private endpoints within a workspace.
+
+    This is a wrapper function for the following API: `Managed Private Endpoints - List Workspace Managed Private Endpoints <https://learn.microsoft.com/rest/api/fabric/core/managed-private-endpoints/list-workspace-managed-private-endpoints>`.
+
+    Parameters
+    ----------
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing the managed private endpoints within a workspace.
+    """
+
+    df = pd.DataFrame(
+        columns=[
+            "Managed Private Endpoint Name",
+            "Managed Private Endpoint Id",
+            "Target Private Link Resource Id",
+            "Provisioning State",
+            "Connection Status",
+            "Connection Description",
+            "Target Subresource Type",
+        ]
+    )
+
+    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
+
+    client = fabric.FabricRestClient()
+    response = client.get(f"/v1/workspaces/{workspace_id}/managedPrivateEndpoints")
+    if response.status_code != 200:
+        raise FabricHTTPException(response)
+
+    responses = pagination(client, response)
+
+    for r in responses:
+        for v in r.get("value", []):
+            conn = v.get("connectionState", {})
+            new_data = {
+                "Managed Private Endpoint Name": v.get("name"),
+                "Managed Private Endpoint Id": v.get("id"),
+                "Target Private Link Resource Id": v.get("targetPrivateLinkResourceId"),
+                "Provisioning State": v.get("provisioningState"),
+                "Connection Status": conn.get("status"),
+                "Connection Description": conn.get("description"),
+                "Target Subresource Type": v.get("targetSubresourceType"),
+            }
+            df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
+
+    return df
+
+
+def delete_managed_private_endpoint(
+    managed_private_endpoint: str, workspace: Optional[str] = None
+):
+    """
+    Deletes a Fabric managed private endpoint.
+
+    This is a wrapper function for the following API: `Managed Private Endpoints - Delete Workspace Managed Private Endpoint <https://learn.microsoft.com/rest/api/fabric/core/managed-private-endpoints/delete-workspace-managed-private-endpoint>`.
+
+    Parameters
+    ----------
+    managed_private_endpoint: str
+        Name of the managed private endpoint.
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
+
+    df = list_managed_private_endpoints(workspace=workspace)
+    df_filt = df[df["Managed Private Endpoint Name"] == managed_private_endpoint]
+
+    if len(df_filt) == 0:
+        raise ValueError(
+            f"{icons.red_dot} The '{managed_private_endpoint}' managed private endpoint does not exist within the '{workspace}' workspace."
+        )
+
+    item_id = df_filt["Managed Private Endpoint Id"].iloc[0]
+
+    client = fabric.FabricRestClient()
+    response = client.delete(
+        f"/v1/workspaces/{workspace_id}/managedPrivateEndpoints/{item_id}"
+    )
+
+    if response.status_code != 200:
+        raise FabricHTTPException(response)
+
+    print(
+        f"{icons.green_dot} The '{managed_private_endpoint}' managed private endpoint within the '{workspace}' workspace has been deleted."
+    )

sempy_labs/_mirrored_warehouses.py

@@ -12,6 +12,8 @@ def list_mirrored_warehouses(workspace: Optional[str] = None) -> pd.DataFrame:
     """
     Shows the mirrored warehouses within a workspace.
 
+    This is a wrapper function for the following API: `Items - List Mirrored Warehouses <https://learn.microsoft.com/rest/api/fabric/mirroredwarehouse/items/list-mirrored-warehouses`_.
+
     Parameters
     ----------
     workspace : str, default=None

sempy_labs/_ml_experiments.py

@@ -14,6 +14,8 @@ def list_ml_experiments(workspace: Optional[str] = None) -> pd.DataFrame:
     """
     Shows the ML experiments within a workspace.
 
+    This is a wrapper function for the following API: `Items - List ML Experiments <https://learn.microsoft.com/rest/api/fabric/mlexperiment/items/list-ml-experiments`_.
+
     Parameters
     ----------
     workspace : str, default=None
@@ -60,6 +62,8 @@ def create_ml_experiment(
     """
     Creates a Fabric ML experiment.
 
+    This is a wrapper function for the following API: `Items - Create ML Experiment <https://learn.microsoft.com/rest/api/fabric/mlexperiment/items/create-ml-experiment`_.
+
     Parameters
     ----------
     name: str
@@ -95,6 +99,8 @@ def delete_ml_experiment(name: str, workspace: Optional[str] = None):
     """
     Deletes a Fabric ML experiment.
 
+    This is a wrapper function for the following API: `Items - Delete ML Experiment <https://learn.microsoft.com/rest/api/fabric/mlexperiment/items/delete-ml-experiment`_.
+
     Parameters
     ----------
     name: str

sempy_labs/_ml_models.py

@@ -14,6 +14,8 @@ def list_ml_models(workspace: Optional[str] = None) -> pd.DataFrame:
     """
     Shows the ML models within a workspace.
 
+    This is a wrapper function for the following API: `Items - List ML Models <https://learn.microsoft.com/rest/api/fabric/mlmodel/items/list-ml-models`_.
+
     Parameters
     ----------
     workspace : str, default=None
@@ -60,6 +62,8 @@ def create_ml_model(
     """
     Creates a Fabric ML model.
 
+    This is a wrapper function for the following API: `Items - Create ML Model <https://learn.microsoft.com/rest/api/fabric/mlmodel/items/create-ml-model`_.
+
     Parameters
     ----------
     name: str
@@ -93,6 +97,8 @@ def delete_ml_model(name: str, workspace: Optional[str] = None):
     """
     Deletes a Fabric ML model.
 
+    This is a wrapper function for the following API: `Items - Delete ML Model <https://learn.microsoft.com/rest/api/fabric/mlmodel/items/delete-ml-model`_.
+
     Parameters
     ----------
     name: str