semantic-link-labs 0.8.2__py3-none-any.whl → 0.8.4__py3-none-any.whl

sempy_labs/_refresh_semantic_model.py

@@ -1,11 +1,19 @@
 import sempy.fabric as fabric
 import time
-from sempy_labs._helper_functions import resolve_dataset_id
+from sempy_labs._helper_functions import (
+    resolve_dataset_id,
+    resolve_workspace_name_and_id,
+    _get_partition_map,
+    _process_and_display_chart,
+)
 from typing import Any, List, Optional, Union
 from sempy._utils._log import log
 import sempy_labs._icons as icons
-from sempy_labs._helper_functions import resolve_workspace_name_and_id
 from sempy.fabric.exceptions import FabricHTTPException
+import pandas as pd
+import warnings
+import ipywidgets as widgets
+import json
 
 
 @log
@@ -18,7 +26,8 @@ def refresh_semantic_model(
     apply_refresh_policy: bool = True,
     max_parallelism: int = 10,
     workspace: Optional[str] = None,
-):
+    visualize: bool = False,
+) -> pd.DataFrame | None:
     """
     Refreshes a semantic model.
 
@@ -44,6 +53,13 @@ def refresh_semantic_model(
         The Fabric workspace name.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
+    visualize : bool, default=False
+        If True, displays a Gantt chart showing the refresh statistics for each table/partition.
+
+    Returns
+    -------
+    pandas.DataFrame | None
+        If 'visualize' is set to True, returns a pandas dataframe showing the SSAS trace output used to generate the visualization.
     """
 
     workspace = fabric.resolve_workspace_name(workspace)
@@ -67,65 +83,186 @@ def refresh_semantic_model(
 
         objects = objects + [extract_names(partition) for partition in partitions]
 
-    refresh_type = (
-        refresh_type.lower().replace("only", "Only").replace("values", "Values")
-    )
+    refresh_type = refresh_type.lower()
+    for prefix, mapped_value in icons.refresh_type_mapping.items():
+        if refresh_type.startswith(prefix):
+            refresh_type = mapped_value
+            break
 
-    if refresh_type not in icons.refreshTypes:
+    valid_refresh_types = list(icons.refresh_type_mapping.values())
+    if refresh_type not in valid_refresh_types:
         raise ValueError(
-            f"{icons.red_dot} Invalid refresh type. Refresh type must be one of these values: {icons.refreshTypes}."
+            f"{icons.red_dot} Invalid refresh type. Refresh type must be one of these values: {valid_refresh_types}."
         )
 
-    if len(objects) == 0:
-        requestID = fabric.refresh_dataset(
-            dataset=dataset,
-            workspace=workspace,
-            refresh_type=refresh_type,
-            retry_count=retry_count,
-            apply_refresh_policy=apply_refresh_policy,
-            max_parallelism=max_parallelism,
+    def refresh_and_trace_dataset(
+        dataset,
+        workspace,
+        refresh_type,
+        retry_count,
+        apply_refresh_policy,
+        max_parallelism,
+        objects,
+        visualize,
+    ):
+        # Ignore specific warnings
+        warnings.filterwarnings(
+            "ignore",
+            message="No trace logs have been recorded. Try starting the trace with a larger 'delay'",
         )
-    else:
-        requestID = fabric.refresh_dataset(
-            dataset=dataset,
-            workspace=workspace,
-            refresh_type=refresh_type,
-            retry_count=retry_count,
-            apply_refresh_policy=apply_refresh_policy,
-            max_parallelism=max_parallelism,
-            objects=objects,
-        )
-    print(
-        f"{icons.in_progress} Refresh of the '{dataset}' semantic model within the '{workspace}' workspace is in progress..."
-    )
-    if len(objects) != 0:
-        print(objects)
 
-    while True:
-        requestDetails = fabric.get_refresh_execution_details(
-            dataset=dataset, refresh_request_id=requestID, workspace=workspace
-        )
-        status = requestDetails.status
+        def extract_failure_error():
+            error_messages = []
+            combined_messages = ""
+            final_message = f"{icons.red_dot} The refresh of the '{dataset}' semantic model within the '{workspace}' workspace has failed."
+            for _, r in fabric.get_refresh_execution_details(
+                refresh_request_id=request_id,
+                dataset=dataset,
+                workspace=workspace,
+            ).messages.iterrows():
+                error_messages.append(f"{r['Type']}: {r['Message']}")
 
-        # Check if the refresh has completed
-        if status == "Completed":
-            break
-        elif status == "Failed":
-            raise ValueError(
-                f"{icons.red_dot} The refresh of the '{dataset}' semantic model within the '{workspace}' workspace has failed."
+            if error_messages:
+                combined_messages = "\n".join(error_messages)
+            final_message += f"'\n' {combined_messages}"
+
+            return final_message
+
+        # Function to perform dataset refresh
+        def refresh_dataset():
+            return fabric.refresh_dataset(
+                dataset=dataset,
+                workspace=workspace,
+                refresh_type=refresh_type,
+                retry_count=retry_count,
+                apply_refresh_policy=apply_refresh_policy,
+                max_parallelism=max_parallelism,
+                objects=objects if objects else None,
             )
-        elif status == "Cancelled":
-            print(
-                f"{icons.yellow_dot} The refresh of the '{dataset}' semantic model within the '{workspace}' workspace has been cancelled."
+
+        def check_refresh_status(request_id):
+            request_details = fabric.get_refresh_execution_details(
+                dataset=dataset, refresh_request_id=request_id, workspace=workspace
             )
-            return
+            return request_details.status
 
-        time.sleep(3)
+        def display_trace_logs(trace, partition_map, widget, title, stop=False):
+            if stop:
+                df = trace.stop()
+            else:
+                df = trace.get_trace_logs()
+            if not df.empty:
+                df = df[
+                    df["Event Subclass"].isin(["ExecuteSql", "Process"])
+                ].reset_index(drop=True)
+                df = pd.merge(
+                    df,
+                    partition_map[
+                        ["PartitionID", "Object Name", "TableName", "PartitionName"]
+                    ],
+                    left_on="Object ID",
+                    right_on="PartitionID",
+                    how="left",
+                )
+                _process_and_display_chart(df, title=title, widget=widget)
+                if stop:
+                    df.drop(["Object Name", "PartitionID"], axis=1, inplace=True)
+                    df.rename(columns={"TableName": "Table Name"}, inplace=True)
+                    df.rename(columns={"PartitionName": "Partition Name"}, inplace=True)
+                    return df
 
-    print(
-        f"{icons.green_dot} Refresh of the '{dataset}' semantic model within the '{workspace}' workspace is complete."
+        # Start the refresh process
+        if not visualize:
+            request_id = refresh_dataset()
+        print(
+            f"{icons.in_progress} Refresh of the '{dataset}' semantic model within the '{workspace}' workspace is in progress..."
+        )
+
+        # Monitor refresh progress and handle tracing if visualize is enabled
+        if visualize:
+            partition_map = _get_partition_map(dataset, workspace)
+            widget = widgets.Output()
+
+            with fabric.create_trace_connection(
+                dataset=dataset, workspace=workspace
+            ) as trace_connection:
+                with trace_connection.create_trace(icons.refresh_event_schema) as trace:
+                    trace.start()
+                    request_id = refresh_dataset()
+
+                    while True:
+                        status = check_refresh_status(request_id)
+                        # Check if the refresh has completed
+                        if status == "Completed":
+                            break
+                        elif status == "Failed":
+                            raise ValueError(extract_failure_error())
+                        elif status == "Cancelled":
+                            print(
+                                f"{icons.yellow_dot} The refresh of the '{dataset}' semantic model within the '{workspace}' workspace has been cancelled."
+                            )
+                            return
+
+                        # Capture and display logs in real-time
+                        display_trace_logs(
+                            trace,
+                            partition_map,
+                            widget,
+                            title="Refresh in progress...",
+                        )
+
+                        time.sleep(3)  # Wait before the next check
+
+                    # Final log display after completion
+                    time.sleep(5)
+
+                    # Stop trace and display final chart
+                    final_df = display_trace_logs(
+                        trace,
+                        partition_map,
+                        widget,
+                        title="Refresh Completed",
+                        stop=True,
+                    )
+
+                    print(
+                        f"{icons.green_dot} Refresh of the '{dataset}' semantic model within the '{workspace}' workspace is complete."
+                    )
+                    return final_df
+
+        # For non-visualize case, only check refresh status
+        else:
+            while True:
+                status = check_refresh_status(request_id)
+                if status == "Completed":
+                    break
+                elif status == "Failed":
+                    raise ValueError(extract_failure_error())
+                elif status == "Cancelled":
+                    print(
+                        f"{icons.yellow_dot} The refresh of the '{dataset}' semantic model within the '{workspace}' workspace has been cancelled."
+                    )
+                    return
+
+                time.sleep(3)
+
+            print(
+                f"{icons.green_dot} Refresh of the '{dataset}' semantic model within the '{workspace}' workspace is complete."
+            )
+
+    final_output = refresh_and_trace_dataset(
+        dataset=dataset,
+        workspace=workspace,
+        refresh_type=refresh_type,
+        retry_count=retry_count,
+        apply_refresh_policy=apply_refresh_policy,
+        max_parallelism=max_parallelism,
+        objects=objects,
+        visualize=visualize,
     )
 
+    return final_output
+
 
 @log
 def cancel_dataset_refresh(
@@ -173,3 +310,116 @@ def cancel_dataset_refresh(
     print(
         f"{icons.green_dot} The '{request_id}' refresh request for the '{dataset}' semantic model within the '{workspace}' workspace has been cancelled."
     )
+
+
+def get_semantic_model_refresh_history(
+    dataset: str, request_id: Optional[str] = None, workspace: Optional[str] = None
+) -> pd.DataFrame:
+    """
+    Obtains the semantic model refresh history (refreshes executed via the Enhanced Refresh API).
+
+    This is a wrapper function for the following API: `Datasets - Get Refresh History In Group <https://learn.microsoft.com/rest/api/power-bi/datasets/get-refresh-history-in-group>`_.
+
+    Parameters
+    ----------
+    dataset : str
+        Name of the semantic model.
+    request_id : str, default=None
+        The request id of a semantic model refresh.
+        Defaults to None which resolves to showing all refresh requests for the given semantic model.
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing the semantic model refresh history.
+    """
+
+    workspace_name = fabric.resolve_workspace_name(workspace)
+    workspace_id = fabric.resolve_workspace_id(workspace_name)
+    df = pd.DataFrame(
+        columns=[
+            "Request Id",
+            "Refresh Type",
+            "Start Time",
+            "End Time",
+            "Status",
+            "Extended Status",
+        ]
+    )
+
+    dataset_id = fabric.resolve_item_id(
+        item_name=dataset, workspace=workspace_id, type="SemanticModel"
+    )
+    client = fabric.PowerBIRestClient()
+    response = client.get(
+        f"/v1.0/myorg/groups/{workspace_id}/datasets/{dataset_id}/refreshes"
+    )
+    data = []
+
+    for i in response.json().get("value", []):
+        error = i.get("serviceExceptionJson")
+        if error:
+            error_json = json.loads(error)
+        if request_id is None:
+            new_data = {
+                "Request Id": i.get("requestId"),
+                "Refresh Type": i.get("refreshType"),
+                "Start Time": i.get("startTime"),
+                "End Time": i.get("endTime"),
+                "Error Code": error_json.get("errorCode") if error else None,
+                "Error Description": (
+                    error_json.get("errorDescription") if error else None
+                ),
+                "Status": i.get("status"),
+                "Extended Status": i.get("extendedStatus"),
+                "Attempts": i.get("refreshAttempts"),
+            }
+            data.append(new_data)
+
+        elif request_id == i.get("requestId"):
+            for attempt in i.get("refreshAttempts", []):
+                attempt_error = attempt.get("serviceExceptionJson")
+                if attempt_error:
+                    attempt_error_json = json.loads(attempt_error)
+                new_data = {
+                    "Request Id": i.get("requestId"),
+                    "Refresh Type": i.get("refreshType"),
+                    "Start Time": i.get("startTime"),
+                    "End Time": i.get("endTime"),
+                    "Error Code": error_json.get("errorCode") if error else None,
+                    "Error Description": (
+                        error_json.get("errorDescription") if error else None
+                    ),
+                    "Status": i.get("status"),
+                    "Extended Status": i.get("extendedStatus"),
+                    "Attempt Id": attempt.get("attemptId"),
+                    "Attempt Start Time": attempt.get("startTime"),
+                    "Attempt End Time": attempt.get("endTime"),
+                    "Attempt Error Code": (
+                        attempt_error_json.get("errorCode") if attempt_error else None
+                    ),
+                    "Attempt Error Description": (
+                        attempt_error_json.get("errorDescription")
+                        if attempt_error
+                        else None
+                    ),
+                    "Type": attempt.get("type"),
+                }
+                data.append(new_data)
+
+    if data:
+        df = pd.DataFrame(data)
+
+    # date_cols = ["Start Time", "End Time"]
+    # df[date_cols] = df[date_cols].apply(pd.to_datetime)
+
+    if "Attempt Id" in df.columns:
+        df["Attempt Id"] = df["Attempt Id"].astype(int)
+        # date_cols = ["Attempt Start Time", "Attempt End Time"]
+        # df[date_cols] = df[date_cols].apply(pd.to_datetime)
+
+    return df

sempy_labs/_spark.py

@@ -12,6 +12,8 @@ def list_custom_pools(workspace: Optional[str] = None) -> pd.DataFrame:
     """
     Lists all `custom pools <https://learn.microsoft.com/fabric/data-engineering/create-custom-spark-pools>`_ within a workspace.
 
+    This is a wrapper function for the following API: `Custom Pools - List Workspace Custom Pools <https://learn.microsoft.com/rest/api/fabric/spark/custom-pools/list-workspace-custom-pools`_.
+
     Parameters
     ----------
     workspace : str, default=None
@@ -25,7 +27,6 @@ def list_custom_pools(workspace: Optional[str] = None) -> pd.DataFrame:
         A pandas dataframe showing all the custom pools within the Fabric workspace.
     """
 
-    # https://learn.microsoft.com/rest/api/fabric/spark/custom-pools/list-workspace-custom-pools
     (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
 
     df = pd.DataFrame(
@@ -99,6 +100,8 @@ def create_custom_pool(
     """
     Creates a `custom pool <https://learn.microsoft.com/fabric/data-engineering/create-custom-spark-pools>`_ within a workspace.
 
+    This is a wrapper function for the following API: `Custom Pools - Create Workspace Custom Pool <https://learn.microsoft.com/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool`_.
+
     Parameters
     ----------
     pool_name : str
@@ -125,7 +128,6 @@ def create_custom_pool(
         or if no lakehouse attached, resolves to the workspace of the notebook.
     """
 
-    # https://learn.microsoft.com/en-us/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool
     (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
 
     request_body = {
@@ -171,6 +173,8 @@ def update_custom_pool(
     """
     Updates the properties of a `custom pool <https://learn.microsoft.com/fabric/data-engineering/create-custom-spark-pools>`_ within a workspace.
 
+    This is a wrapper function for the following API: `Custom Pools - Update Workspace Custom Pool <https://learn.microsoft.com/rest/api/fabric/spark/custom-pools/update-workspace-custom-pool`_.
+
     Parameters
     ----------
     pool_name : str
@@ -205,7 +209,6 @@ def update_custom_pool(
         or if no lakehouse attached, resolves to the workspace of the notebook.
     """
 
-    # https://learn.microsoft.com/en-us/rest/api/fabric/spark/custom-pools/update-workspace-custom-pool?tabs=HTTP
     (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
 
     df = list_custom_pools(workspace=workspace)
@@ -267,6 +270,8 @@ def delete_custom_pool(pool_name: str, workspace: Optional[str] = None):
     """
     Deletes a `custom pool <https://learn.microsoft.com/fabric/data-engineering/create-custom-spark-pools>`_ within a workspace.
 
+    This is a wrapper function for the following API: `Custom Pools - Delete Workspace Custom Pool <https://learn.microsoft.com/rest/api/fabric/spark/custom-pools/delete-workspace-custom-pool`_.
+
     Parameters
     ----------
     pool_name : str
@@ -304,6 +309,8 @@ def get_spark_settings(
     """
     Shows the spark settings for a workspace.
 
+    This is a wrapper function for the following API: `Workspace Settings - Get Spark Settings <https://learn.microsoft.com/rest/api/fabric/spark/workspace-settings/get-spark-settings`_.
+
     Parameters
     ----------
     workspace : str, default=None
@@ -319,7 +326,6 @@ def get_spark_settings(
         A pandas dataframe showing the spark settings for a workspace.
     """
 
-    # https://learn.microsoft.com/en-us/rest/api/fabric/spark/workspace-settings/get-spark-settings?tabs=HTTP
     (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
 
     df = pd.DataFrame(
@@ -392,6 +398,8 @@ def update_spark_settings(
     """
     Updates the spark settings for a workspace.
 
+    This is a wrapper function for the following API: `Workspace Settings - Update Spark Settings <https://learn.microsoft.com/rest/api/fabric/spark/workspace-settings/update-spark-settings`_.
+
     Parameters
     ----------
     automatic_log_enabled : bool, default=None
@@ -424,7 +432,6 @@ def update_spark_settings(
         or if no lakehouse attached, resolves to the workspace of the notebook.
     """
 
-    # https://learn.microsoft.com/en-us/rest/api/fabric/spark/workspace-settings/update-spark-settings?tabs=HTTP
     (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
 
     request_body = get_spark_settings(workspace=workspace, return_dataframe=False)

sempy_labs/_sql.py

@@ -9,7 +9,7 @@ from sempy.fabric.exceptions import FabricHTTPException
 from sempy_labs._helper_functions import resolve_warehouse_id, resolve_lakehouse_id
 
 
-def bytes2mswin_bstr(value: bytes) -> bytes:
+def _bytes2mswin_bstr(value: bytes) -> bytes:
     """Convert a sequence of bytes into a (MS-Windows) BSTR (as bytes).
 
     See https://github.com/mkleehammer/pyodbc/issues/228#issuecomment-319190980
@@ -68,7 +68,7 @@ class ConnectBase:
 
         # Set up the connection string
         access_token = SynapseTokenProvider()()
-        tokenstruct = bytes2mswin_bstr(access_token.encode())
+        tokenstruct = _bytes2mswin_bstr(access_token.encode())
         conn_str = f"DRIVER={{ODBC Driver 18 for SQL Server}};SERVER={tds_endpoint};DATABASE={name};Encrypt=Yes;"
 
         if timeout is not None:

sempy_labs/_translations.py

@@ -40,7 +40,7 @@ def translate_semantic_model(
     from pyspark.sql import SparkSession
     from sempy_labs.tom import connect_semantic_model
 
-    def clean_text(text, exclude_chars):
+    def _clean_text(text, exclude_chars):
         if exclude_chars:
             for char in exclude_chars:
                 text = text.replace(char, " ")
@@ -55,13 +55,15 @@ def translate_semantic_model(
         columns=["Object Type", "Name", "Description", "Display Folder"]
     )
 
+    icons.sll_tags.append("TranslateSemanticModel")
+
     with connect_semantic_model(
         dataset=dataset, readonly=False, workspace=workspace
     ) as tom:
 
         for o in tom.model.Tables:
-            oName = clean_text(o.Name, exclude_characters)
-            oDescription = clean_text(o.Description, exclude_characters)
+            oName = _clean_text(o.Name, exclude_characters)
+            oDescription = _clean_text(o.Description, exclude_characters)
             new_data = {
                 "Object Type": "Table",
                 "Name": o.Name,
@@ -75,9 +77,9 @@ def translate_semantic_model(
                 [df_prep, pd.DataFrame(new_data, index=[0])], ignore_index=True
             )
         for o in tom.all_columns():
-            oName = clean_text(o.Name, exclude_characters)
-            oDescription = clean_text(o.Description, exclude_characters)
-            oDisplayFolder = clean_text(o.DisplayFolder, exclude_characters)
+            oName = _clean_text(o.Name, exclude_characters)
+            oDescription = _clean_text(o.Description, exclude_characters)
+            oDisplayFolder = _clean_text(o.DisplayFolder, exclude_characters)
             new_data = {
                 "Object Type": "Column",
                 "Name": o.Name,
@@ -91,9 +93,9 @@ def translate_semantic_model(
                 [df_prep, pd.DataFrame(new_data, index=[0])], ignore_index=True
             )
         for o in tom.all_measures():
-            oName = clean_text(o.Name, exclude_characters)
-            oDescription = clean_text(o.Description, exclude_characters)
-            oDisplayFolder = clean_text(o.DisplayFolder, exclude_characters)
+            oName = _clean_text(o.Name, exclude_characters)
+            oDescription = _clean_text(o.Description, exclude_characters)
+            oDisplayFolder = _clean_text(o.DisplayFolder, exclude_characters)
             new_data = {
                 "Object Type": "Measure",
                 "Name": o.Name,
@@ -107,9 +109,9 @@ def translate_semantic_model(
                 [df_prep, pd.DataFrame(new_data, index=[0])], ignore_index=True
             )
         for o in tom.all_hierarchies():
-            oName = clean_text(o.Name, exclude_characters)
-            oDescription = clean_text(o.Description, exclude_characters)
-            oDisplayFolder = clean_text(o.DisplayFolder, exclude_characters)
+            oName = _clean_text(o.Name, exclude_characters)
+            oDescription = _clean_text(o.Description, exclude_characters)
+            oDisplayFolder = _clean_text(o.DisplayFolder, exclude_characters)
             new_data = {
                 "Object Type": "Hierarchy",
                 "Name": o.Name,
@@ -123,8 +125,8 @@ def translate_semantic_model(
                 [df_prep, pd.DataFrame(new_data, index=[0])], ignore_index=True
             )
         for o in tom.all_levels():
-            oName = clean_text(o.Name, exclude_characters)
-            oDescription = clean_text(o.Description, exclude_characters)
+            oName = _clean_text(o.Name, exclude_characters)
+            oDescription = _clean_text(o.Description, exclude_characters)
             new_data = {
                 "Object Type": "Level",
                 "Name": o.Name,