semantic-link-labs 0.6.0__py3-none-any.whl → 0.7.1__py3-none-any.whl

sempy_labs/_clear_cache.py

@@ -1,5 +1,5 @@
 import sempy.fabric as fabric
-from ._helper_functions import resolve_dataset_id
+from ._helper_functions import resolve_dataset_id, is_default_semantic_model
 from typing import Optional
 import sempy_labs._icons as icons
 
@@ -20,13 +20,18 @@ def clear_cache(dataset: str, workspace: Optional[str] = None):
     """
 
     workspace = fabric.resolve_workspace_name(workspace)
+    if is_default_semantic_model(dataset=dataset, workspace=workspace):
+        raise ValueError(
+            f"{icons.red_dot} Cannot run XMLA operations against a default semantic model. Please choose a different semantic model. "
+            "See here for more information: https://learn.microsoft.com/fabric/data-warehouse/semantic-models"
+        )
 
-    datasetID = resolve_dataset_id(dataset=dataset, workspace=workspace)
+    dataset_id = resolve_dataset_id(dataset=dataset, workspace=workspace)
 
     xmla = f"""
             <ClearCache xmlns="http://schemas.microsoft.com/analysisservices/2003/engine">
-                <Object>  
-                    <DatabaseID>{datasetID}</DatabaseID>
+                <Object>
+                    <DatabaseID>{dataset_id}</DatabaseID>
                 </Object>
             </ClearCache>
             """

sempy_labs/_generate_semantic_model.py

@@ -1,17 +1,18 @@
 import sempy.fabric as fabric
 import pandas as pd
 import json
-import base64
-import time
 import os
 from typing import Optional
 from sempy_labs._helper_functions import (
     resolve_lakehouse_name,
     resolve_workspace_name_and_id,
+    resolve_dataset_id,
+    _conv_b64,
+    _decode_b64,
+    lro,
 )
 from sempy_labs.lakehouse._lakehouse import lakehouse_attached
 import sempy_labs._icons as icons
-from sempy.fabric.exceptions import FabricHTTPException
 
 
 def create_blank_semantic_model(
@@ -34,8 +35,7 @@ def create_blank_semantic_model(
         or if no lakehouse attached, resolves to the workspace of the notebook.
     """
 
-    if workspace is None:
-        workspace = fabric.resolve_workspace_name()
+    workspace = fabric.resolve_workspace_name(workspace)
 
     min_compat = 1500
 
@@ -89,9 +89,7 @@ def create_semantic_model_from_bim(
 
     (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
 
-    objectType = "SemanticModel"
-
-    dfI = fabric.list_items(workspace=workspace, type=objectType)
+    dfI = fabric.list_items(workspace=workspace, type="SemanticModel")
     dfI_filt = dfI[(dfI["Display Name"] == dataset)]
 
     if len(dfI_filt) > 0:
@@ -102,19 +100,11 @@ def create_semantic_model_from_bim(
     client = fabric.FabricRestClient()
     defPBIDataset = {"version": "1.0", "settings": {}}
 
-    def conv_b64(file):
-
-        loadJson = json.dumps(file)
-        f = base64.b64encode(loadJson.encode("utf-8")).decode("utf-8")
-
-        return f
-
-    payloadPBIDefinition = conv_b64(defPBIDataset)
-    payloadBim = conv_b64(bim_file)
+    payloadPBIDefinition = _conv_b64(defPBIDataset)
+    payloadBim = _conv_b64(bim_file)
 
     request_body = {
         "displayName": dataset,
-        "type": objectType,
         "definition": {
             "parts": [
                 {
@@ -131,26 +121,16 @@ def create_semantic_model_from_bim(
         },
     }
 
-    response = client.post(f"/v1/workspaces/{workspace_id}/items", json=request_body)
+    response = client.post(
+        f"/v1/workspaces/{workspace_id}/semanticModels",
+        json=request_body,
+    )
 
-    if response.status_code == 201:
-        print(
-            f"{icons.green_dot} The '{dataset}' semantic model has been created within the '{workspace}' workspace."
-        )
-        print(response.json())
-    elif response.status_code == 202:
-        operationId = response.headers["x-ms-operation-id"]
-        response = client.get(f"/v1/operations/{operationId}")
-        response_body = json.loads(response.content)
-        while response_body["status"] != "Succeeded":
-            time.sleep(3)
-            response = client.get(f"/v1/operations/{operationId}")
-            response_body = json.loads(response.content)
-        response = client.get(f"/v1/operations/{operationId}/result")
-        print(
-            f"{icons.green_dot} The '{dataset}' semantic model has been created within the '{workspace}' workspace."
-        )
-        print(response.json())
+    lro(client, response, status_codes=[201, 202])
+
+    print(
+        f"{icons.green_dot} The '{dataset}' semantic model has been created within the '{workspace}' workspace."
+    )
 
 
 def deploy_semantic_model(
@@ -196,11 +176,10 @@ def deploy_semantic_model(
         target_dataset = source_dataset
 
     if target_dataset == source_dataset and target_workspace == source_workspace:
-        print(
+        raise ValueError(
             f"{icons.red_dot} The 'dataset' and 'new_dataset' parameters have the same value. And, the 'workspace' and 'new_dataset_workspace' "
             f"parameters have the same value. At least one of these must be different. Please update the parameters."
         )
-        return
 
     bim = get_semantic_model_bim(dataset=source_dataset, workspace=source_workspace)
 
@@ -217,7 +196,7 @@ def get_semantic_model_bim(
     workspace: Optional[str] = None,
     save_to_file_name: Optional[str] = None,
     lakehouse_workspace: Optional[str] = None,
-):
+) -> dict:
     """
     Extracts the Model.bim file for a given semantic model.
 
@@ -238,48 +217,43 @@ def get_semantic_model_bim(
 
     Returns
     -------
-    str
+    dict
         The Model.bim file for the semantic model.
     """
 
     (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
-    if lakehouse_workspace is None:
-        lakehouse_workspace = workspace
 
     fmt = "TMSL"
     client = fabric.FabricRestClient()
-    itemList = fabric.list_items(workspace=workspace, type="SemanticModel")
-    itemListFilt = itemList[(itemList["Display Name"] == dataset)]
-    itemId = itemListFilt["Id"].iloc[0]
+    dataset_id = resolve_dataset_id(dataset=dataset, workspace=workspace)
     response = client.post(
-        f"/v1/workspaces/{workspace_id}/items/{itemId}/getDefinition?format={fmt}",
-        lro_wait=True,
+        f"/v1/workspaces/{workspace_id}/semanticModels/{dataset_id}/getDefinition?format={fmt}",
     )
-
-    df_items = pd.json_normalize(response.json()["definition"]["parts"])
+    result = lro(client, response).json()
+    df_items = pd.json_normalize(result["definition"]["parts"])
     df_items_filt = df_items[df_items["path"] == "model.bim"]
     payload = df_items_filt["payload"].iloc[0]
-    bimFile = base64.b64decode(payload).decode("utf-8")
+    bimFile = _decode_b64(payload)
     bimJson = json.loads(bimFile)
 
     if save_to_file_name is not None:
-        lakeAttach = lakehouse_attached()
-        if lakeAttach is False:
+        if not lakehouse_attached():
             raise ValueError(
                 f"{icons.red_dot} In order to save the model.bim file, a lakehouse must be attached to the notebook. Please attach a lakehouse to this notebook."
             )
 
         lakehouse_id = fabric.get_lakehouse_id()
-        lakehouse = resolve_lakehouse_name(lakehouse_id, lakehouse_workspace)
+        lake_workspace = fabric.resolve_workspace_name()
+        lakehouse = resolve_lakehouse_name(lakehouse_id, lake_workspace)
         folderPath = "/lakehouse/default/Files"
         fileExt = ".bim"
         if not save_to_file_name.endswith(fileExt):
-            save_to_file_name = save_to_file_name + fileExt
+            save_to_file_name = f"{save_to_file_name}{fileExt}"
         filePath = os.path.join(folderPath, save_to_file_name)
         with open(filePath, "w") as json_file:
             json.dump(bimJson, json_file, indent=4)
         print(
-            f"The .bim file for the '{dataset}' semantic model has been saved to the '{lakehouse}' in this location: '{filePath}'.\n\n"
+            f"{icons.green_dot} The .bim file for the '{dataset}' semantic model has been saved to the '{lakehouse}' in this location: '{filePath}'.\n\n"
         )
 
     return bimJson

sempy_labs/_helper_functions.py

@@ -1,10 +1,17 @@
 import sempy.fabric as fabric
 import re
+import json
+import base64
 import pandas as pd
+from functools import wraps
+import datetime
+import time
 from pyspark.sql import SparkSession
-from typing import Optional, Tuple
+from typing import Optional, Tuple, List
 from uuid import UUID
 import sempy_labs._icons as icons
+from sempy.fabric.exceptions import FabricHTTPException
+import urllib.parse
 
 
 def create_abfss_path(
@@ -284,26 +291,34 @@ def get_direct_lake_sql_endpoint(dataset: str, workspace: Optional[str] = None)
         The ID of SQL Endpoint.
     """
 
+    from sempy_labs.tom import connect_semantic_model
+
     if workspace is None:
         workspace_id = fabric.get_workspace_id()
         workspace = fabric.resolve_workspace_name(workspace_id)
 
-    dfP = fabric.list_partitions(dataset=dataset, workspace=workspace)
-    dfP_filt = dfP[dfP["Mode"] == "DirectLake"]
+    # dfP = fabric.list_partitions(dataset=dataset, workspace=workspace)
+    # dfP_filt = dfP[dfP["Mode"] == "DirectLake"]
 
-    if len(dfP_filt) == 0:
-        raise ValueError(
-            f"The '{dataset}' semantic model in the '{workspace}' workspace is not in Direct Lake mode."
-        )
+    # if len(dfP_filt) == 0:
+    #    raise ValueError(
+    #        f"The '{dataset}' semantic model in the '{workspace}' workspace is not in Direct Lake mode."
+    #    )
 
-    dfE = fabric.list_expressions(dataset=dataset, workspace=workspace)
-    dfE_filt = dfE[dfE["Name"] == "DatabaseQuery"]
-    expr = dfE_filt["Expression"].iloc[0]
+    with connect_semantic_model(
+        dataset=dataset, readonly=True, workspace=workspace
+    ) as tom:
+        sqlEndpointId = None
+        for e in tom.model.Expressions:
+            if e.Name == "DatabaseQuery":
+                expr = e.Expression
+                matches = re.findall(r'"([^"]+)"', expr)
+                sqlEndpointId = matches[1]
 
-    matches = re.findall(r'"([^"]*)"', expr)
-    sqlEndpointId = matches[1]
+        if sqlEndpointId is None:
+            raise ValueError("SQL Endpoint not found.")
 
-    return sqlEndpointId
+        return sqlEndpointId
 
 
 def generate_embedded_filter(filter: str):
@@ -375,6 +390,7 @@ def save_as_delta_table(
     dataframe,
     delta_table_name: str,
     write_mode: str,
+    merge_schema: Optional[bool] = False,
     lakehouse: Optional[str] = None,
     workspace: Optional[str] = None,
 ):
@@ -389,6 +405,8 @@ def save_as_delta_table(
         The name of the delta table.
     write_mode : str
         The write mode for the save operation. Options: 'append', 'overwrite'.
+    merge_schema : bool, default=False
+        Merges the schemas of the dataframe to the delta table.
     lakehouse : str, default=None
         The Fabric lakehouse used by the Direct Lake semantic model.
         Defaults to None which resolves to the lakehouse attached to the notebook.
@@ -440,7 +458,13 @@ def save_as_delta_table(
         lakehouse_workspace_id=workspace_id,
         delta_table_name=delta_table_name,
     )
-    spark_df.write.mode(write_mode).format("delta").save(filePath)
+
+    if merge_schema:
+        spark_df.write.mode(write_mode).format("delta").option(
+            "mergeSchema", "true"
+        ).save(filePath)
+    else:
+        spark_df.write.mode(write_mode).format("delta").save(filePath)
     print(
         f"{icons.green_dot} The dataframe has been saved as the '{delta_table_name}' table in the '{lakehouse}' lakehouse within the '{workspace}' workspace."
     )
@@ -506,3 +530,326 @@ def resolve_workspace_name_and_id(workspace: Optional[str] = None) -> Tuple[str,
         workspace_id = fabric.resolve_workspace_id(workspace)
 
     return str(workspace), str(workspace_id)
+
+
+def _extract_json(dataframe: pd.DataFrame) -> dict:
+
+    payload = dataframe["payload"].iloc[0]
+    json_file = _decode_b64(payload)
+
+    return json.loads(json_file)
+
+
+def _conv_b64(file):
+
+    loadJson = json.dumps(file)
+    f = base64.b64encode(loadJson.encode("utf-8")).decode("utf-8")
+
+    return f
+
+
+def _decode_b64(file, format: Optional[str] = "utf-8"):
+
+    result = base64.b64decode(file).decode(format)
+
+    return result
+
+
+def is_default_semantic_model(dataset: str, workspace: Optional[str] = None) -> bool:
+    """
+    Identifies whether a semantic model is a default semantic model.
+
+    Parameters
+    ----------
+    dataset : str
+        The name of the semantic model.
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    bool
+        A True/False value indicating whether the semantic model is a default semantic model.
+    """
+
+    workspace = fabric.resolve_workspace_name(workspace)
+
+    dfI = fabric.list_items(workspace=workspace)
+    filtered_df = dfI.groupby("Display Name").filter(
+        lambda x: set(["Warehouse", "SemanticModel"]).issubset(set(x["Type"]))
+        or set(["Lakehouse", "SemanticModel"]).issubset(set(x["Type"]))
+    )
+    default_semantic_models = filtered_df["Display Name"].unique().tolist()
+
+    return dataset in default_semantic_models
+
+
+def resolve_item_type(item_id: UUID, workspace: Optional[str] = None) -> str:
+    """
+    Obtains the item type for a given Fabric Item Id within a Fabric workspace.
+
+    Parameters
+    ----------
+    item_id : UUID
+        The item/artifact Id.
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    str
+        The item type for the item Id.
+    """
+
+    workspace = fabric.resolve_workspace_name(workspace)
+    dfI = fabric.list_items(workspace=workspace)
+    dfI_filt = dfI[dfI["Id"] == item_id]
+
+    if len(dfI_filt) == 0:
+        raise ValueError(
+            f"Invalid 'item_id' parameter. The '{item_id}' item was not found in the '{workspace}' workspace."
+        )
+    item_type = dfI_filt["Type"].iloc[0]
+
+    return item_type
+
+
+def resolve_dataset_from_report(
+    report: str, workspace: Optional[str] = None
+) -> Tuple[UUID, str, UUID, str]:
+    """
+    Obtains the basic semantic model properties from which the report's data is sourced.
+
+    Parameters
+    ----------
+    report : str
+        The name of the Power BI report.
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    Tuple[UUID, str, UUID, str]
+        The semantic model UUID, semantic model name, semantic model workspace UUID, semantic model workspace name
+    """
+
+    workspace = fabric.resolve_workspace_name(workspace)
+
+    dfR = fabric.list_reports(workspace=workspace)
+    dfR_filt = dfR[dfR["Name"] == report]
+    if len(dfR_filt) == 0:
+        raise ValueError(
+            f"{icons.red_dot} The '{report}' report does not exist within the '{workspace}' workspace."
+        )
+    dataset_id = dfR_filt["Dataset Id"].iloc[0]
+    dataset_workspace_id = dfR_filt["Dataset Workspace Id"].iloc[0]
+    dataset_workspace = fabric.resolve_workspace_name(dataset_workspace_id)
+    dataset_name = resolve_dataset_name(
+        dataset_id=dataset_id, workspace=dataset_workspace
+    )
+
+    return dataset_id, dataset_name, dataset_workspace_id, dataset_workspace
+
+
+def _add_part(target_dict, path, payload):
+
+    part = {"path": path, "payload": payload, "payloadType": "InlineBase64"}
+
+    target_dict["definition"]["parts"].append(part)
+
+
+def resolve_workspace_capacity(workspace: Optional[str] = None) -> Tuple[UUID, str]:
+    """
+    Obtains the capacity Id and capacity name for a given workspace.
+
+    Parameters
+    ----------
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    Tuple[UUID, str]
+        capacity Id; capacity came.
+    """
+
+    workspace = fabric.resolve_workspace_name(workspace)
+    filter_condition = urllib.parse.quote(workspace)
+    dfW = fabric.list_workspaces(filter=f"name eq '{filter_condition}'")
+    capacity_id = dfW["Capacity Id"].iloc[0]
+    dfC = fabric.list_capacities()
+    dfC_filt = dfC[dfC["Id"] == capacity_id]
+    if len(dfC_filt) == 1:
+        capacity_name = dfC_filt["Display Name"].iloc[0]
+    else:
+        capacity_name = None
+
+    return capacity_id, capacity_name
+
+
+def get_capacity_id(workspace: Optional[str] = None) -> UUID:
+    """
+    Obtains the Capacity Id for a given workspace.
+
+    Parameters
+    ----------
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    UUID
+        The capacity Id.
+    """
+
+    workspace = fabric.resolve_workspace_name(workspace)
+    filter_condition = urllib.parse.quote(workspace)
+    dfW = fabric.list_workspaces(filter=f"name eq '{filter_condition}'")
+    if len(dfW) == 0:
+        raise ValueError(f"{icons.red_dot} The '{workspace}' does not exist'.")
+
+    return dfW["Capacity Id"].iloc[0]
+
+
+def get_capacity_name(workspace: Optional[str] = None) -> str:
+    """
+    Obtains the capacity name for a given workspace.
+
+    Parameters
+    ----------
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    str
+        The capacity name.
+    """
+
+    capacity_id = get_capacity_id(workspace)
+    dfC = fabric.list_capacities()
+    dfC_filt = dfC[dfC["Id"] == capacity_id]
+    if len(dfC_filt) == 0:
+        raise ValueError(
+            f"{icons.red_dot} The '{capacity_id}' capacity Id does not exist."
+        )
+
+    return dfC_filt["Display Name"].iloc[0]
+
+
+def resolve_capacity_name(capacity_id: Optional[UUID] = None) -> str:
+    """
+    Obtains the capacity name for a given capacity Id.
+
+    Parameters
+    ----------
+    capacity_id : UUID, default=None
+        The capacity Id.
+        Defaults to None which resolves to the capacity name of the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the capacity name of the workspace of the notebook.
+
+    Returns
+    -------
+    str
+        The capacity name.
+    """
+
+    if capacity_id is None:
+        return get_capacity_name()
+
+    dfC = fabric.list_capacities()
+    dfC_filt = dfC[dfC["Id"] == capacity_id]
+
+    if len(dfC_filt) == 0:
+        raise ValueError(
+            f"{icons.red_dot} The '{capacity_id}' capacity Id does not exist."
+        )
+
+    return dfC_filt["Display Name"].iloc[0]
+
+
+def retry(sleep_time: int, timeout_error_message: str):
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            start_time = datetime.datetime.now()
+            timeout = datetime.timedelta(minutes=1)
+            while datetime.datetime.now() - start_time <= timeout:
+                try:
+                    return func(*args, **kwargs)
+                except Exception:
+                    time.sleep(sleep_time)
+            raise TimeoutError(timeout_error_message)
+
+        return wrapper
+
+    return decorator
+
+
+def lro(
+    client,
+    response,
+    status_codes: Optional[List[str]] = [200, 202],
+    sleep_time: Optional[int] = 1,
+    return_status_code: Optional[bool] = False,
+):
+
+    if response.status_code not in status_codes:
+        raise FabricHTTPException(response)
+    if response.status_code == status_codes[0]:
+        if return_status_code:
+            result = response.status_code
+        else:
+            result = response
+    if response.status_code == status_codes[1]:
+        operationId = response.headers["x-ms-operation-id"]
+        response = client.get(f"/v1/operations/{operationId}")
+        response_body = json.loads(response.content)
+        while response_body["status"] not in ["Succeeded", "Failed"]:
+            time.sleep(sleep_time)
+            response = client.get(f"/v1/operations/{operationId}")
+            response_body = json.loads(response.content)
+        if response_body["status"] != "Succeeded":
+            raise FabricHTTPException(response)
+        if return_status_code:
+            result = response.status_code
+        else:
+            response = client.get(f"/v1/operations/{operationId}/result")
+            result = response
+
+    return result
+
+
+def pagination(client, response):
+
+    responses = []
+    response_json = response.json()
+    responses.append(response_json)
+
+    # Check for pagination
+    continuation_token = response_json.get("continuationToken")
+    continuation_uri = response_json.get("continuationUri")
+
+    # Loop to handle pagination
+    while continuation_token is not None:
+        response = client.get(continuation_uri)
+        response_json = response.json()
+        responses.append(response_json)
+
+        # Update the continuation token and URI for the next iteration
+        continuation_token = response_json.get("continuationToken")
+        continuation_uri = response_json.get("continuationUri")
+
+    return responses

sempy_labs/_icons.py

@@ -8,17 +8,26 @@ start_bold = "\033[1m"
 end_bold = "\033[0m"
 bullet = "\u2022"
 warning = "⚠️"
+error = "\u274C"
+info = "ℹ️"
 data_type_mapping = {
     "string": "String",
-    "bigint": "Int64",
     "int": "Int64",
+    "tinyint": "Int64",
     "smallint": "Int64",
+    "bigint": "Int64",
     "boolean": "Boolean",
     "timestamp": "DateTime",
     "date": "DateTime",
     "decimal(38,18)": "Decimal",
+    "decimal(19,4)": "Decimal",
     "double": "Double",
+    "float": "Double",
 }
 measure_icon = "\u2211"
 table_icon = "\u229E"
 column_icon = "\u229F"
+model_bpa_name = "ModelBPA"
+report_bpa_name = "ReportBPA"
+severity_mapping = {warning: "Warning", error: "Error", info: "Info"}
+special_characters = ['"', "/", '"', ":", "|", "<", ">", "*", "?", "'", "!"]