semantic-link-labs 0.7.2__py3-none-any.whl → 0.7.4__py3-none-any.whl

sempy_labs/_deployment_pipelines.py

@@ -0,0 +1,171 @@
+import sempy.fabric as fabric
+import pandas as pd
+from sempy_labs._helper_functions import (
+    pagination,
+)
+import sempy_labs._icons as icons
+from sempy.fabric.exceptions import FabricHTTPException
+
+
+def list_deployment_pipelines() -> pd.DataFrame:
+    """
+    Shows a list of deployment pipelines the user can access.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing a list of deployment pipelines the user can access.
+    """
+
+    df = pd.DataFrame(
+        columns=["Deployment Pipeline Id", "Deployment Pipeline Name", "Description"]
+    )
+
+    client = fabric.FabricRestClient()
+    response = client.get("/v1/deploymentPipelines")
+
+    if response.status_code != 200:
+        raise FabricHTTPException(response)
+
+    responses = pagination(client, response)
+
+    for r in responses:
+        for v in r.get("value", []):
+            new_data = {
+                "Deployment Pipeline Id": v.get("id"),
+                "Deployment Pipeline Name": v.get("displayName"),
+                "Description": v.get("description"),
+            }
+            df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
+
+    return df
+
+
+def list_deployment_pipeline_stages(deployment_pipeline: str) -> pd.DataFrame:
+    """
+    Shows the specified deployment pipeline stages.
+
+    Parameters
+    ----------
+    deployment_pipeline : str
+        The deployment pipeline name.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing the specified deployment pipeline stages.
+    """
+
+    from sempy_labs._helper_functions import resolve_deployment_pipeline_id
+
+    df = pd.DataFrame(
+        columns=[
+            "Deployment Pipeline Stage Id",
+            "Deployment Pipeline Stage Name",
+            "Order",
+            "Description",
+            "Workspace Id",
+            "Workspace Name",
+            "Public",
+        ]
+    )
+
+    deployment_pipeline_id = resolve_deployment_pipeline_id(
+        deployment_pipeline=deployment_pipeline
+    )
+    client = fabric.FabricRestClient()
+    response = client.get(f"/v1/deploymentPipelines/{deployment_pipeline_id}/stages")
+
+    if response.status_code != 200:
+        raise FabricHTTPException(response)
+
+    responses = pagination(client, response)
+
+    for r in responses:
+        for v in r.get("value", []):
+            new_data = {
+                "Deployment Pipeline Stage Id": v["id"],
+                "Deployment Pipeline Stage Name": v["displayName"],
+                "Description": v["description"],
+                "Order": v["order"],
+                "Workspace Id": v["workspaceId"],
+                "Workspace Name": v["workspaceName"],
+                "Public": v["isPublic"],
+            }
+            df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
+
+    df["Order"] = df["Order"].astype(int)
+    df["Public"] = df["Public"].astype(bool)
+
+    return df
+
+
+def list_deployment_pipeline_stage_items(
+    deployment_pipeline: str, stage_name: str
+) -> pd.DataFrame:
+    """
+    Shows the supported items from the workspace assigned to the specified stage of the specified deployment pipeline.
+
+    Parameters
+    ----------
+    deployment_pipeline : str
+        The deployment pipeline name.
+    stage_name : str
+        The deployment pipeline stage name.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing the supported items from the workspace assigned to the specified stage of the specified deployment pipeline.
+    """
+
+    from sempy_labs._helper_functions import resolve_deployment_pipeline_id
+
+    df = pd.DataFrame(
+        columns=[
+            "Deployment Pipeline Stage Item Id",
+            "Deployment Pipeline Stage Item Name",
+            "Item Type",
+            "Source Item Id",
+            "Target Item Id",
+            "Last Deployment Time",
+        ]
+    )
+
+    deployment_pipeline_id = resolve_deployment_pipeline_id(
+        deployment_pipeline=deployment_pipeline
+    )
+    dfPS = list_deployment_pipeline_stages(deployment_pipeline=deployment_pipeline)
+    dfPS_filt = dfPS[dfPS["Deployment Pipeline Stage Name"] == stage_name]
+
+    if len(dfPS_filt) == 0:
+        raise ValueError(
+            f"{icons.red_dot} The '{stage_name}' stage does not exist within the '{deployment_pipeline}' deployment pipeline."
+        )
+    stage_id = dfPS_filt["Deployment Pipeline Stage ID"].iloc[0]
+
+    client = fabric.FabricRestClient()
+    response = client.get(
+        f"/v1/deploymentPipelines/{deployment_pipeline_id}/stages/{stage_id}/items"
+    )
+
+    if response.status_code != 200:
+        raise FabricHTTPException(response)
+
+    responses = pagination(client, response)
+
+    for r in responses:
+        for v in r.get("value", []):
+            new_data = {
+                "Deployment Pipeline Stage Item Id": v.get("itemId"),
+                "Deployment Pipeline Stage Item Name": v.get("itemDisplayName"),
+                "Item Type": v.get("itemType"),
+                "Source Item Id": v.get("sourceItemId"),
+                "Target Item Id": v.get("targetItemId"),
+                "Last Deployment Time": v.get("lastDeploymentTime"),
+            }
+            df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
+
+    df["Last Deployment Time"] = pd.to_datetime(df["Last Deployment Time"])
+
+    return df

sempy_labs/_environments.py

@@ -0,0 +1,156 @@
+import sempy.fabric as fabric
+import pandas as pd
+import sempy_labs._icons as icons
+from typing import Optional
+from sempy_labs._helper_functions import (
+    resolve_workspace_name_and_id,
+    lro,
+    pagination,
+)
+from sempy.fabric.exceptions import FabricHTTPException
+
+
+def create_environment(
+    environment: str, description: Optional[str] = None, workspace: Optional[str] = None
+):
+    """
+    Creates a Fabric environment.
+
+    Parameters
+    ----------
+    environment: str
+        Name of the environment.
+    description : str, default=None
+        A description of the environment.
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
+
+    request_body = {"displayName": environment}
+
+    if description:
+        request_body["description"] = description
+
+    client = fabric.FabricRestClient()
+    response = client.post(
+        f"/v1/workspaces/{workspace_id}/environments", json=request_body
+    )
+
+    lro(client, response, status_codes=[201, 202])
+
+    print(
+        f"{icons.green_dot} The '{environment}' environment has been created within the '{workspace}' workspace."
+    )
+
+
+def list_environments(workspace: Optional[str] = None) -> pd.DataFrame:
+    """
+    Shows the environments within a workspace.
+
+    Parameters
+    ----------
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing the environments within a workspace.
+    """
+
+    df = pd.DataFrame(columns=["Environment Name", "Environment Id", "Description"])
+
+    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
+
+    client = fabric.FabricRestClient()
+    response = client.get(f"/v1/workspaces/{workspace_id}/environments")
+    if response.status_code != 200:
+        raise FabricHTTPException(response)
+
+    responses = pagination(client, response)
+
+    for r in responses:
+        for v in r.get("value", []):
+            new_data = {
+                "Environment Name": v.get("displayName"),
+                "Environment Id": v.get("id"),
+                "Description": v.get("description"),
+            }
+            df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
+
+    return df
+
+
+def delete_environment(environment: str, workspace: Optional[str] = None):
+    """
+    Deletes a Fabric environment.
+
+    Parameters
+    ----------
+    environment: str
+        Name of the environment.
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    from sempy_labs._helper_functions import resolve_environment_id
+
+    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
+    environment_id = resolve_environment_id(
+        environment=environment, workspace=workspace
+    )
+
+    client = fabric.FabricRestClient()
+    response = client.delete(
+        f"/v1/workspaces/{workspace_id}/environments/{environment_id}"
+    )
+
+    if response.status_code != 200:
+        raise FabricHTTPException(response)
+
+    print(
+        f"{icons.green_dot} The '{environment}' environment within the '{workspace}' workspace has been deleted."
+    )
+
+
+def publish_environment(environment: str, workspace: Optional[str] = None):
+    """
+    Publishes a Fabric environment.
+
+    Parameters
+    ----------
+    environment: str
+        Name of the environment.
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    # https://learn.microsoft.com/en-us/rest/api/fabric/environment/spark-libraries/publish-environment?tabs=HTTP
+
+    from sempy_labs._helper_functions import resolve_environment_id
+
+    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
+    environment_id = resolve_environment_id(
+        environment=environment, workspace=workspace
+    )
+
+    client = fabric.FabricRestClient()
+    response = client.post(
+        f"/v1/workspaces/{workspace_id}/environments/{environment_id}/staging/publish"
+    )
+
+    lro(client, response)
+
+    print(
+        f"{icons.green_dot} The '{environment}' environment within the '{workspace}' workspace has been published."
+    )

sempy_labs/_generate_semantic_model.py

@@ -13,12 +13,14 @@ from sempy_labs._helper_functions import (
 )
 from sempy_labs.lakehouse._lakehouse import lakehouse_attached
 import sempy_labs._icons as icons
+from sempy_labs._refresh_semantic_model import refresh_semantic_model
 
 
 def create_blank_semantic_model(
     dataset: str,
     compatibility_level: int = 1605,
     workspace: Optional[str] = None,
+    overwrite: Optional[bool] = True,
 ):
     """
     Creates a new blank semantic model (no tables/columns etc.).
@@ -33,34 +35,42 @@ def create_blank_semantic_model(
         The Fabric workspace name.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
+    overwrite : bool, default=False
+        If set to True, overwrites the existing semantic model in the workspace if it exists.
     """
 
     workspace = fabric.resolve_workspace_name(workspace)
+    dfD = fabric.list_datasets(workspace=workspace, mode="rest")
+    dfD_filt = dfD[dfD["Dataset Name"] == dataset]
 
-    min_compat = 1500
+    if len(dfD_filt) > 0 and not overwrite:
+        raise ValueError(
+            f"{icons.warning} The '{dataset}' semantic model already exists within the '{workspace}' workspace. The 'overwrite' parameter is set to False so the blank new semantic model was not created."
+        )
 
+    min_compat = 1500
     if compatibility_level < min_compat:
         raise ValueError(
             f"{icons.red_dot} Compatiblity level must be at least {min_compat}."
         )
 
     tmsl = f"""
-  {{
-    "createOrReplace": {{
-      "object": {{
-        "database": '{dataset}'
-      }},
-      "database": {{
-        "name": '{dataset}',
-        "compatibilityLevel": {compatibility_level},
-        "model": {{
-          "culture": "en-US",
-          "defaultPowerBIDataSourceVersion": "powerBI_V3"
+    {{
+        "createOrReplace": {{
+        "object": {{
+            "database": '{dataset}'
+        }},
+        "database": {{
+            "name": '{dataset}',
+            "compatibilityLevel": {compatibility_level},
+            "model": {{
+            "culture": "en-US",
+            "defaultPowerBIDataSourceVersion": "powerBI_V3"
+            }}
+        }}
         }}
-      }}
     }}
-  }}
-  """
+    """
 
     fabric.execute_tmsl(script=tmsl, workspace=workspace)
 
@@ -89,12 +99,12 @@ def create_semantic_model_from_bim(
 
     (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
 
-    dfI = fabric.list_items(workspace=workspace, type="SemanticModel")
-    dfI_filt = dfI[(dfI["Display Name"] == dataset)]
+    dfI = fabric.list_datasets(workspace=workspace, mode="rest")
+    dfI_filt = dfI[(dfI["Dataset Name"] == dataset)]
 
     if len(dfI_filt) > 0:
         raise ValueError(
-            f"{icons.red_dot} '{dataset}' already exists as a semantic model in the '{workspace}' workspace."
+            f"{icons.red_dot} The '{dataset}' semantic model already exists as a semantic model in the '{workspace}' workspace."
         )
 
     client = fabric.FabricRestClient()
@@ -133,12 +143,77 @@ def create_semantic_model_from_bim(
     )
 
 
+def update_semantic_model_from_bim(
+    dataset: str, bim_file: dict, workspace: Optional[str] = None
+):
+    """
+    Updates a semantic model definition based on a Model.bim file.
+
+    Parameters
+    ----------
+    dataset : str
+        Name of the semantic model.
+    bim_file : dict
+        The model.bim file.
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
+
+    dfD = fabric.list_datasets(workspace=workspace, mode="rest")
+    dfD_filt = dfD[dfD["Dataset Name"] == dataset]
+    if len(dfD_filt) == 0:
+        raise ValueError(
+            f"{icons.red_dot} The '{dataset}' semantic model within the '{workspace}' workspace does not exist."
+        )
+    dataset_id = dfD_filt["Dataset Id"].iloc[0]
+
+    client = fabric.FabricRestClient()
+    defPBIDataset = {"version": "1.0", "settings": {}}
+
+    payloadPBIDefinition = _conv_b64(defPBIDataset)
+    payloadBim = _conv_b64(bim_file)
+
+    request_body = {
+        "displayName": dataset,
+        "definition": {
+            "parts": [
+                {
+                    "path": "model.bim",
+                    "payload": payloadBim,
+                    "payloadType": "InlineBase64",
+                },
+                {
+                    "path": "definition.pbidataset",
+                    "payload": payloadPBIDefinition,
+                    "payloadType": "InlineBase64",
+                },
+            ]
+        },
+    }
+
+    response = client.post(
+        f"/v1/workspaces/{workspace_id}/semanticModels/{dataset_id}/updateDefinition",
+        json=request_body,
+    )
+
+    lro(client, response, status_codes=[200, 202])
+
+    print(
+        f"{icons.green_dot} The '{dataset}' semantic model has been updated within the '{workspace}' workspace."
+    )
+
+
 def deploy_semantic_model(
     source_dataset: str,
     source_workspace: Optional[str] = None,
     target_dataset: Optional[str] = None,
     target_workspace: Optional[str] = None,
     refresh_target_dataset: Optional[bool] = True,
+    overwrite: Optional[bool] = False,
 ):
     """
     Deploys a semantic model based on an existing semantic model.
@@ -159,14 +234,10 @@ def deploy_semantic_model(
         or if no lakehouse attached, resolves to the workspace of the notebook.
     refresh_target_dataset : bool, default=True
         If set to True, this will initiate a full refresh of the target semantic model in the target workspace.
-
-    Returns
-    -------
-
+    overwrite : bool, default=False
+        If set to True, overwrites the existing semantic model in the workspace if it exists.
     """
 
-    from sempy_labs import refresh_semantic_model
-
     source_workspace = fabric.resolve_workspace_name(source_workspace)
 
     if target_workspace is None:
@@ -181,11 +252,28 @@ def deploy_semantic_model(
             f"parameters have the same value. At least one of these must be different. Please update the parameters."
         )
 
+    dfD = fabric.list_datasets(workspace=target_workspace, mode="rest")
+    dfD_filt = dfD[dfD["Dataset Name"] == target_dataset]
+    if len(dfD_filt) > 0 and not overwrite:
+        raise ValueError(
+            f"{icons.warning} The '{target_dataset}' semantic model already exists within the '{target_workspace}' workspace. The 'overwrite' parameter is set to False so the source semantic model was not deployed to the target destination."
+        )
+
     bim = get_semantic_model_bim(dataset=source_dataset, workspace=source_workspace)
 
-    create_semantic_model_from_bim(
-        dataset=target_dataset, bim_file=bim, workspace=target_workspace
-    )
+    # Create the semantic model if the model does not exist
+    if len(dfD_filt) == 0:
+        create_semantic_model_from_bim(
+            dataset=target_dataset,
+            bim_file=bim,
+            workspace=target_workspace,
+            overwrite=overwrite,
+        )
+    # Update the semantic model if the model exists
+    else:
+        update_semantic_model_from_bim(
+            dataset=target_dataset, bim_file=bim, workspace=target_workspace
+        )
 
     if refresh_target_dataset:
         refresh_semantic_model(dataset=target_dataset, workspace=target_workspace)
@@ -257,3 +345,36 @@ def get_semantic_model_bim(
         )
 
     return bimJson
+
+
+def get_semantic_model_size(dataset: str, workspace: Optional[str] = None):
+
+    workspace = fabric.resolve_workspace_name(workspace)
+
+    dict = fabric.evaluate_dax(
+        dataset=dataset,
+        workspace=workspace,
+        dax_string="""
+        EVALUATE SELECTCOLUMNS(FILTER(INFO.STORAGETABLECOLUMNS(), [COLUMN_TYPE] = "BASIC_DATA"),[DICTIONARY_SIZE])
+        """,
+    )
+
+    used_size = fabric.evaluate_dax(
+        dataset=dataset,
+        workspace=workspace,
+        dax_string="""
+        EVALUATE SELECTCOLUMNS(INFO.STORAGETABLECOLUMNSEGMENTS(),[USED_SIZE])
+        """,
+    )
+    dict_size = dict["[DICTIONARY_SIZE]"].sum()
+    used_size = used_size["[USED_SIZE]"].sum()
+    model_size = dict_size + used_size
+    # Calculate proper bytes size by dividing by 1024 and multiplying by 1000 - per 1000
+    if model_size >= 10**9:
+        result = model_size / (1024**3) * 10**9
+    elif model_size >= 10**6:
+        result = model_size / (1024**2) * 10**6
+    elif model_size >= 10**3:
+        result = model_size / (1024) * 10**3
+
+    return result