semantic-link-labs 0.7.1__py3-none-any.whl → 0.7.3__py3-none-any.whl

sempy_labs/_generate_semantic_model.py

@@ -13,12 +13,14 @@ from sempy_labs._helper_functions import (
 )
 from sempy_labs.lakehouse._lakehouse import lakehouse_attached
 import sempy_labs._icons as icons
+from sempy_labs._refresh_semantic_model import refresh_semantic_model
 
 
 def create_blank_semantic_model(
     dataset: str,
     compatibility_level: int = 1605,
     workspace: Optional[str] = None,
+    overwrite: Optional[bool] = True,
 ):
     """
     Creates a new blank semantic model (no tables/columns etc.).
@@ -33,34 +35,42 @@ def create_blank_semantic_model(
         The Fabric workspace name.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
+    overwrite : bool, default=False
+        If set to True, overwrites the existing semantic model in the workspace if it exists.
     """
 
     workspace = fabric.resolve_workspace_name(workspace)
+    dfD = fabric.list_datasets(workspace=workspace, mode="rest")
+    dfD_filt = dfD[dfD["Dataset Name"] == dataset]
 
-    min_compat = 1500
+    if len(dfD_filt) > 0 and not overwrite:
+        raise ValueError(
+            f"{icons.warning} The '{dataset}' semantic model already exists within the '{workspace}' workspace. The 'overwrite' parameter is set to False so the blank new semantic model was not created."
+        )
 
+    min_compat = 1500
     if compatibility_level < min_compat:
         raise ValueError(
             f"{icons.red_dot} Compatiblity level must be at least {min_compat}."
         )
 
     tmsl = f"""
-  {{
-    "createOrReplace": {{
-      "object": {{
-        "database": '{dataset}'
-      }},
-      "database": {{
-        "name": '{dataset}',
-        "compatibilityLevel": {compatibility_level},
-        "model": {{
-          "culture": "en-US",
-          "defaultPowerBIDataSourceVersion": "powerBI_V3"
+    {{
+        "createOrReplace": {{
+        "object": {{
+            "database": '{dataset}'
+        }},
+        "database": {{
+            "name": '{dataset}',
+            "compatibilityLevel": {compatibility_level},
+            "model": {{
+            "culture": "en-US",
+            "defaultPowerBIDataSourceVersion": "powerBI_V3"
+            }}
+        }}
         }}
-      }}
     }}
-  }}
-  """
+    """
 
     fabric.execute_tmsl(script=tmsl, workspace=workspace)
 
@@ -89,12 +99,12 @@ def create_semantic_model_from_bim(
 
     (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
 
-    dfI = fabric.list_items(workspace=workspace, type="SemanticModel")
-    dfI_filt = dfI[(dfI["Display Name"] == dataset)]
+    dfI = fabric.list_datasets(workspace=workspace, mode="rest")
+    dfI_filt = dfI[(dfI["Dataset Name"] == dataset)]
 
     if len(dfI_filt) > 0:
         raise ValueError(
-            f"{icons.red_dot} '{dataset}' already exists as a semantic model in the '{workspace}' workspace."
+            f"{icons.red_dot} The '{dataset}' semantic model already exists as a semantic model in the '{workspace}' workspace."
         )
 
     client = fabric.FabricRestClient()
@@ -133,12 +143,77 @@ def create_semantic_model_from_bim(
     )
 
 
+def update_semantic_model_from_bim(
+    dataset: str, bim_file: dict, workspace: Optional[str] = None
+):
+    """
+    Updates a semantic model definition based on a Model.bim file.
+
+    Parameters
+    ----------
+    dataset : str
+        Name of the semantic model.
+    bim_file : dict
+        The model.bim file.
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
+
+    dfD = fabric.list_datasets(workspace=workspace, mode="rest")
+    dfD_filt = dfD[dfD["Dataset Name"] == dataset]
+    if len(dfD_filt) == 0:
+        raise ValueError(
+            f"{icons.red_dot} The '{dataset}' semantic model within the '{workspace}' workspace does not exist."
+        )
+    dataset_id = dfD_filt["Dataset Id"].iloc[0]
+
+    client = fabric.FabricRestClient()
+    defPBIDataset = {"version": "1.0", "settings": {}}
+
+    payloadPBIDefinition = _conv_b64(defPBIDataset)
+    payloadBim = _conv_b64(bim_file)
+
+    request_body = {
+        "displayName": dataset,
+        "definition": {
+            "parts": [
+                {
+                    "path": "model.bim",
+                    "payload": payloadBim,
+                    "payloadType": "InlineBase64",
+                },
+                {
+                    "path": "definition.pbidataset",
+                    "payload": payloadPBIDefinition,
+                    "payloadType": "InlineBase64",
+                },
+            ]
+        },
+    }
+
+    response = client.post(
+        f"/v1/workspaces/{workspace_id}/semanticModels/{dataset_id}/updateDefinition",
+        json=request_body,
+    )
+
+    lro(client, response, status_codes=[200, 202])
+
+    print(
+        f"{icons.green_dot} The '{dataset}' semantic model has been updated within the '{workspace}' workspace."
+    )
+
+
 def deploy_semantic_model(
     source_dataset: str,
     source_workspace: Optional[str] = None,
     target_dataset: Optional[str] = None,
     target_workspace: Optional[str] = None,
     refresh_target_dataset: Optional[bool] = True,
+    overwrite: Optional[bool] = False,
 ):
     """
     Deploys a semantic model based on an existing semantic model.
@@ -159,14 +234,10 @@ def deploy_semantic_model(
         or if no lakehouse attached, resolves to the workspace of the notebook.
     refresh_target_dataset : bool, default=True
         If set to True, this will initiate a full refresh of the target semantic model in the target workspace.
-
-    Returns
-    -------
-
+    overwrite : bool, default=False
+        If set to True, overwrites the existing semantic model in the workspace if it exists.
     """
 
-    from sempy_labs import refresh_semantic_model
-
     source_workspace = fabric.resolve_workspace_name(source_workspace)
 
     if target_workspace is None:
@@ -181,11 +252,28 @@ def deploy_semantic_model(
             f"parameters have the same value. At least one of these must be different. Please update the parameters."
         )
 
+    dfD = fabric.list_datasets(workspace=target_workspace, mode="rest")
+    dfD_filt = dfD[dfD["Dataset Name"] == target_dataset]
+    if len(dfD_filt) > 0 and not overwrite:
+        raise ValueError(
+            f"{icons.warning} The '{target_dataset}' semantic model already exists within the '{target_workspace}' workspace. The 'overwrite' parameter is set to False so the source semantic model was not deployed to the target destination."
+        )
+
     bim = get_semantic_model_bim(dataset=source_dataset, workspace=source_workspace)
 
-    create_semantic_model_from_bim(
-        dataset=target_dataset, bim_file=bim, workspace=target_workspace
-    )
+    # Create the semantic model if the model does not exist
+    if len(dfD_filt) == 0:
+        create_semantic_model_from_bim(
+            dataset=target_dataset,
+            bim_file=bim,
+            workspace=target_workspace,
+            overwrite=overwrite,
+        )
+    # Update the semantic model if the model exists
+    else:
+        update_semantic_model_from_bim(
+            dataset=target_dataset, bim_file=bim, workspace=target_workspace
+        )
 
     if refresh_target_dataset:
         refresh_semantic_model(dataset=target_dataset, workspace=target_workspace)
@@ -257,3 +345,36 @@ def get_semantic_model_bim(
         )
 
     return bimJson
+
+
+def get_semantic_model_size(dataset: str, workspace: Optional[str] = None):
+
+    workspace = fabric.resolve_workspace_name(workspace)
+
+    dict = fabric.evaluate_dax(
+        dataset=dataset,
+        workspace=workspace,
+        dax_string="""
+        EVALUATE SELECTCOLUMNS(FILTER(INFO.STORAGETABLECOLUMNS(), [COLUMN_TYPE] = "BASIC_DATA"),[DICTIONARY_SIZE])
+        """,
+    )
+
+    used_size = fabric.evaluate_dax(
+        dataset=dataset,
+        workspace=workspace,
+        dax_string="""
+        EVALUATE SELECTCOLUMNS(INFO.STORAGETABLECOLUMNSEGMENTS(),[USED_SIZE])
+        """,
+    )
+    dict_size = dict["[DICTIONARY_SIZE]"].sum()
+    used_size = used_size["[USED_SIZE]"].sum()
+    model_size = dict_size + used_size
+    # Calculate proper bytes size by dividing by 1024 and multiplying by 1000 - per 1000
+    if model_size >= 10**9:
+        result = model_size / (1024**3) * 10**9
+    elif model_size >= 10**6:
+        result = model_size / (1024**2) * 10**6
+    elif model_size >= 10**3:
+        result = model_size / (1024) * 10**3
+
+    return result

sempy_labs/_git.py

@@ -0,0 +1,380 @@
+import sempy.fabric as fabric
+import pandas as pd
+import sempy_labs._icons as icons
+from typing import Optional, List
+from sempy_labs._helper_functions import (
+    resolve_workspace_name_and_id,
+    lro,
+)
+from sempy.fabric.exceptions import FabricHTTPException
+
+
+def connect_workspace_to_git(
+    organization_name: str,
+    project_name: str,
+    repository_name: str,
+    branch_name: str,
+    directory_name: str,
+    git_provider_type: str = "AzureDevOps",
+    workspace: Optional[str] = None,
+):
+    """
+    Connects a workspace to a git repository.
+
+    Parameters
+    ----------
+    organization_name : str
+        The organization name.
+    project_name : str
+        The project name.
+    repository_name : str
+        The repository name.
+    branch_name : str
+        The branch name.
+    directory_name : str
+        The directory name.
+    git_provider_type : str, default="AzureDevOps"
+        A `Git provider type <https://learn.microsoft.com/rest/api/fabric/core/git/connect?tabs=HTTP#gitprovidertype>`_.
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    # https://learn.microsoft.com/en-us/rest/api/fabric/core/git/connect?tabs=HTTP
+
+    workspace, workspace_id = resolve_workspace_name_and_id(workspace)
+
+    request_body = {
+        "gitProviderDetails": {
+            "organizationName": organization_name,
+            "projectName": project_name,
+            "gitProviderType": git_provider_type,
+            "repositoryName": repository_name,
+            "branchName": branch_name,
+            "directoryName": directory_name,
+        }
+    }
+
+    client = fabric.FabricRestClient()
+    response = client.post(
+        f"/v1/workspaces/{workspace_id}/git/connect", json=request_body
+    )
+    if response.status_code != 200:
+        raise FabricHTTPException(response)
+
+    print(
+        f"{icons.green_dot} The '{workspace}' workspace has been connected to the '{project_name}' Git project within the '{repository_name}' repository."
+    )
+
+
+def disconnect_workspace_from_git(workspace: Optional[str] = None):
+    """
+    Disconnects a workpsace from a git repository.
+
+    Parameters
+    ----------
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    # https://learn.microsoft.com/en-us/rest/api/fabric/core/git/disconnect?tabs=HTTP
+
+    workspace, workspace_id = resolve_workspace_name_and_id(workspace)
+
+    client = fabric.FabricRestClient()
+    response = client.post(f"/v1/workspaces/{workspace_id}/git/disconnect")
+    if response.status_code != 200:
+        raise FabricHTTPException(response)
+
+    print(
+        f"{icons.green_dot} The '{workspace}' workspace has been disconnected from Git."
+    )
+
+
+def get_git_status(workspace: Optional[str] = None) -> pd.DataFrame:
+    """
+    Obtains the Git status of items in the workspace, that can be committed to Git.
+
+    Parameters
+    ----------
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing the Git status of items in the workspace.
+    """
+
+    # https://learn.microsoft.com/en-us/rest/api/fabric/core/git/get-status?tabs=HTTP
+
+    workspace, workspace_id = resolve_workspace_name_and_id(workspace)
+
+    df = pd.DataFrame(
+        columns=[
+            "Workspace Head",
+            "Remote Commit Hash",
+            "Object ID",
+            "Logical ID",
+            "Item Type",
+            "Item Name",
+            "Workspace Change",
+            "Remote Change",
+            "Conflict Type",
+        ]
+    )
+
+    client = fabric.FabricRestClient()
+    response = client.get(f"/v1/workspaces/{workspace_id}/git/status")
+
+    if response not in [200, 202]:
+        raise FabricHTTPException(response)
+
+    result = lro(client, response).json()
+
+    for v in result.get("value", []):
+        changes = v.get("changes", [])
+        item_metadata = changes.get("itemMetadata", {})
+        item_identifier = item_metadata.get("itemIdentifier", {})
+
+        new_data = {
+            "Workspace Head": v.get("workspaceHead"),
+            "Remote Commit Hash": v.get("remoteCommitHash"),
+            "Object ID": item_identifier.get("objectId"),
+            "Logical ID": item_identifier.get("logicalId"),
+            "Item Type": item_metadata.get("itemType"),
+            "Item Name": item_metadata.get("displayName"),
+            "Remote Change": changes.get("remoteChange"),
+            "Workspace Change": changes.get("workspaceChange"),
+            "Conflict Type": changes.get("conflictType"),
+        }
+        df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
+
+    return df
+
+
+def get_git_connection(workspace: Optional[str] = None) -> pd.DataFrame:
+    """
+    Obtains the Git status of items in the workspace, that can be committed to Git.
+
+    Parameters
+    ----------
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing the Git status of items in the workspace.
+    """
+
+    # https://learn.microsoft.com/en-us/rest/api/fabric/core/git/get-status?tabs=HTTP
+
+    workspace, workspace_id = resolve_workspace_name_and_id(workspace)
+
+    df = pd.DataFrame(
+        columns=[
+            "Organization Name",
+            "Project Name",
+            "Git Provider Type",
+            "Repository Name",
+            "Branch Name",
+            "Directory Name",
+            "Workspace Head",
+            "Last Sync Time",
+            "Git Connection State",
+        ]
+    )
+
+    client = fabric.FabricRestClient()
+    response = client.get(f"/v1/workspaces/{workspace_id}/git/connection")
+
+    if response.status_code != 200:
+        raise FabricHTTPException(response)
+
+    for v in response.json().get("value", []):
+        provider_details = v.get("gitProviderDetails", {})
+        sync_details = v.get("gitSyncDetails", {})
+        new_data = {
+            "Organization Name": provider_details.get("organizationName"),
+            "Project Name": provider_details.get("projectName"),
+            "Git Provider Type": provider_details.get("gitProviderType"),
+            "Repository Name": provider_details.get("repositoryName"),
+            "Branch Name": provider_details.get("branchName"),
+            "Directory Name": provider_details.get("directoryName"),
+            "Workspace Head": sync_details.get("head"),
+            "Last Sync Time": sync_details.get("lastSyncTime"),
+            "Git Conneciton State": v.get("gitConnectionState"),
+        }
+        df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
+
+    return df
+
+
+def initialize_git_connection(workspace: Optional[str] = None):
+    """
+    Initializes a connection for a workspace that is connected to Git.
+
+    Parameters
+    ----------
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    # https://learn.microsoft.com/en-us/rest/api/fabric/core/git/initialize-connection?tabs=HTTP
+
+    workspace, workspace_id = resolve_workspace_name_and_id(workspace)
+
+    client = fabric.FabricRestClient()
+    response = client.post(f"/v1/workspaces/{workspace_id}/git/initializeConnection")
+
+    if response not in [200, 202]:
+        raise FabricHTTPException(response)
+
+    lro(client, response)
+
+    print(
+        f"{icons.green_dot} The '{workspace}' workspace git connection has been initialized."
+    )
+
+
+def commit_to_git(
+    comment: str, item_ids: str | List[str] = None, workspace: Optional[str] = None
+):
+    """
+    Commits all or a selection of items within a workspace to Git.
+
+    Parameters
+    ----------
+    comment : str
+        The Git commit comment.
+    item_ids : str | List[str], default=None
+        A list of item Ids to commit to Git.
+        Defaults to None which commits all items to Git.
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    # https://learn.microsoft.com/en-us/rest/api/fabric/core/git/commit-to-git?tabs=HTTP
+
+    workspace, workspace_id = resolve_workspace_name_and_id(workspace)
+
+    gs = get_git_status(workspace=workspace)
+    workspace_head = gs["Workspace Head"].iloc[0]
+
+    if item_ids is None:
+        commit_mode = "All"
+    else:
+        commit_mode = "Selective"
+
+    if isinstance(item_ids, str):
+        item_ids = [item_ids]
+
+    request_body = {
+        "mode": commit_mode,
+        "workspaceHead": workspace_head,
+        "comment": comment,
+    }
+
+    if item_ids is not None:
+        request_body["items"] = [{"objectId": item_id} for item_id in item_ids]
+
+    client = fabric.FabricRestClient()
+    response = client.post(
+        f"/v1/workspaces/{workspace_id}/git/commitToGit",
+        json=request_body,
+    )
+
+    if response.status_code not in [200, 202]:
+        raise FabricHTTPException(response)
+
+    lro(client, response)
+
+    if commit_mode == "All":
+        print(
+            f"{icons.green_dot} All items within the '{workspace}' workspace have been committed to Git."
+        )
+    else:
+        print(
+            f"{icons.green_dot} The {item_ids} items ithin the '{workspace}' workspace have been committed to Git."
+        )
+
+
+def update_from_git(
+    remote_commit_hash: str,
+    conflict_resolution_policy: str,
+    workspace_head: Optional[str] = None,
+    allow_override: Optional[bool] = False,
+    workspace: Optional[str] = None,
+):
+    """
+    Updates the workspace with commits pushed to the connected branch.
+
+    Parameters
+    ----------
+    workspace_head : str
+        Full SHA hash that the workspace is synced to. This value may be null only after Initialize Connection.
+        In other cases, the system will validate that the given value is aligned with the head known to the system.
+    remove_commit_hash : str
+        Remote full SHA commit hash.
+    confilict_resolution_policy : str
+        The `conflict resolution policy <https://learn.microsoft.com/rest/api/fabric/core/git/update-from-git?tabs=HTTP#conflictresolutionpolicy>`_.
+    allow_override : bool, default=False
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    # https://learn.microsoft.com/en-us/rest/api/fabric/core/git/update-from-git?tabs=HTTP
+
+    workspace, workspace_id = resolve_workspace_name_and_id(workspace)
+
+    conflict_resolution_policies = ["PreferWorkspace", "PreferRemote"]
+    if "remote" in conflict_resolution_policies.lower():
+        conflict_resolution_policies = "PreferRemote"
+    elif "workspace" in conflict_resolution_policies.lower():
+        conflict_resolution_policies = "PreferWorkspace"
+
+    if conflict_resolution_policy not in conflict_resolution_policies:
+        raise ValueError(
+            f"{icons.red_dot} Invalid conflict resolution policy. Valid options: {conflict_resolution_policies}."
+        )
+
+    request_body = {}
+    request_body["remoteCommitHash"] = remote_commit_hash
+    request_body["conflictResolution"] = {
+        "conflictResolutionType": "Workspace",
+        "conflictResolutionPolicy": conflict_resolution_policy,
+    }
+
+    if workspace_head is not None:
+        request_body["workspaceHead"] = workspace_head
+    if allow_override is not None:
+        request_body["options"] = {"allowOverrideItems": allow_override}
+
+    client = fabric.FabricRestClient()
+    response = client.post(
+        f"/v1/workspaces/{workspace_id}/git/updateFromGit",
+        json=request_body,
+    )
+
+    if response.status_code not in [200, 202]:
+        raise FabricHTTPException(response)
+
+    lro(client, response)
+
+    print(
+        f"{icons.green_dot} The '{workspace}' workspace has been updated with commits pushed to the connected branch."
+    )

sempy_labs/_helper_functions.py

@@ -12,6 +12,7 @@ from uuid import UUID
 import sempy_labs._icons as icons
 from sempy.fabric.exceptions import FabricHTTPException
 import urllib.parse
+from azure.core.credentials import TokenCredential, AccessToken
 
 
 def create_abfss_path(
@@ -853,3 +854,59 @@ def pagination(client, response):
         continuation_uri = response_json.get("continuationUri")
 
     return responses
+
+
+def resolve_deployment_pipeline_id(deployment_pipeline: str) -> UUID:
+
+    from sempy_labs._deployment_pipelines import list_deployment_pipelines
+
+    dfP = list_deployment_pipelines()
+    dfP_filt = dfP[dfP["Deployment Pipeline Name"] == deployment_pipeline]
+    if len(dfP_filt) == 0:
+        raise ValueError(
+            f"{icons.red_dot} The '{deployment_pipeline}' deployment pipeline is not valid."
+        )
+    deployment_pipeline_id = dfP_filt["Deployment Pipeline Id"].iloc[0]
+
+    return deployment_pipeline_id
+
+
+class FabricTokenCredential(TokenCredential):
+
+    def get_token(
+        self,
+        scopes: str,
+        claims: Optional[str] = None,
+        tenant_id: Optional[str] = None,
+        enable_cae: Optional[bool] = False,
+        **kwargs: any,
+    ) -> AccessToken:
+
+        from notebookutils import mssparkutils
+        token = mssparkutils.credentials.getToken(scopes)
+        access_token = AccessToken(token, 0)
+
+        return access_token
+
+
+def get_adls_client(account_name):
+
+    from azure.storage.filedatalake import DataLakeServiceClient
+
+    account_url = f"https://{account_name}.dfs.core.windows.net"
+
+    service_client = DataLakeServiceClient(
+        account_url, credential=FabricTokenCredential()
+    )
+
+    return service_client
+
+
+def resolve_warehouse_id(warehouse: str, workspace: Optional[str]):
+
+    workspace = fabric.resolve_workspace_name(workspace)
+    warehouse_id = fabric.resolve_item_id(
+        item_name=warehouse, type="Warehouse", workspace=workspace
+    )
+
+    return warehouse_id