semantic-link-labs 0.7.2__py3-none-any.whl → 0.7.4__py3-none-any.whl

sempy_labs/_git.py

@@ -0,0 +1,380 @@
+import sempy.fabric as fabric
+import pandas as pd
+import sempy_labs._icons as icons
+from typing import Optional, List
+from sempy_labs._helper_functions import (
+    resolve_workspace_name_and_id,
+    lro,
+)
+from sempy.fabric.exceptions import FabricHTTPException
+
+
+def connect_workspace_to_git(
+    organization_name: str,
+    project_name: str,
+    repository_name: str,
+    branch_name: str,
+    directory_name: str,
+    git_provider_type: str = "AzureDevOps",
+    workspace: Optional[str] = None,
+):
+    """
+    Connects a workspace to a git repository.
+
+    Parameters
+    ----------
+    organization_name : str
+        The organization name.
+    project_name : str
+        The project name.
+    repository_name : str
+        The repository name.
+    branch_name : str
+        The branch name.
+    directory_name : str
+        The directory name.
+    git_provider_type : str, default="AzureDevOps"
+        A `Git provider type <https://learn.microsoft.com/rest/api/fabric/core/git/connect?tabs=HTTP#gitprovidertype>`_.
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    # https://learn.microsoft.com/en-us/rest/api/fabric/core/git/connect?tabs=HTTP
+
+    workspace, workspace_id = resolve_workspace_name_and_id(workspace)
+
+    request_body = {
+        "gitProviderDetails": {
+            "organizationName": organization_name,
+            "projectName": project_name,
+            "gitProviderType": git_provider_type,
+            "repositoryName": repository_name,
+            "branchName": branch_name,
+            "directoryName": directory_name,
+        }
+    }
+
+    client = fabric.FabricRestClient()
+    response = client.post(
+        f"/v1/workspaces/{workspace_id}/git/connect", json=request_body
+    )
+    if response.status_code != 200:
+        raise FabricHTTPException(response)
+
+    print(
+        f"{icons.green_dot} The '{workspace}' workspace has been connected to the '{project_name}' Git project within the '{repository_name}' repository."
+    )
+
+
+def disconnect_workspace_from_git(workspace: Optional[str] = None):
+    """
+    Disconnects a workpsace from a git repository.
+
+    Parameters
+    ----------
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    # https://learn.microsoft.com/en-us/rest/api/fabric/core/git/disconnect?tabs=HTTP
+
+    workspace, workspace_id = resolve_workspace_name_and_id(workspace)
+
+    client = fabric.FabricRestClient()
+    response = client.post(f"/v1/workspaces/{workspace_id}/git/disconnect")
+    if response.status_code != 200:
+        raise FabricHTTPException(response)
+
+    print(
+        f"{icons.green_dot} The '{workspace}' workspace has been disconnected from Git."
+    )
+
+
+def get_git_status(workspace: Optional[str] = None) -> pd.DataFrame:
+    """
+    Obtains the Git status of items in the workspace, that can be committed to Git.
+
+    Parameters
+    ----------
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing the Git status of items in the workspace.
+    """
+
+    # https://learn.microsoft.com/en-us/rest/api/fabric/core/git/get-status?tabs=HTTP
+
+    workspace, workspace_id = resolve_workspace_name_and_id(workspace)
+
+    df = pd.DataFrame(
+        columns=[
+            "Workspace Head",
+            "Remote Commit Hash",
+            "Object ID",
+            "Logical ID",
+            "Item Type",
+            "Item Name",
+            "Workspace Change",
+            "Remote Change",
+            "Conflict Type",
+        ]
+    )
+
+    client = fabric.FabricRestClient()
+    response = client.get(f"/v1/workspaces/{workspace_id}/git/status")
+
+    if response not in [200, 202]:
+        raise FabricHTTPException(response)
+
+    result = lro(client, response).json()
+
+    for v in result.get("value", []):
+        changes = v.get("changes", [])
+        item_metadata = changes.get("itemMetadata", {})
+        item_identifier = item_metadata.get("itemIdentifier", {})
+
+        new_data = {
+            "Workspace Head": v.get("workspaceHead"),
+            "Remote Commit Hash": v.get("remoteCommitHash"),
+            "Object ID": item_identifier.get("objectId"),
+            "Logical ID": item_identifier.get("logicalId"),
+            "Item Type": item_metadata.get("itemType"),
+            "Item Name": item_metadata.get("displayName"),
+            "Remote Change": changes.get("remoteChange"),
+            "Workspace Change": changes.get("workspaceChange"),
+            "Conflict Type": changes.get("conflictType"),
+        }
+        df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
+
+    return df
+
+
+def get_git_connection(workspace: Optional[str] = None) -> pd.DataFrame:
+    """
+    Obtains the Git status of items in the workspace, that can be committed to Git.
+
+    Parameters
+    ----------
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing the Git status of items in the workspace.
+    """
+
+    # https://learn.microsoft.com/en-us/rest/api/fabric/core/git/get-status?tabs=HTTP
+
+    workspace, workspace_id = resolve_workspace_name_and_id(workspace)
+
+    df = pd.DataFrame(
+        columns=[
+            "Organization Name",
+            "Project Name",
+            "Git Provider Type",
+            "Repository Name",
+            "Branch Name",
+            "Directory Name",
+            "Workspace Head",
+            "Last Sync Time",
+            "Git Connection State",
+        ]
+    )
+
+    client = fabric.FabricRestClient()
+    response = client.get(f"/v1/workspaces/{workspace_id}/git/connection")
+
+    if response.status_code != 200:
+        raise FabricHTTPException(response)
+
+    for v in response.json().get("value", []):
+        provider_details = v.get("gitProviderDetails", {})
+        sync_details = v.get("gitSyncDetails", {})
+        new_data = {
+            "Organization Name": provider_details.get("organizationName"),
+            "Project Name": provider_details.get("projectName"),
+            "Git Provider Type": provider_details.get("gitProviderType"),
+            "Repository Name": provider_details.get("repositoryName"),
+            "Branch Name": provider_details.get("branchName"),
+            "Directory Name": provider_details.get("directoryName"),
+            "Workspace Head": sync_details.get("head"),
+            "Last Sync Time": sync_details.get("lastSyncTime"),
+            "Git Conneciton State": v.get("gitConnectionState"),
+        }
+        df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
+
+    return df
+
+
+def initialize_git_connection(workspace: Optional[str] = None):
+    """
+    Initializes a connection for a workspace that is connected to Git.
+
+    Parameters
+    ----------
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    # https://learn.microsoft.com/en-us/rest/api/fabric/core/git/initialize-connection?tabs=HTTP
+
+    workspace, workspace_id = resolve_workspace_name_and_id(workspace)
+
+    client = fabric.FabricRestClient()
+    response = client.post(f"/v1/workspaces/{workspace_id}/git/initializeConnection")
+
+    if response not in [200, 202]:
+        raise FabricHTTPException(response)
+
+    lro(client, response)
+
+    print(
+        f"{icons.green_dot} The '{workspace}' workspace git connection has been initialized."
+    )
+
+
+def commit_to_git(
+    comment: str, item_ids: str | List[str] = None, workspace: Optional[str] = None
+):
+    """
+    Commits all or a selection of items within a workspace to Git.
+
+    Parameters
+    ----------
+    comment : str
+        The Git commit comment.
+    item_ids : str | List[str], default=None
+        A list of item Ids to commit to Git.
+        Defaults to None which commits all items to Git.
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    # https://learn.microsoft.com/en-us/rest/api/fabric/core/git/commit-to-git?tabs=HTTP
+
+    workspace, workspace_id = resolve_workspace_name_and_id(workspace)
+
+    gs = get_git_status(workspace=workspace)
+    workspace_head = gs["Workspace Head"].iloc[0]
+
+    if item_ids is None:
+        commit_mode = "All"
+    else:
+        commit_mode = "Selective"
+
+    if isinstance(item_ids, str):
+        item_ids = [item_ids]
+
+    request_body = {
+        "mode": commit_mode,
+        "workspaceHead": workspace_head,
+        "comment": comment,
+    }
+
+    if item_ids is not None:
+        request_body["items"] = [{"objectId": item_id} for item_id in item_ids]
+
+    client = fabric.FabricRestClient()
+    response = client.post(
+        f"/v1/workspaces/{workspace_id}/git/commitToGit",
+        json=request_body,
+    )
+
+    if response.status_code not in [200, 202]:
+        raise FabricHTTPException(response)
+
+    lro(client, response)
+
+    if commit_mode == "All":
+        print(
+            f"{icons.green_dot} All items within the '{workspace}' workspace have been committed to Git."
+        )
+    else:
+        print(
+            f"{icons.green_dot} The {item_ids} items ithin the '{workspace}' workspace have been committed to Git."
+        )
+
+
+def update_from_git(
+    remote_commit_hash: str,
+    conflict_resolution_policy: str,
+    workspace_head: Optional[str] = None,
+    allow_override: Optional[bool] = False,
+    workspace: Optional[str] = None,
+):
+    """
+    Updates the workspace with commits pushed to the connected branch.
+
+    Parameters
+    ----------
+    workspace_head : str
+        Full SHA hash that the workspace is synced to. This value may be null only after Initialize Connection.
+        In other cases, the system will validate that the given value is aligned with the head known to the system.
+    remove_commit_hash : str
+        Remote full SHA commit hash.
+    confilict_resolution_policy : str
+        The `conflict resolution policy <https://learn.microsoft.com/rest/api/fabric/core/git/update-from-git?tabs=HTTP#conflictresolutionpolicy>`_.
+    allow_override : bool, default=False
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    # https://learn.microsoft.com/en-us/rest/api/fabric/core/git/update-from-git?tabs=HTTP
+
+    workspace, workspace_id = resolve_workspace_name_and_id(workspace)
+
+    conflict_resolution_policies = ["PreferWorkspace", "PreferRemote"]
+    if "remote" in conflict_resolution_policies.lower():
+        conflict_resolution_policies = "PreferRemote"
+    elif "workspace" in conflict_resolution_policies.lower():
+        conflict_resolution_policies = "PreferWorkspace"
+
+    if conflict_resolution_policy not in conflict_resolution_policies:
+        raise ValueError(
+            f"{icons.red_dot} Invalid conflict resolution policy. Valid options: {conflict_resolution_policies}."
+        )
+
+    request_body = {}
+    request_body["remoteCommitHash"] = remote_commit_hash
+    request_body["conflictResolution"] = {
+        "conflictResolutionType": "Workspace",
+        "conflictResolutionPolicy": conflict_resolution_policy,
+    }
+
+    if workspace_head is not None:
+        request_body["workspaceHead"] = workspace_head
+    if allow_override is not None:
+        request_body["options"] = {"allowOverrideItems": allow_override}
+
+    client = fabric.FabricRestClient()
+    response = client.post(
+        f"/v1/workspaces/{workspace_id}/git/updateFromGit",
+        json=request_body,
+    )
+
+    if response.status_code not in [200, 202]:
+        raise FabricHTTPException(response)
+
+    lro(client, response)
+
+    print(
+        f"{icons.green_dot} The '{workspace}' workspace has been updated with commits pushed to the connected branch."
+    )

sempy_labs/_helper_functions.py

@@ -6,12 +6,12 @@ import pandas as pd
 from functools import wraps
 import datetime
 import time
-from pyspark.sql import SparkSession
 from typing import Optional, Tuple, List
 from uuid import UUID
 import sempy_labs._icons as icons
 from sempy.fabric.exceptions import FabricHTTPException
 import urllib.parse
+from azure.core.credentials import TokenCredential, AccessToken
 
 
 def create_abfss_path(
@@ -391,6 +391,7 @@ def save_as_delta_table(
     delta_table_name: str,
     write_mode: str,
     merge_schema: Optional[bool] = False,
+    schema: Optional[dict] = None,
     lakehouse: Optional[str] = None,
     workspace: Optional[str] = None,
 ):
@@ -407,6 +408,8 @@ def save_as_delta_table(
         The write mode for the save operation. Options: 'append', 'overwrite'.
     merge_schema : bool, default=False
         Merges the schemas of the dataframe to the delta table.
+    schema : dict, default=None
+        A dictionary showing the schema of the columns and their data types.
     lakehouse : str, default=None
         The Fabric lakehouse used by the Direct Lake semantic model.
         Defaults to None which resolves to the lakehouse attached to the notebook.
@@ -414,13 +417,22 @@ def save_as_delta_table(
         The Fabric workspace name.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
-
-    Returns
-    -------
-    UUID
-        The ID of the Power BI report.
     """
 
+    from pyspark.sql import SparkSession
+    from pyspark.sql.types import (
+        StringType,
+        IntegerType,
+        FloatType,
+        DateType,
+        StructType,
+        StructField,
+        BooleanType,
+        LongType,
+        DoubleType,
+        TimestampType,
+    )
+
     if workspace is None:
         workspace_id = fabric.get_workspace_id()
         workspace = fabric.resolve_workspace_name(workspace_id)
@@ -449,9 +461,32 @@ def save_as_delta_table(
         )
 
     dataframe.columns = dataframe.columns.str.replace(" ", "_")
-
     spark = SparkSession.builder.getOrCreate()
-    spark_df = spark.createDataFrame(dataframe)
+
+    type_mapping = {
+        "string": StringType(),
+        "str": StringType(),
+        "integer": IntegerType(),
+        "int": IntegerType(),
+        "float": FloatType(),
+        "date": DateType(),
+        "bool": BooleanType(),
+        "boolean": BooleanType(),
+        "long": LongType(),
+        "double": DoubleType(),
+        "timestamp": TimestampType(),
+    }
+
+    if schema is None:
+        spark_df = spark.createDataFrame(dataframe)
+    else:
+        schema_map = StructType(
+            [
+                StructField(column_name, type_mapping[data_type], True)
+                for column_name, data_type in schema.items()
+            ]
+        )
+        spark_df = spark.createDataFrame(dataframe, schema_map)
 
     filePath = create_abfss_path(
         lakehouse_id=lakehouse_id,
@@ -780,6 +815,37 @@ def resolve_capacity_name(capacity_id: Optional[UUID] = None) -> str:
     return dfC_filt["Display Name"].iloc[0]
 
 
+def resolve_capacity_id(capacity_name: Optional[str] = None) -> UUID:
+    """
+    Obtains the capacity Id for a given capacity name.
+
+    Parameters
+    ----------
+    capacity_name : str, default=None
+        The capacity name.
+        Defaults to None which resolves to the capacity id of the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the capacity name of the workspace of the notebook.
+
+    Returns
+    -------
+    UUID
+        The capacity Id.
+    """
+
+    if capacity_name is None:
+        return get_capacity_id()
+
+    dfC = fabric.list_capacities()
+    dfC_filt = dfC[dfC["Display Name"] == capacity_name]
+
+    if len(dfC_filt) == 0:
+        raise ValueError(
+            f"{icons.red_dot} The '{capacity_name}' capacity does not exist."
+        )
+
+    return dfC_filt["Id"].iloc[0]
+
+
 def retry(sleep_time: int, timeout_error_message: str):
     def decorator(func):
         @wraps(func)
@@ -853,3 +919,132 @@ def pagination(client, response):
         continuation_uri = response_json.get("continuationUri")
 
     return responses
+
+
+def resolve_deployment_pipeline_id(deployment_pipeline: str) -> UUID:
+
+    from sempy_labs._deployment_pipelines import list_deployment_pipelines
+
+    dfP = list_deployment_pipelines()
+    dfP_filt = dfP[dfP["Deployment Pipeline Name"] == deployment_pipeline]
+    if len(dfP_filt) == 0:
+        raise ValueError(
+            f"{icons.red_dot} The '{deployment_pipeline}' deployment pipeline is not valid."
+        )
+    deployment_pipeline_id = dfP_filt["Deployment Pipeline Id"].iloc[0]
+
+    return deployment_pipeline_id
+
+
+class FabricTokenCredential(TokenCredential):
+
+    def get_token(
+        self,
+        scopes: str,
+        claims: Optional[str] = None,
+        tenant_id: Optional[str] = None,
+        enable_cae: Optional[bool] = False,
+        **kwargs: any,
+    ) -> AccessToken:
+
+        from notebookutils import mssparkutils
+
+        token = mssparkutils.credentials.getToken(scopes)
+        access_token = AccessToken(token, 0)
+
+        return access_token
+
+
+def get_adls_client(account_name):
+
+    from azure.storage.filedatalake import DataLakeServiceClient
+
+    account_url = f"https://{account_name}.dfs.core.windows.net"
+
+    service_client = DataLakeServiceClient(
+        account_url, credential=FabricTokenCredential()
+    )
+
+    return service_client
+
+
+def resolve_warehouse_id(warehouse: str, workspace: Optional[str]):
+
+    workspace = fabric.resolve_workspace_name(workspace)
+    warehouse_id = fabric.resolve_item_id(
+        item_name=warehouse, type="Warehouse", workspace=workspace
+    )
+
+    return warehouse_id
+
+
+def get_language_codes(languages: str | List[str]):
+
+    if isinstance(languages, str):
+        languages = [languages]
+
+    for i, lang in enumerate(languages):
+        for k, v in icons.language_map.items():
+            if v == lang.capitalize():
+                languages[i] = k
+                break
+
+    return languages
+
+
+def resolve_environment_id(environment: str, workspace: Optional[str] = None) -> UUID:
+    """
+    Obtains the environment Id for a given environment.
+
+    Parameters
+    ----------
+    environment: str
+        Name of the environment.
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    from sempy_labs._environments import list_environments
+
+    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
+
+    dfE = list_environments(workspace=workspace)
+    dfE_filt = dfE[dfE["Environment Name"] == environment]
+    if len(dfE_filt) == 0:
+        raise ValueError(
+            f"{icons.red_dot} The '{environment}' environment does not exist within the '{workspace}' workspace."
+        )
+
+    return dfE_filt["Environment Id"].iloc[0]
+
+
+def get_azure_token_credentials(
+    key_vault_uri: str,
+    key_vault_tenant_id: str,
+    key_vault_client_id: str,
+    key_vault_client_secret: str,
+) -> Tuple[str, str, dict]:
+
+    from notebookutils import mssparkutils
+    from azure.identity import ClientSecretCredential
+
+    tenant_id = mssparkutils.credentials.getSecret(key_vault_uri, key_vault_tenant_id)
+    client_id = mssparkutils.credentials.getSecret(key_vault_uri, key_vault_client_id)
+    client_secret = mssparkutils.credentials.getSecret(
+        key_vault_uri, key_vault_client_secret
+    )
+
+    credential = ClientSecretCredential(
+        tenant_id=tenant_id, client_id=client_id, client_secret=client_secret
+    )
+
+    token = credential.get_token("https://management.azure.com/.default").token
+
+    headers = {
+        "Authorization": f"Bearer {token}",
+        "Content-Type": "application/json",
+    }
+
+    return token, credential, headers

sempy_labs/_icons.py

@@ -31,3 +31,46 @@ model_bpa_name = "ModelBPA"
 report_bpa_name = "ReportBPA"
 severity_mapping = {warning: "Warning", error: "Error", info: "Info"}
 special_characters = ['"', "/", '"', ":", "|", "<", ">", "*", "?", "'", "!"]
+
+language_map = {
+    "it-IT": "Italian",
+    "es-ES": "Spanish",
+    "he-IL": "Hebrew",
+    "pt-PT": "Portuguese",
+    "zh-CN": "Chinese",
+    "fr-FR": "French",
+    "da-DK": "Danish",
+    "cs-CZ": "Czech",
+    "de-DE": "German",
+    "el-GR": "Greek",
+    "fa-IR": "Persian",
+    "ga-IE": "Irish",
+    "hi-IN": "Hindi",
+    "hu-HU": "Hungarian",
+    "is-IS": "Icelandic",
+    "ja-JP": "Japanese",
+    "nl-NL": "Dutch",
+    "pl-PL": "Polish",
+    "pt-BR": "Portuguese",
+    "ru-RU": "Russian",
+    "te-IN": "Telugu",
+    "ta-IN": "Tamil",
+    "th-TH": "Thai",
+    "zu-ZA": "Zulu",
+    "am-ET": "Amharic",
+    "ar-AE": "Arabic",
+    "sv-SE": "Swedish",
+    "ko-KR": "Korean",
+    "id-ID": "Indonesian",
+    "mt-MT": "Maltese",
+    "ro-RO": "Romanian",
+    "sk-SK": "Slovak",
+    "sl-SL": "Slovenian",
+    "tr-TR": "Turkish",
+    "uk-UA": "Ukrainian",
+    "bg-BG": "Bulgarian",
+    "ca-ES": "Catalan",
+    "fi-FI": "Finnish",
+}
+workspace_roles = ["Admin", "Member", "Viewer", "Contributor"]
+principal_types = ["App", "Group", "None", "User"]