semantic-link-labs 0.7.4__py3-none-any.whl → 0.8.1__py3-none-any.whl

sempy_labs/_helper_functions.py

@@ -2,21 +2,23 @@ import sempy.fabric as fabric
 import re
 import json
 import base64
+import time
+import uuid
+from sempy.fabric.exceptions import FabricHTTPException
 import pandas as pd
 from functools import wraps
 import datetime
-import time
 from typing import Optional, Tuple, List
 from uuid import UUID
 import sempy_labs._icons as icons
-from sempy.fabric.exceptions import FabricHTTPException
 import urllib.parse
 from azure.core.credentials import TokenCredential, AccessToken
+import deltalake
 
 
 def create_abfss_path(
     lakehouse_id: UUID, lakehouse_workspace_id: UUID, delta_table_name: str
-):
+) -> str:
     """
     Creates an abfss path for a delta table in a Fabric lakehouse.
 
@@ -38,7 +40,7 @@ def create_abfss_path(
     return f"abfss://{lakehouse_workspace_id}@onelake.dfs.fabric.microsoft.com/{lakehouse_id}/Tables/{delta_table_name}"
 
 
-def format_dax_object_name(table: str, column: str):
+def format_dax_object_name(table: str, column: str) -> str:
     """
     Formats a table/column combination to the 'Table Name'[Column Name] format.
 
@@ -60,7 +62,7 @@ def format_dax_object_name(table: str, column: str):
 
 def create_relationship_name(
     from_table: str, from_column: str, to_table: str, to_column: str
-):
+) -> str:
     """
     Formats a relationship's table/columns into a fully qualified name.
 
@@ -88,7 +90,7 @@ def create_relationship_name(
     )
 
 
-def resolve_report_id(report: str, workspace: Optional[str] = None):
+def resolve_report_id(report: str, workspace: Optional[str] = None) -> UUID:
     """
     Obtains the ID of the Power BI report.
 
@@ -116,7 +118,7 @@ def resolve_report_id(report: str, workspace: Optional[str] = None):
     return obj
 
 
-def resolve_report_name(report_id: UUID, workspace: Optional[str] = None):
+def resolve_report_name(report_id: UUID, workspace: Optional[str] = None) -> str:
     """
     Obtains the name of the Power BI report.
 
@@ -146,7 +148,7 @@ def resolve_report_name(report_id: UUID, workspace: Optional[str] = None):
     return obj
 
 
-def resolve_dataset_id(dataset: str, workspace: Optional[str] = None):
+def resolve_dataset_id(dataset: str, workspace: Optional[str] = None) -> UUID:
     """
     Obtains the ID of the semantic model.
 
@@ -176,7 +178,7 @@ def resolve_dataset_id(dataset: str, workspace: Optional[str] = None):
     return obj
 
 
-def resolve_dataset_name(dataset_id: UUID, workspace: Optional[str] = None):
+def resolve_dataset_name(dataset_id: UUID, workspace: Optional[str] = None) -> str:
     """
     Obtains the name of the semantic model.
 
@@ -208,7 +210,7 @@ def resolve_dataset_name(dataset_id: UUID, workspace: Optional[str] = None):
 
 def resolve_lakehouse_name(
     lakehouse_id: Optional[UUID] = None, workspace: Optional[str] = None
-):
+) -> str:
     """
     Obtains the name of the Fabric lakehouse.
 
@@ -242,7 +244,7 @@ def resolve_lakehouse_name(
     return obj
 
 
-def resolve_lakehouse_id(lakehouse: str, workspace: Optional[str] = None):
+def resolve_lakehouse_id(lakehouse: str, workspace: Optional[str] = None) -> UUID:
     """
     Obtains the ID of the Fabric lakehouse.
 
@@ -321,7 +323,7 @@ def get_direct_lake_sql_endpoint(dataset: str, workspace: Optional[str] = None)
         return sqlEndpointId
 
 
-def generate_embedded_filter(filter: str):
+def generate_embedded_filter(filter: str) -> str:
     """
     Converts the filter expression to a filter expression which can be used by a Power BI embedded URL.
 
@@ -390,7 +392,7 @@ def save_as_delta_table(
     dataframe,
     delta_table_name: str,
     write_mode: str,
-    merge_schema: Optional[bool] = False,
+    merge_schema: bool = False,
     schema: Optional[dict] = None,
     lakehouse: Optional[str] = None,
     workspace: Optional[str] = None,
@@ -869,7 +871,7 @@ def lro(
     response,
     status_codes: Optional[List[str]] = [200, 202],
     sleep_time: Optional[int] = 1,
-    return_status_code: Optional[bool] = False,
+    return_status_code: bool = False,
 ):
 
     if response.status_code not in status_codes:
@@ -922,6 +924,19 @@ def pagination(client, response):
 
 
 def resolve_deployment_pipeline_id(deployment_pipeline: str) -> UUID:
+    """
+    Obtains the Id for a given deployment pipeline.
+
+    Parameters
+    ----------
+    deployment_pipeline : str
+        The deployment pipeline name
+
+    Returns
+    -------
+    UUID
+        The deployment pipeline Id.
+    """
 
     from sempy_labs._deployment_pipelines import list_deployment_pipelines
 
@@ -943,7 +958,7 @@ class FabricTokenCredential(TokenCredential):
         scopes: str,
         claims: Optional[str] = None,
         tenant_id: Optional[str] = None,
-        enable_cae: Optional[bool] = False,
+        enable_cae: bool = False,
         **kwargs: any,
     ) -> AccessToken:
 
@@ -968,15 +983,26 @@ def get_adls_client(account_name):
     return service_client
 
 
-def resolve_warehouse_id(warehouse: str, workspace: Optional[str]):
+def resolve_warehouse_id(warehouse: str, workspace: Optional[str]) -> UUID:
+    """
+    Obtains the Id for a given warehouse.
+
+    Parameters
+    ----------
+    warehouse : str
+        The warehouse name
+
+    Returns
+    -------
+    UUID
+        The warehouse Id.
+    """
 
     workspace = fabric.resolve_workspace_name(workspace)
-    warehouse_id = fabric.resolve_item_id(
+    return fabric.resolve_item_id(
         item_name=warehouse, type="Warehouse", workspace=workspace
     )
 
-    return warehouse_id
-
 
 def get_language_codes(languages: str | List[str]):
 
@@ -992,34 +1018,6 @@ def get_language_codes(languages: str | List[str]):
     return languages
 
 
-def resolve_environment_id(environment: str, workspace: Optional[str] = None) -> UUID:
-    """
-    Obtains the environment Id for a given environment.
-
-    Parameters
-    ----------
-    environment: str
-        Name of the environment.
-    workspace : str, default=None
-        The Fabric workspace name.
-        Defaults to None which resolves to the workspace of the attached lakehouse
-        or if no lakehouse attached, resolves to the workspace of the notebook.
-    """
-
-    from sempy_labs._environments import list_environments
-
-    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
-
-    dfE = list_environments(workspace=workspace)
-    dfE_filt = dfE[dfE["Environment Name"] == environment]
-    if len(dfE_filt) == 0:
-        raise ValueError(
-            f"{icons.red_dot} The '{environment}' environment does not exist within the '{workspace}' workspace."
-        )
-
-    return dfE_filt["Environment Id"].iloc[0]
-
-
 def get_azure_token_credentials(
     key_vault_uri: str,
     key_vault_tenant_id: str,
@@ -1048,3 +1046,98 @@ def get_azure_token_credentials(
     }
 
     return token, credential, headers
+
+
+def convert_to_alphanumeric_lowercase(input_string):
+
+    # Removes non-alphanumeric characters
+    cleaned_string = re.sub(r"[^a-zA-Z0-9]", "", input_string)
+    cleaned_string = cleaned_string.lower()
+
+    return cleaned_string
+
+
+def resolve_environment_id(environment: str, workspace: Optional[str] = None) -> UUID:
+    """
+    Obtains the environment Id for a given environment.
+
+    Parameters
+    ----------
+    environment: str
+        Name of the environment.
+
+    Returns
+    -------
+    UUID
+        The environment Id.
+    """
+
+    workspace = fabric.resolve_workspace_name(workspace)
+    return fabric.resolve_item_id(
+        item_name=environment, type="Environment", workspace=workspace
+    )
+
+
+def make_clickable(val):
+
+    return f'<a target="_blank" href="{val}">{val}</a>'
+
+
+def convert_to_friendly_case(text: str) -> str:
+    """
+    Converts a string of pascal/camel/snake case to business-friendly case.
+
+    Parameters
+    ----------
+    text : str
+        The text to convert.
+
+    Returns
+    -------
+    str
+        Text converted into a business-friendly text.
+    """
+    if text is not None:
+        text = text.replace("_", " ")
+        # Insert space before each capital letter, avoiding double spaces
+        text = re.sub(r"(?<!\s)(?=[A-Z])", " ", text)
+        # Strip leading/trailing whitespace and capitalize the first letter of each word
+        text = text.strip().title()
+
+    return text
+
+
+def resolve_notebook_id(notebook: str, workspace: Optional[str] = None) -> UUID:
+    """
+    Obtains the notebook Id for a given notebook.
+
+    Parameters
+    ----------
+    notebook: str
+        Name of the notebook.
+
+    Returns
+    -------
+    UUID
+        The notebook Id.
+    """
+
+    workspace = fabric.resolve_workspace_name(workspace)
+    return fabric.resolve_item_id(
+        item_name=notebook, type="Notebook", workspace=workspace
+    )
+
+
+def generate_guid():
+
+    return str(uuid.uuid4())
+
+
+def get_max_run_id(table_name: str) -> int:
+
+    table_path = f"/lakehouse/default/Tables/{table_name}/"
+    delta_table = deltalake.DeltaTable(table_path)
+    data = delta_table.to_pandas()
+    max_run_id = data["RunId"].max()
+
+    return max_run_id

sempy_labs/_icons.py

@@ -74,3 +74,43 @@ language_map = {
 }
 workspace_roles = ["Admin", "Member", "Viewer", "Contributor"]
 principal_types = ["App", "Group", "None", "User"]
+azure_api_version = "2023-11-01"
+migrate_capacity_suffix = "fsku"
+sku_mapping = {
+    "A1": "F8",
+    "EM1": "F8",
+    "A2": "F16",
+    "EM2": "F16",
+    "A3": "F32",
+    "EM3": "F32",
+    "A4": "F64",
+    "P1": "F64",
+    "A5": "F128",
+    "P2": "F128",
+    "A6": "F256",
+    "P3": "F256",
+    "A7": "F512",
+    "P4": "F512",
+    "P5": "F1024",
+}
+refreshTypes = [
+    "full",
+    "automatic",
+    "dataOnly",
+    "calculate",
+    "clearValues",
+    "defragment",
+]
+
+itemTypes = {
+    "DataPipeline": "dataPipelines",
+    "Eventstream": "eventstreams",
+    "KQLDatabase": "kqlDatabases",
+    "KQLQueryset": "kqlQuerysets",
+    "Lakehouse": "lakehouses",
+    "MLExperiment": "mlExperiments",
+    "MLModel": "mlModels",
+    "Notebook": "notebooks",
+    "Warehouse": "warehouses",
+}
+default_schema = "dbo"

sempy_labs/_kql_databases.py

@@ -0,0 +1,134 @@
+import sempy.fabric as fabric
+import pandas as pd
+import sempy_labs._icons as icons
+from typing import Optional
+from sempy_labs._helper_functions import (
+    resolve_workspace_name_and_id,
+    lro,
+    pagination,
+)
+from sempy.fabric.exceptions import FabricHTTPException
+
+
+def list_kql_databases(workspace: Optional[str] = None) -> pd.DataFrame:
+    """
+    Shows the KQL databases within a workspace.
+
+    Parameters
+    ----------
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing the KQL databases within a workspace.
+    """
+
+    df = pd.DataFrame(
+        columns=[
+            "KQL Database Name",
+            "KQL Database Id",
+            "Description",
+            "Parent Eventhouse Item Id",
+            "Query Service URI",
+            "Ingestion Service URI",
+            "Database Type",
+        ]
+    )
+
+    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
+
+    client = fabric.FabricRestClient()
+    response = client.get(f"/v1/workspaces/{workspace_id}/kqlDatabases")
+    if response.status_code != 200:
+        raise FabricHTTPException(response)
+
+    responses = pagination(client, response)
+
+    for r in responses:
+        for v in r.get("value", []):
+            prop = v.get("properties", {})
+
+            new_data = {
+                "KQL Database Name": v.get("displayName"),
+                "KQL Database Id": v.get("id"),
+                "Description": v.get("description"),
+                "Parent Eventhouse Item Id": prop.get("parentEventhouseItemId"),
+                "Query Service URI": prop.get("queryServiceUri"),
+                "Ingestion Service URI": prop.get("ingestionServiceUri"),
+                "Database Type": prop.get("databaseType"),
+            }
+            df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
+
+    return df
+
+
+def create_kql_database(
+    name: str, description: Optional[str] = None, workspace: Optional[str] = None
+):
+    """
+    Creates a KQL database.
+
+    Parameters
+    ----------
+    name: str
+        Name of the KQL database.
+    description : str, default=None
+        A description of the environment.
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
+
+    request_body = {"displayName": name}
+
+    if description:
+        request_body["description"] = description
+
+    client = fabric.FabricRestClient()
+    response = client.post(
+        f"/v1/workspaces/{workspace_id}/kqlDatabases", json=request_body
+    )
+
+    lro(client, response, status_codes=[201, 202])
+
+    print(
+        f"{icons.green_dot} The '{name}' KQL database has been created within the '{workspace}' workspace."
+    )
+
+
+def delete_kql_database(name: str, workspace: Optional[str] = None):
+    """
+    Deletes a KQL database.
+
+    Parameters
+    ----------
+    name: str
+        Name of the KQL database.
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
+    kql_database_id = fabric.resolve_item_id(
+        item_name=name, type="KQLDatabase", workspace=workspace
+    )
+
+    client = fabric.FabricRestClient()
+    response = client.delete(
+        f"/v1/workspaces/{workspace_id}/kqlDatabases/{kql_database_id}"
+    )
+
+    if response.status_code != 200:
+        raise FabricHTTPException(response)
+    print(
+        f"{icons.green_dot} The '{name}' KQL database within the '{workspace}' workspace has been deleted."
+    )

sempy_labs/_kql_querysets.py

@@ -0,0 +1,124 @@
+import sempy.fabric as fabric
+import pandas as pd
+import sempy_labs._icons as icons
+from typing import Optional
+from sempy_labs._helper_functions import (
+    resolve_workspace_name_and_id,
+    lro,
+    pagination,
+)
+from sempy.fabric.exceptions import FabricHTTPException
+
+
+def list_kql_querysets(workspace: Optional[str] = None) -> pd.DataFrame:
+    """
+    Shows the KQL querysets within a workspace.
+
+    Parameters
+    ----------
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing the KQL querysets within a workspace.
+    """
+
+    df = pd.DataFrame(
+        columns=[
+            "KQL Queryset Name",
+            "KQL Queryset Id",
+            "Description",
+        ]
+    )
+
+    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
+
+    client = fabric.FabricRestClient()
+    response = client.get(f"/v1/workspaces/{workspace_id}/kqlQuerysets")
+    if response.status_code != 200:
+        raise FabricHTTPException(response)
+
+    responses = pagination(client, response)
+
+    for r in responses:
+        for v in r.get("value", []):
+            new_data = {
+                "KQL Queryset Name": v.get("displayName"),
+                "KQL Queryset Id": v.get("id"),
+                "Description": v.get("description"),
+            }
+            df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
+
+    return df
+
+
+def create_kql_queryset(
+    name: str, description: Optional[str] = None, workspace: Optional[str] = None
+):
+    """
+    Creates a KQL queryset.
+
+    Parameters
+    ----------
+    name: str
+        Name of the KQL queryset.
+    description : str, default=None
+        A description of the environment.
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
+
+    request_body = {"displayName": name}
+
+    if description:
+        request_body["description"] = description
+
+    client = fabric.FabricRestClient()
+    response = client.post(
+        f"/v1/workspaces/{workspace_id}/kqlQuerysets", json=request_body
+    )
+
+    lro(client, response, status_codes=[201, 202])
+
+    print(
+        f"{icons.green_dot} The '{name}' KQL queryset has been created within the '{workspace}' workspace."
+    )
+
+
+def delete_kql_queryset(name: str, workspace: Optional[str] = None):
+    """
+    Deletes a KQL queryset.
+
+    Parameters
+    ----------
+    name: str
+        Name of the KQL queryset.
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
+    kql_database_id = fabric.resolve_item_id(
+        item_name=name, type="KQLQueryset", workspace=workspace
+    )
+
+    client = fabric.FabricRestClient()
+    response = client.delete(
+        f"/v1/workspaces/{workspace_id}/kqlQuerysets/{kql_database_id}"
+    )
+
+    if response.status_code != 200:
+        raise FabricHTTPException(response)
+    print(
+        f"{icons.green_dot} The '{name}' KQL queryset within the '{workspace}' workspace has been deleted."
+    )