semantic-link-labs 0.9.1__py3-none-any.whl → 0.9.3__py3-none-any.whl

sempy_labs/_helper_functions.py

@@ -15,6 +15,8 @@ from azure.core.credentials import TokenCredential, AccessToken
 import urllib.parse
 import numpy as np
 from IPython.display import display, HTML
+import requests
+import sempy_labs._authentication as auth
 
 
 def _build_url(url: str, params: dict) -> str:
@@ -29,27 +31,35 @@ def _build_url(url: str, params: dict) -> str:
 
 
 def create_abfss_path(
-    lakehouse_id: UUID, lakehouse_workspace_id: UUID, delta_table_name: str
+    lakehouse_id: UUID,
+    lakehouse_workspace_id: UUID,
+    delta_table_name: Optional[str] = None,
 ) -> str:
     """
     Creates an abfss path for a delta table in a Fabric lakehouse.
 
     Parameters
     ----------
-    lakehouse_id : UUID
+    lakehouse_id : uuid.UUID
         ID of the Fabric lakehouse.
-    lakehouse_workspace_id : UUID
+    lakehouse_workspace_id : uuid.UUID
         ID of the Fabric workspace.
-    delta_table_name : str
+    delta_table_name : str, default=None
         Name of the delta table name.
 
     Returns
     -------
     str
-        An abfss path which can be used to save/reference a delta table in a Fabric lakehouse.
+        An abfss path which can be used to save/reference a delta table in a Fabric lakehouse or lakehouse.
     """
 
-    return f"abfss://{lakehouse_workspace_id}@onelake.dfs.fabric.microsoft.com/{lakehouse_id}/Tables/{delta_table_name}"
+    fp = _get_default_file_path()
+    path = f"abfss://{lakehouse_workspace_id}@{fp}/{lakehouse_id}"
+
+    if delta_table_name is not None:
+        path += f"/Tables/{delta_table_name}"
+
+    return path
 
 
 def _get_default_file_path() -> str:
@@ -166,6 +176,16 @@ def resolve_report_name(report_id: UUID, workspace: Optional[str | UUID] = None)
     )
 
 
+def resolve_item_id(
+    item: str | UUID, type: str, workspace: Optional[str] = None
+) -> UUID:
+
+    if _is_valid_uuid(item):
+        return item
+    else:
+        return fabric.resolve_item_id(item_name=item, type=type, workspace=workspace)
+
+
 def resolve_item_name_and_id(
     item: str | UUID, type: Optional[str] = None, workspace: Optional[str | UUID] = None
 ) -> Tuple[str, UUID]:
@@ -244,7 +264,7 @@ def resolve_dataset_id(
 
     Returns
     -------
-    UUID
+    uuid.UUID
         The ID of the semantic model.
     """
 
@@ -524,8 +544,9 @@ def save_as_delta_table(
             f"{icons.red_dot} Invalid 'delta_table_name'. Delta tables in the lakehouse cannot have spaces in their names."
         )
 
-    dataframe.columns = dataframe.columns.str.replace(" ", "_")
-    spark = SparkSession.builder.getOrCreate()
+    dataframe.columns = [col.replace(" ", "_") for col in dataframe.columns]
+
+    spark = _create_spark_session()
 
     type_mapping = {
         "string": StringType(),
@@ -719,7 +740,7 @@ def resolve_item_type(item_id: UUID, workspace: Optional[str | UUID] = None) ->
 
     if dfI_filt.empty:
         raise ValueError(
-            f"Invalid 'item_id' parameter. The '{item_id}' item was not found in the '{workspace_name}' workspace."
+            f"{icons.red_dot} Invalid 'item_id' parameter. The '{item_id}' item was not found in the '{workspace_name}' workspace."
         )
     return dfI_filt["Type"].iloc[0]
 
@@ -780,7 +801,7 @@ def resolve_workspace_capacity(
 
     Returns
     -------
-    Tuple[UUID, str]
+    Tuple[uuid.UUID, str]
         capacity Id; capacity came.
     """
 
@@ -811,7 +832,7 @@ def get_capacity_id(workspace: Optional[str | UUID] = None) -> UUID:
 
     Returns
     -------
-    UUID
+    uuid.UUID
         The capacity Id.
     """
 
@@ -849,7 +870,7 @@ def get_capacity_name(workspace: Optional[str | UUID] = None) -> str:
     capacity_id = get_capacity_id(workspace)
     dfC = fabric.list_capacities()
     dfC_filt = dfC[dfC["Id"] == capacity_id]
-    if len(dfC_filt) == 0:
+    if dfC_filt.empty:
         raise ValueError(
             f"{icons.red_dot} The '{capacity_id}' capacity Id does not exist."
         )
@@ -880,7 +901,7 @@ def resolve_capacity_name(capacity_id: Optional[UUID] = None) -> str:
     dfC = fabric.list_capacities()
     dfC_filt = dfC[dfC["Id"] == capacity_id]
 
-    if len(dfC_filt) == 0:
+    if dfC_filt.empty:
         raise ValueError(
             f"{icons.red_dot} The '{capacity_id}' capacity Id does not exist."
         )
@@ -901,7 +922,7 @@ def resolve_capacity_id(capacity_name: Optional[str] = None) -> UUID:
 
     Returns
     -------
-    UUID
+    uuid.UUID
         The capacity Id.
     """
 
@@ -911,7 +932,7 @@ def resolve_capacity_id(capacity_name: Optional[str] = None) -> UUID:
     dfC = fabric.list_capacities()
     dfC_filt = dfC[dfC["Display Name"] == capacity_name]
 
-    if len(dfC_filt) == 0:
+    if dfC_filt.empty:
         raise ValueError(
             f"{icons.red_dot} The '{capacity_name}' capacity does not exist."
         )
@@ -994,14 +1015,14 @@ def pagination(client, response):
     return responses
 
 
-def resolve_deployment_pipeline_id(deployment_pipeline: str) -> UUID:
+def resolve_deployment_pipeline_id(deployment_pipeline: str | UUID) -> UUID:
     """
     Obtains the Id for a given deployment pipeline.
 
     Parameters
     ----------
-    deployment_pipeline : str
-        The deployment pipeline name
+    deployment_pipeline : str | uuid.UUID
+        The deployment pipeline name or ID.
 
     Returns
     -------
@@ -1011,15 +1032,17 @@ def resolve_deployment_pipeline_id(deployment_pipeline: str) -> UUID:
 
     from sempy_labs._deployment_pipelines import list_deployment_pipelines
 
-    dfP = list_deployment_pipelines()
-    dfP_filt = dfP[dfP["Deployment Pipeline Name"] == deployment_pipeline]
-    if len(dfP_filt) == 0:
-        raise ValueError(
-            f"{icons.red_dot} The '{deployment_pipeline}' deployment pipeline is not valid."
-        )
-    deployment_pipeline_id = dfP_filt["Deployment Pipeline Id"].iloc[0]
+    if _is_valid_uuid(deployment_pipeline):
+        return deployment_pipeline
+    else:
 
-    return deployment_pipeline_id
+        dfP = list_deployment_pipelines()
+        dfP_filt = dfP[dfP["Deployment Pipeline Name"] == deployment_pipeline]
+        if len(dfP_filt) == 0:
+            raise ValueError(
+                f"{icons.red_dot} The '{deployment_pipeline}' deployment pipeline is not valid."
+            )
+        return dfP_filt["Deployment Pipeline Id"].iloc[0]
 
 
 class FabricTokenCredential(TokenCredential):
@@ -1209,7 +1232,7 @@ def resolve_notebook_id(notebook: str, workspace: Optional[str | UUID] = None) -
 
     Returns
     -------
-    UUID
+    uuid.UUID
         The notebook Id.
     """
 
@@ -1224,23 +1247,33 @@ def generate_guid():
 
 
 def _get_column_aggregate(
-    lakehouse: str,
     table_name: str,
     column_name: str = "RunId",
+    lakehouse: Optional[str | UUID] = None,
+    workspace: Optional[str | UUID] = None,
     function: str = "max",
     default_value: int = 0,
 ) -> int:
 
-    from pyspark.sql import SparkSession
+    from pyspark.sql.functions import approx_count_distinct
+    from pyspark.sql import functions as F
 
-    spark = SparkSession.builder.getOrCreate()
     function = function.upper()
-    query = f"SELECT {function}({column_name}) FROM {lakehouse}.{table_name}"
-    if "COUNT" in function and "DISTINCT" in function:
-        query = f"SELECT COUNT(DISTINCT({column_name})) FROM {lakehouse}.{table_name}"
-    dfSpark = spark.sql(query)
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    lakehouse_id = resolve_lakehouse_id(lakehouse, workspace)
+    path = create_abfss_path(lakehouse_id, workspace_id, table_name)
+
+    spark = _create_spark_session()
+    df = spark.read.format("delta").load(path)
 
-    return dfSpark.collect()[0][0] or default_value
+    if function in {"COUNTDISTINCT", "DISTINCTCOUNT"}:
+        result = df.select(F.count_distinct(F.col(column_name)))
+    elif "APPROX" in function:
+        result = df.select(approx_count_distinct(column_name))
+    else:
+        result = df.selectExpr(f"{function}({column_name})")
+
+    return result.collect()[0][0] or default_value
 
 
 def _make_list_unique(my_list):
@@ -1446,3 +1479,161 @@ def _get_fabric_context_setting(name: str):
 def get_tenant_id():
 
     _get_fabric_context_setting(name="trident.tenant.id")
+
+
+def _base_api(
+    request: str,
+    client: str = "fabric",
+    method: str = "get",
+    payload: Optional[str] = None,
+    status_codes: Optional[int] = 200,
+    uses_pagination: bool = False,
+    lro_return_json: bool = False,
+    lro_return_status_code: bool = False,
+):
+
+    from sempy_labs._authentication import _get_headers
+
+    if (lro_return_json or lro_return_status_code) and status_codes is None:
+        status_codes = [200, 202]
+
+    if isinstance(status_codes, int):
+        status_codes = [status_codes]
+
+    if client == "fabric":
+        c = fabric.FabricRestClient()
+    elif client == "fabric_sp":
+        c = fabric.FabricRestClient(token_provider=auth.token_provider.get())
+    elif client in ["azure", "graph"]:
+        pass
+    else:
+        raise ValueError(f"{icons.red_dot} The '{client}' client is not supported.")
+
+    if client not in ["azure", "graph"]:
+        if method == "get":
+            response = c.get(request)
+        elif method == "delete":
+            response = c.delete(request)
+        elif method == "post":
+            response = c.post(request, json=payload)
+        elif method == "patch":
+            response = c.patch(request, json=payload)
+        elif method == "put":
+            response = c.put(request, json=payload)
+        else:
+            raise NotImplementedError
+    else:
+        headers = _get_headers(auth.token_provider.get(), audience=client)
+        response = requests.request(
+            method.upper(),
+            f"https://graph.microsoft.com/v1.0/{request}",
+            headers=headers,
+            json=payload,
+        )
+
+    if lro_return_json:
+        return lro(c, response, status_codes).json()
+    elif lro_return_status_code:
+        return lro(c, response, status_codes, return_status_code=True)
+    else:
+        if response.status_code not in status_codes:
+            raise FabricHTTPException(response)
+        if uses_pagination:
+            responses = pagination(c, response)
+            return responses
+        else:
+            return response
+
+
+def _create_dataframe(columns: dict) -> pd.DataFrame:
+
+    return pd.DataFrame(columns=list(columns.keys()))
+
+
+def _update_dataframe_datatypes(dataframe: pd.DataFrame, column_map: dict):
+    """
+    Updates the datatypes of columns in a pandas dataframe based on a column map.
+
+    Example:
+    {
+        "Order": "int",
+        "Public": "bool",
+    }
+    """
+
+    for column, data_type in column_map.items():
+        if column in dataframe.columns:
+            if data_type == "int":
+                dataframe[column] = dataframe[column].astype(int)
+            elif data_type == "bool":
+                dataframe[column] = dataframe[column].astype(bool)
+            elif data_type == "float":
+                dataframe[column] = dataframe[column].astype(float)
+            elif data_type == "datetime":
+                dataframe[column] = pd.to_datetime(dataframe[column])
+            # This is for a special case in admin.list_reports where datetime itself does not work. Coerce fixes the issue.
+            elif data_type == "datetime_coerce":
+                dataframe[column] = pd.to_datetime(dataframe[column], errors="coerce")
+            # This is for list_synonyms since the weight column is float and can have NaN values.
+            elif data_type == "float_fillna":
+                dataframe[column] = dataframe[column].fillna(0).astype(float)
+            # This is to avoid NaN values in integer columns (for delta analyzer)
+            elif data_type == "int_fillna":
+                dataframe[column] = dataframe[column].fillna(0).astype(int)
+            elif data_type in ["str", "string"]:
+                dataframe[column] = dataframe[column].astype(str)
+            else:
+                raise NotImplementedError
+
+
+def _print_success(item_name, item_type, workspace_name, action="created"):
+    if action == "created":
+        print(
+            f"{icons.green_dot} The '{item_name}' {item_type} has been successfully created in the '{workspace_name}' workspace."
+        )
+    elif action == "deleted":
+        print(
+            f"{icons.green_dot} The '{item_name}' {item_type} has been successfully deleted from the '{workspace_name}' workspace."
+        )
+    else:
+        raise NotImplementedError
+
+
+def _pure_python_notebook() -> bool:
+
+    from sempy.fabric._environment import _on_jupyter
+
+    return _on_jupyter()
+
+
+def _create_spark_session():
+
+    if _pure_python_notebook():
+        raise ValueError(
+            f"{icons.red_dot} This function is only available in a PySpark notebook."
+        )
+
+    from pyspark.sql import SparkSession
+
+    return SparkSession.builder.getOrCreate()
+
+
+def _read_delta_table(path: str):
+
+    spark = _create_spark_session()
+
+    return spark.read.format("delta").load(path)
+
+
+def _delta_table_row_count(table_name: str) -> int:
+
+    spark = _create_spark_session()
+
+    return spark.table(table_name).count()
+
+
+def _run_spark_sql_query(query):
+
+    spark = _create_spark_session()
+
+    return spark.sql(query)

sempy_labs/_job_scheduler.py

@@ -1,14 +1,13 @@
-import sempy.fabric as fabric
 from sempy._utils._log import log
 import pandas as pd
 from typing import Optional
 from sempy_labs._helper_functions import (
     resolve_workspace_name_and_id,
     resolve_item_name_and_id,
-    pagination,
-    lro,
+    _update_dataframe_datatypes,
+    _base_api,
+    _create_dataframe,
 )
-from sempy.fabric.exceptions import FabricHTTPException
 from uuid import UUID
 import sempy_labs._icons as icons
 
@@ -44,32 +43,26 @@ def list_item_job_instances(
         item=item, type=type, workspace=workspace
     )
 
-    client = fabric.FabricRestClient()
-    response = client.get(
-        f"v1/workspaces/{workspace_id}/items/{item_id}/jobs/instances"
+    columns = {
+        "Job Instance Id": "string",
+        "Item Name": "string",
+        "Item Id": "string",
+        "Item Type": "string",
+        "Job Type": "string",
+        "Invoke Type": "string",
+        "Status": "string",
+        "Root Activity Id": "string",
+        "Start Time UTC": "datetime",
+        "End Time UTC": "string",
+        "Error Message": "string",
+    }
+    df = _create_dataframe(columns=columns)
+
+    responses = _base_api(
+        request=f"v1/workspaces/{workspace_id}/items/{item_id}/jobs/instances",
+        uses_pagination=True,
     )
 
-    if response.status_code != 200:
-        raise FabricHTTPException(response)
-
-    df = pd.DataFrame(
-        columns=[
-            "Job Instance Id",
-            "Item Name",
-            "Item Id",
-            "Item Type",
-            "Job Type",
-            "Invoke Type",
-            "Status",
-            "Root Activity Id",
-            "Start Time UTC",
-            "End Time UTC",
-            "Failure Reason",
-        ]
-    )
-
-    responses = pagination(client, response)
-
     if not responses[0].get("value"):
         return df
 
@@ -95,6 +88,8 @@ def list_item_job_instances(
     if dfs:
         df = pd.concat(dfs, ignore_index=True)
 
+    df = _update_dataframe_datatypes(dataframe=df, column_map=columns)
+
     return df
 
 
@@ -134,31 +129,26 @@ def list_item_schedules(
         item=item, type=type, workspace=workspace
     )
 
-    df = pd.DataFrame(
-        columns=[
-            "Job Schedule Id",
-            "Enabled",
-            "Created Date Time",
-            "Start Date Time",
-            "End Date Time",
-            "Local Time Zone Id",
-            "Type",
-            "Interval",
-            "Weekdays",
-            "Times",
-            "Owner Id",
-            "Owner Type",
-        ]
+    columns = {
+        "Job Schedule Id": "string",
+        "Enabled": "bool",
+        "Created Date Time": "datetime",
+        "Start Date Time": "datetime",
+        "End Date Time": "string",
+        "Local Time Zone Id": "string",
+        "Type": "string",
+        "Interval": "string",
+        "Weekdays": "string",
+        "Times": "string",
+        "Owner Id": "string",
+        "Owner Type": "string",
+    }
+    df = _create_dataframe(columns=columns)
+
+    response = _base_api(
+        request=f"v1/workspaces/{workspace_id}/items/{item_id}/jobs/{job_type}/schedules"
     )
 
-    client = fabric.FabricRestClient()
-    response = client.get(
-        f"v1/workspaces/{workspace_id}/items/{item_id}/jobs/{job_type}/schedules"
-    )
-
-    if response.status_code != 200:
-        raise FabricHTTPException(response)
-
     for v in response.json().get("value", []):
         config = v.get("configuration", {})
         own = v.get("owner", {})
@@ -179,9 +169,7 @@ def list_item_schedules(
 
         df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
 
-    df["Enabled"] = df["Enabled"].astype(bool)
-    df["Created Date Time"] = pd.to_datetime(df["Created Date Time"])
-    df["Start Date Time"] = pd.to_datetime(df["Start Date Time"])
+    _update_dataframe_datatypes(dataframe=df, column_map=columns)
 
     return df
 
@@ -217,11 +205,11 @@ def run_on_demand_item_job(
         item=item, type=type, workspace=workspace
     )
 
-    client = fabric.FabricRestClient()
-    response = client.post(
-        f"v1/workspaces/{workspace_id}/items/{item_id}/jobs/instances?jobType={job_type}"
+    _base_api(
+        request=f"v1/workspaces/{workspace_id}/items/{item_id}/jobs/instances?jobType={job_type}",
+        method="post",
+        lro_return_status_code=True,
+        status_codes=202,
     )
 
-    lro(client, response, return_status_code=True)
-
     print(f"{icons.green_dot} The '{item_name}' {type.lower()} has been executed.")

sempy_labs/_kql_databases.py

@@ -4,10 +4,9 @@ import sempy_labs._icons as icons
 from typing import Optional
 from sempy_labs._helper_functions import (
     resolve_workspace_name_and_id,
-    lro,
-    pagination,
+    _base_api,
+    _create_dataframe,
 )
-from sempy.fabric.exceptions import FabricHTTPException
 from uuid import UUID
 
 
@@ -30,26 +29,22 @@ def list_kql_databases(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
         A pandas dataframe showing the KQL databases within a workspace.
     """
 
-    df = pd.DataFrame(
-        columns=[
-            "KQL Database Name",
-            "KQL Database Id",
-            "Description",
-            "Parent Eventhouse Item Id",
-            "Query Service URI",
-            "Ingestion Service URI",
-            "Database Type",
-        ]
-    )
+    columns = {
+        "KQL Database Name": "string",
+        "KQL Database Id": "string",
+        "Description": "string",
+        "Parent Eventhouse Item Id": "string",
+        "Query Service URI": "string",
+        "Ingestion Service URI": "string",
+        "Database Type": "string",
+    }
+    df = _create_dataframe(columns=columns)
 
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
 
-    client = fabric.FabricRestClient()
-    response = client.get(f"/v1/workspaces/{workspace_id}/kqlDatabases")
-    if response.status_code != 200:
-        raise FabricHTTPException(response)
-
-    responses = pagination(client, response)
+    responses = _base_api(
+        request=f"v1/workspaces/{workspace_id}/kqlDatabases", uses_pagination=True
+    )
 
     for r in responses:
         for v in r.get("value", []):
@@ -91,18 +86,19 @@ def create_kql_database(
 
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
 
-    request_body = {"displayName": name}
+    payload = {"displayName": name}
 
     if description:
-        request_body["description"] = description
-
-    client = fabric.FabricRestClient()
-    response = client.post(
-        f"/v1/workspaces/{workspace_id}/kqlDatabases", json=request_body
+        payload["description"] = description
+
+    _base_api(
+        request=f"v1/workspaces/{workspace_id}/kqlDatabases",
+        method="post",
+        payload=payload,
+        status_codes=[201, 202],
+        lro_return_status_code=True,
     )
 
-    lro(client, response, status_codes=[201, 202])
-
     print(
         f"{icons.green_dot} The '{name}' KQL database has been created within the '{workspace_name}' workspace."
     )
@@ -129,13 +125,10 @@ def delete_kql_database(name: str, workspace: Optional[str | UUID] = None):
         item_name=name, type="KQLDatabase", workspace=workspace_id
     )
 
-    client = fabric.FabricRestClient()
-    response = client.delete(
-        f"/v1/workspaces/{workspace_id}/kqlDatabases/{kql_database_id}"
+    _base_api(
+        request=f"/v1/workspaces/{workspace_id}/kqlDatabases/{kql_database_id}",
+        method="delete",
     )
-
-    if response.status_code != 200:
-        raise FabricHTTPException(response)
     print(
         f"{icons.green_dot} The '{name}' KQL database within the '{workspace_name}' workspace has been deleted."
     )