semantic-link-labs 0.9.0__py3-none-any.whl → 0.9.2__py3-none-any.whl

sempy_labs/_spark.py

@@ -1,11 +1,12 @@
-import sempy.fabric as fabric
 import pandas as pd
 import sempy_labs._icons as icons
 from typing import Optional
 from sempy_labs._helper_functions import (
     resolve_workspace_name_and_id,
+    _update_dataframe_datatypes,
+    _base_api,
+    _create_dataframe,
 )
-from sempy.fabric.exceptions import FabricHTTPException
 from uuid import UUID
 
 
@@ -30,26 +31,22 @@ def list_custom_pools(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
 
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
 
-    df = pd.DataFrame(
-        columns=[
-            "Custom Pool ID",
-            "Custom Pool Name",
-            "Type",
-            "Node Family",
-            "Node Size",
-            "Auto Scale Enabled",
-            "Auto Scale Min Node Count",
-            "Auto Scale Max Node Count",
-            "Dynamic Executor Allocation Enabled",
-            "Dynamic Executor Allocation Min Executors",
-            "Dynamic Executor Allocation Max Executors",
-        ]
-    )
+    columns = {
+        "Custom Pool ID": "string",
+        "Custom Pool Name": "string",
+        "Type": "string",
+        "Node Family": "string",
+        "Node Size": "string",
+        "Auto Scale Enabled": "bool",
+        "Auto Scale Min Node Count": "int",
+        "Auto Scale Max Node Count": "int",
+        "Dynamic Executor Allocation Enabled": "bool",
+        "Dynamic Executor Allocation Min Executors": "int",
+        "Dynamic Executor Allocation Max Executors": "int",
+    }
+    df = _create_dataframe(columns=columns)
 
-    client = fabric.FabricRestClient()
-    response = client.get(f"/v1/workspaces/{workspace_id}/spark/pools")
-    if response.status_code != 200:
-        raise FabricHTTPException(response)
+    response = _base_api(request=f"/v1/workspaces/{workspace_id}/spark/pools")
 
     for i in response.json()["value"]:
 
@@ -71,17 +68,7 @@ def list_custom_pools(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
         }
         df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
 
-    bool_cols = ["Auto Scale Enabled", "Dynamic Executor Allocation Enabled"]
-    int_cols = [
-        "Auto Scale Min Node Count",
-        "Auto Scale Max Node Count",
-        "Dynamic Executor Allocation Enabled",
-        "Dynamic Executor Allocation Min Executors",
-        "Dynamic Executor Allocation Max Executors",
-    ]
-
-    df[bool_cols] = df[bool_cols].astype(bool)
-    df[int_cols] = df[int_cols].astype(int)
+    _update_dataframe_datatypes(dataframe=df, column_map=columns)
 
     return df
 
@@ -131,7 +118,7 @@ def create_custom_pool(
 
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
 
-    request_body = {
+    payload = {
         "name": pool_name,
         "nodeFamily": node_family,
         "nodeSize": node_size,
@@ -147,13 +134,12 @@ def create_custom_pool(
         },
     }
 
-    client = fabric.FabricRestClient()
-    response = client.post(
-        f"/v1/workspaces/{workspace_id}/spark/pools", json=request_body
+    _base_api(
+        request=f"/v1/workspaces/{workspace_id}/spark/pools",
+        payload=payload,
+        method="post",
+        status_codes=201,
     )
-
-    if response.status_code != 201:
-        raise FabricHTTPException(response)
     print(
         f"{icons.green_dot} The '{pool_name}' spark pool has been created within the '{workspace_name}' workspace."
     )
@@ -239,7 +225,7 @@ def update_custom_pool(
     if max_executors is None:
         max_executors = int(df_pool["Max Executors"].iloc[0])
 
-    request_body = {
+    payload = {
         "name": pool_name,
         "nodeFamily": node_family,
         "nodeSize": node_size,
@@ -255,13 +241,11 @@ def update_custom_pool(
         },
     }
 
-    client = fabric.FabricRestClient()
-    response = client.post(
-        f"/v1/workspaces/{workspace_id}/spark/pools", json=request_body
+    _base_api(
+        request=f"/v1/workspaces/{workspace_id}/spark/pools",
+        payload=payload,
+        method="post",
     )
-
-    if response.status_code != 200:
-        raise FabricHTTPException(response)
     print(
         f"{icons.green_dot} The '{pool_name}' spark pool within the '{workspace_name}' workspace has been updated."
     )
@@ -288,17 +272,15 @@ def delete_custom_pool(pool_name: str, workspace: Optional[str | UUID] = None):
     dfL = list_custom_pools(workspace=workspace_id)
     dfL_filt = dfL[dfL["Custom Pool Name"] == pool_name]
 
-    if len(dfL_filt) == 0:
+    if dfL_filt.empty:
         raise ValueError(
             f"{icons.red_dot} The '{pool_name}' custom pool does not exist within the '{workspace_name}' workspace."
         )
     pool_id = dfL_filt["Custom Pool ID"].iloc[0]
 
-    client = fabric.FabricRestClient()
-    response = client.delete(f"/v1/workspaces/{workspace_id}/spark/pools/{pool_id}")
-
-    if response.status_code != 200:
-        raise FabricHTTPException(response)
+    _base_api(
+        request=f"/v1/workspaces/{workspace_id}/spark/pools/{pool_id}", method="delete"
+    )
     print(
         f"{icons.green_dot} The '{pool_name}' spark pool has been deleted from the '{workspace_name}' workspace."
     )
@@ -343,10 +325,7 @@ def get_spark_settings(
         ]
     )
 
-    client = fabric.FabricRestClient()
-    response = client.get(f"/v1/workspaces/{workspace_id}/spark/settings")
-    if response.status_code != 200:
-        raise FabricHTTPException(response)
+    response = _base_api(request=f"/v1/workspaces/{workspace_id}/spark/settings")
 
     i = response.json()
     p = i.get("pool")
@@ -369,15 +348,13 @@ def get_spark_settings(
     }
     df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
 
-    bool_cols = [
-        "Automatic Log Enabled",
-        "High Concurrency Enabled",
-        "Customize Compute Enabled",
-    ]
-    # int_cols = ["Max Node Count", "Max Executors"]
+    column_map = {
+        "Automatic Log Enabled": "bool",
+        "High Concurrency Enabled": "bool",
+        "Customize Compute Enabled": "bool",
+    }
 
-    df[bool_cols] = df[bool_cols].astype(bool)
-    # df[int_cols] = df[int_cols].astype(int)
+    _update_dataframe_datatypes(dataframe=df, column_map=column_map)
 
     if return_dataframe:
         return df
@@ -435,34 +412,32 @@ def update_spark_settings(
 
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
 
-    request_body = get_spark_settings(workspace=workspace, return_dataframe=False)
+    payload = get_spark_settings(workspace=workspace, return_dataframe=False)
 
     if automatic_log_enabled is not None:
-        request_body["automaticLog"]["enabled"] = automatic_log_enabled
+        payload["automaticLog"]["enabled"] = automatic_log_enabled
     if high_concurrency_enabled is not None:
-        request_body["highConcurrency"][
+        payload["highConcurrency"][
             "notebookInteractiveRunEnabled"
         ] = high_concurrency_enabled
     if customize_compute_enabled is not None:
-        request_body["pool"]["customizeComputeEnabled"] = customize_compute_enabled
+        payload["pool"]["customizeComputeEnabled"] = customize_compute_enabled
     if default_pool_name is not None:
-        request_body["pool"]["defaultPool"]["name"] = default_pool_name
+        payload["pool"]["defaultPool"]["name"] = default_pool_name
     if max_node_count is not None:
-        request_body["pool"]["starterPool"]["maxNodeCount"] = max_node_count
+        payload["pool"]["starterPool"]["maxNodeCount"] = max_node_count
     if max_executors is not None:
-        request_body["pool"]["starterPool"]["maxExecutors"] = max_executors
+        payload["pool"]["starterPool"]["maxExecutors"] = max_executors
     if environment_name is not None:
-        request_body["environment"]["name"] = environment_name
+        payload["environment"]["name"] = environment_name
     if runtime_version is not None:
-        request_body["environment"]["runtimeVersion"] = runtime_version
+        payload["environment"]["runtimeVersion"] = runtime_version
 
-    client = fabric.FabricRestClient()
-    response = client.patch(
-        f"/v1/workspaces/{workspace_id}/spark/settings", json=request_body
+    _base_api(
+        request=f"/v1/workspaces/{workspace_id}/spark/settings",
+        payload=payload,
+        method="patch",
     )
-
-    if response.status_code != 200:
-        raise FabricHTTPException(response)
     print(
         f"{icons.green_dot} The spark settings within the '{workspace_name}' workspace have been updated accordingly."
     )

sempy_labs/_sql.py

@@ -1,14 +1,13 @@
-import sempy.fabric as fabric
 import pandas as pd
 from typing import Optional, Union, List
 from sempy._utils._log import log
 import struct
 from itertools import chain, repeat
-from sempy.fabric.exceptions import FabricHTTPException
 from sempy_labs._helper_functions import (
     resolve_lakehouse_name_and_id,
     resolve_item_name_and_id,
     resolve_workspace_name_and_id,
+    _base_api,
 )
 from uuid import UUID
 
@@ -47,21 +46,18 @@ class ConnectBase:
 
         # Resolve the appropriate ID and name (warehouse or lakehouse)
         if endpoint_type == "warehouse":
-            (resource_id, resource_name) = resolve_item_name_and_id(
+            (resource_name, resource_id) = resolve_item_name_and_id(
                 item=item, type=endpoint_type.capitalize(), workspace=workspace_id
             )
         else:
-            (resource_id, resource_name) = resolve_lakehouse_name_and_id(
+            (resource_name, resource_id) = resolve_lakehouse_name_and_id(
                 lakehouse=item, workspace=workspace_id
             )
 
         # Get the TDS endpoint
-        client = fabric.FabricRestClient()
-        response = client.get(
-            f"v1/workspaces/{workspace_id}/{endpoint_type}s/{resource_id}"
+        response = _base_api(
+            request=f"v1/workspaces/{workspace_id}/{endpoint_type}s/{resource_id}"
         )
-        if response.status_code != 200:
-            raise FabricHTTPException(response)
 
         if endpoint_type == "warehouse":
             tds_endpoint = response.json().get("properties", {}).get("connectionString")
@@ -150,7 +146,7 @@ class ConnectWarehouse(ConnectBase):
         timeout: Optional[int] = None,
     ):
         super().__init__(
-            name=warehouse,
+            item=warehouse,
             workspace=workspace,
             timeout=timeout,
             endpoint_type="warehouse",
@@ -165,7 +161,7 @@ class ConnectLakehouse(ConnectBase):
         timeout: Optional[int] = None,
     ):
         super().__init__(
-            name=lakehouse,
+            item=lakehouse,
             workspace=workspace,
             timeout=timeout,
             endpoint_type="lakehouse",

sempy_labs/_vertipaq.py

@@ -69,6 +69,8 @@ def vertipaq_analyzer(
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
     (dataset_name, dataset_id) = resolve_dataset_name_and_id(dataset, workspace_id)
 
+    fabric.refresh_tom_cache(workspace=workspace)
+
     vertipaq_map = {
         "Model": {
             "Dataset Name": [icons.data_type_string, icons.no_format],
@@ -163,9 +165,12 @@ def vertipaq_analyzer(
     dfP = fabric.list_partitions(
         dataset=dataset_id, extended=True, workspace=workspace_id
     )
-    artifact_type, lakehouse_name, lakehouse_id, lakehouse_workspace_id = (
-        get_direct_lake_source(dataset=dataset_id, workspace=workspace_id)
-    )
+
+    artifact_type = None
+    if is_direct_lake:
+        artifact_type, lakehouse_name, lakehouse_id, lakehouse_workspace_id = (
+            get_direct_lake_source(dataset=dataset_id, workspace=workspace_id)
+        )
 
     dfR["Missing Rows"] = 0
     dfR["Missing Rows"] = dfR["Missing Rows"].astype(int)

sempy_labs/_warehouses.py

@@ -1,13 +1,13 @@
 import sempy.fabric as fabric
 from sempy_labs._helper_functions import (
     resolve_workspace_name_and_id,
-    pagination,
-    lro,
+    _base_api,
+    _create_dataframe,
+    _update_dataframe_datatypes,
 )
 import pandas as pd
 from typing import Optional
 import sempy_labs._icons as icons
-from sempy.fabric.exceptions import FabricHTTPException
 from uuid import UUID
 
 
@@ -38,23 +38,24 @@ def create_warehouse(
 
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
 
-    request_body = {"displayName": warehouse}
+    payload = {"displayName": warehouse}
 
     if description:
-        request_body["description"] = description
+        payload["description"] = description
     if case_insensitive_collation:
-        request_body.setdefault("creationPayload", {})
-        request_body["creationPayload"][
+        payload.setdefault("creationPayload", {})
+        payload["creationPayload"][
             "defaultCollation"
         ] = "Latin1_General_100_CI_AS_KS_WS_SC_UTF8"
 
-    client = fabric.FabricRestClient()
-    response = client.post(
-        f"/v1/workspaces/{workspace_id}/warehouses/", json=request_body
+    _base_api(
+        request=f"/v1/workspaces/{workspace_id}/warehouses",
+        payload=payload,
+        method="post",
+        lro_return_status_code=True,
+        status_codes=[201, 202],
     )
 
-    lro(client, response, status_codes=[201, 202])
-
     print(
         f"{icons.green_dot} The '{warehouse}' warehouse has been created within the '{workspace_name}' workspace."
     )
@@ -79,25 +80,21 @@ def list_warehouses(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
         A pandas dataframe showing the warehouses within a workspace.
     """
 
-    df = pd.DataFrame(
-        columns=[
-            "Warehouse Name",
-            "Warehouse Id",
-            "Description",
-            "Connection Info",
-            "Created Date",
-            "Last Updated Time",
-        ]
-    )
+    columns = {
+        "Warehouse Name": "string",
+        "Warehouse Id": "string",
+        "Description": "string",
+        "Connection Info": "string",
+        "Created Date": "datetime",
+        "Last Updated Time": "datetime",
+    }
+    df = _create_dataframe(columns=columns)
 
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
 
-    client = fabric.FabricRestClient()
-    response = client.get(f"/v1/workspaces/{workspace_id}/warehouses")
-    if response.status_code != 200:
-        raise FabricHTTPException(response)
-
-    responses = pagination(client, response)
+    responses = _base_api(
+        reqeust=f"/v1/workspaces/{workspace_id}/warehouses", uses_pagination=True
+    )
 
     for r in responses:
         for v in r.get("value", []):
@@ -113,6 +110,8 @@ def list_warehouses(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
             }
             df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
 
+    _update_dataframe_datatypes(dataframe=df, column_map=columns)
+
     return df
 
 
@@ -138,11 +137,9 @@ def delete_warehouse(name: str, workspace: Optional[str | UUID] = None):
         item_name=name, type="Warehouse", workspace=workspace_id
     )
 
-    client = fabric.FabricRestClient()
-    response = client.delete(f"/v1/workspaces/{workspace_id}/warehouses/{item_id}")
-
-    if response.status_code != 200:
-        raise FabricHTTPException(response)
+    _base_api(
+        request=f"/v1/workspaces/{workspace_id}/warehouses/{item_id}", method="delete"
+    )
 
     print(
         f"{icons.green_dot} The '{name}' warehouse within the '{workspace_name}' workspace has been deleted."

sempy_labs/_workloads.py

@@ -1,8 +1,11 @@
-import sempy.fabric as fabric
 import pandas as pd
 from typing import Optional
 import sempy_labs._icons as icons
-from sempy.fabric.exceptions import FabricHTTPException
+from sempy_labs._helper_functions import (
+    _update_dataframe_datatypes,
+    _base_api,
+    _create_dataframe,
+)
 
 
 def list_workloads(capacity_name: str) -> pd.DataFrame:
@@ -25,17 +28,16 @@ def list_workloads(capacity_name: str) -> pd.DataFrame:
 
     from sempy_labs._helper_functions import resolve_capacity_id
 
-    df = pd.DataFrame(
-        columns=["Workload Name", "State", "Max Memory Percentage Set By User"]
-    )
+    columns = {
+        "Workload Name": "string",
+        "State": "string",
+        "Max Memory Percentage Set By User": "int",
+    }
+    df = _create_dataframe(columns=columns)
 
     capacity_id = resolve_capacity_id(capacity_name=capacity_name)
 
-    client = fabric.PowerBIRestClient()
-    response = client.get(f"/v1.0/myorg/capacities/{capacity_id}/Workloads")
-
-    if response.status_code != 200:
-        raise FabricHTTPException(response)
+    response = _base_api(request=f"/v1.0/myorg/capacities/{capacity_id}/Workloads")
 
     for v in response.json().get("value", []):
         new_data = {
@@ -45,8 +47,7 @@ def list_workloads(capacity_name: str) -> pd.DataFrame:
         }
         df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
 
-    int_cols = ["Max Memory Percentage Set By User"]
-    df[int_cols] = df[int_cols].astype(int)
+    _update_dataframe_datatypes(dataframe=df, column_map=columns)
 
     return df
 
@@ -92,12 +93,9 @@ def patch_workload(
             f"{icons.red_dot} Invalid max memory percentage. Must be a value between 0-100."
         )
 
-    client = fabric.PowerBIRestClient()
     url = f"/v1.0/myorg/capacities/{capacity_id}/Workloads/{workload_name}"
-    get_response = client.get(url)
-    if get_response.status_code != 200:
-        raise FabricHTTPException(get_response)
 
+    get_response = _base_api(request=url)
     get_json = get_response.json().get("value")
     current_state = get_json.get("state")
     current_max_memory = get_json.get("maxMemoryPercentageSetByUser")
@@ -118,10 +116,7 @@ def patch_workload(
     else:
         payload["maxMemoryPercentageSetByUser"] = current_max_memory
 
-    response = client.patch(url, json=payload)
-
-    if response.status_code != 200:
-        raise FabricHTTPException(response)
+    _base_api(request=url, method="patch", payload=payload)
 
     print(
         f"The '{workload_name}' workload within the '{capacity_name}' capacity has been updated accordingly."

sempy_labs/_workspace_identity.py

@@ -1,11 +1,9 @@
-import sempy.fabric as fabric
 from sempy_labs._helper_functions import (
     resolve_workspace_name_and_id,
-    lro,
+    _base_api,
 )
 from typing import Optional
 import sempy_labs._icons as icons
-from sempy.fabric.exceptions import FabricHTTPException
 from uuid import UUID
 
 
@@ -25,13 +23,12 @@ def provision_workspace_identity(workspace: Optional[str | UUID] = None):
 
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
 
-    client = fabric.FabricRestClient()
-    response = client.post(f"/v1/workspaces/{workspace_id}/provisionIdentity")
-
-    if response.status_code not in [200, 202]:
-        raise FabricHTTPException(response)
-
-    lro(client, response)
+    _base_api(
+        request=f"/v1/workspaces/{workspace_id}/provisionIdentity",
+        method="post",
+        lro_return_status_code=True,
+        status_codes=None,
+    )
 
     print(
         f"{icons.green_dot} A workspace identity has been provisioned for the '{workspace_name}' workspace."
@@ -54,13 +51,12 @@ def deprovision_workspace_identity(workspace: Optional[str | UUID] = None):
 
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
 
-    client = fabric.FabricRestClient()
-    response = client.post(f"/v1/workspaces/{workspace_id}/deprovisionIdentity")
-
-    if response.status_code not in [200, 202]:
-        raise FabricHTTPException(response)
-
-    lro(client, response)
+    _base_api(
+        request=f"/v1/workspaces/{workspace_id}/deprovisionIdentity",
+        method="post",
+        lro_return_status_code=True,
+        status_codes=None,
+    )
 
     print(
         f"{icons.green_dot} The workspace identity has been deprovisioned from the '{workspace_name}' workspace."