semantic-link-labs 0.9.2__py3-none-any.whl → 0.9.3__py3-none-any.whl

sempy_labs/_gateways.py

@@ -21,6 +21,8 @@ def list_gateways() -> pd.DataFrame:
 
     This is a wrapper function for the following API: `Gateways - List Gateways <https://learn.microsoft.com/rest/api/fabric/core/gateways/list-gateways>`_.
 
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
     Returns
     -------
     pandas.DataFrame
@@ -41,7 +43,9 @@ def list_gateways() -> pd.DataFrame:
     }
     df = _create_dataframe(columns=columns)
 
-    responses = _base_api(request="/v1/gateways", uses_pagination=True)
+    responses = _base_api(
+        request="/v1/gateways", client="fabric_sp", uses_pagination=True
+    )
 
     for r in responses:
         for v in r.get("value", []):
@@ -85,6 +89,8 @@ def delete_gateway(gateway: str | UUID):
 
     This is a wrapper function for the following API: `Gateways - Delete Gateway <https://learn.microsoft.com/rest/api/fabric/core/gateways/delete-gateway>`_.
 
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
     Parameters
     ----------
     gateway : str | uuid.UUID
@@ -92,7 +98,7 @@ def delete_gateway(gateway: str | UUID):
     """
 
     gateway_id = _resolve_gateway_id(gateway)
-    _base_api(request=f"/v1/gateways/{gateway_id}", method="delete")
+    _base_api(request=f"/v1/gateways/{gateway_id}", client="fabric_sp", method="delete")
     print(f"{icons.green_dot} The '{gateway}' gateway has been deleted.")
 
 
@@ -102,6 +108,8 @@ def list_gateway_role_assigments(gateway: str | UUID) -> pd.DataFrame:
 
     This is a wrapper function for the following API: `Gateways - List Gateway Role Assignments <https://learn.microsoft.com/rest/api/fabric/core/gateways/list-gateway-role-assignments>`_.
 
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
     Parameters
     ----------
     gateway : str | uuid.UUID
@@ -122,7 +130,9 @@ def list_gateway_role_assigments(gateway: str | UUID) -> pd.DataFrame:
     df = _create_dataframe(columns=columns)
     gateway_id = _resolve_gateway_id(gateway)
     responses = _base_api(
-        request=f"/v1/gateways/{gateway_id}/roleAssignments", uses_pagination=True
+        request=f"/v1/gateways/{gateway_id}/roleAssignments",
+        client="fabric_sp",
+        uses_pagination=True,
     )
 
     for r in responses:
@@ -145,6 +155,8 @@ def delete_gateway_role_assignment(gateway: str | UUID, role_assignment_id: UUID
 
     This is a wrapper function for the following API: `Gateways - Delete Gateway Role Assignment <https://learn.microsoft.com/rest/api/fabric/core/gateways/delete-gateway-role-assignment>`_.
 
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
     Parameters
     ----------
     gateway : str | uuid.UUID
@@ -156,6 +168,7 @@ def delete_gateway_role_assignment(gateway: str | UUID, role_assignment_id: UUID
     gateway_id = _resolve_gateway_id(gateway)
     _base_api(
         request=f"/v1/gateways/{gateway_id}/roleAssignments/{role_assignment_id}",
+        client="fabric_sp",
         method="delete",
     )
 
@@ -187,6 +200,8 @@ def delete_gateway_member(gateway: str | UUID, gateway_member: str | UUID):
 
     This is a wrapper function for the following API: `Gateways - Delete Gateway Member <https://learn.microsoft.com/rest/api/fabric/core/gateways/delete-gateway-member>`_.
 
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
     Parameters
     ----------
     gateway : str | uuid.UUID
@@ -200,7 +215,11 @@ def delete_gateway_member(gateway: str | UUID, gateway_member: str | UUID):
         gateway=gateway_id, gateway_member=gateway_member
     )
 
-    _base_api(request=f"/v1/gateways/{gateway_id}/members/{member_id}", method="delete")
+    _base_api(
+        request=f"/v1/gateways/{gateway_id}/members/{member_id}",
+        client="fabric_sp",
+        method="delete",
+    )
     print(
         f"{icons.green_dot} The '{member_id}' member for the '{gateway}' gateway has been deleted."
     )
@@ -212,6 +231,8 @@ def list_gateway_members(gateway: str | UUID) -> pd.DataFrame:
 
     This is a wrapper function for the following API: `Gateways - List Gateway Members <https://learn.microsoft.com/rest/api/fabric/core/gateways/list-gateway-members>`_.
 
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
     Parameters
     ----------
     gateway : str | uuid.UUID
@@ -235,7 +256,9 @@ def list_gateway_members(gateway: str | UUID) -> pd.DataFrame:
     }
     df = _create_dataframe(columns=columns)
 
-    response = _base_api(request=f"/v1/gateways/{gateway_id}/members")
+    response = _base_api(
+        request=f"/v1/gateways/{gateway_id}/members", client="fabric_sp"
+    )
 
     for v in response.json().get("value", []):
         new_data = {
@@ -269,6 +292,8 @@ def create_vnet_gateway(
 
     This is a wrapper function for the following API: `Gateways - Create Gateway <https://learn.microsoft.com/rest/api/fabric/core/gateways/create-gateway>`_.
 
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
     Parameters
     ----------
     name : str
@@ -304,7 +329,13 @@ def create_vnet_gateway(
         "numberOfMemberGateways": number_of_member_gateways,
     }
 
-    _base_api(request="/v1/gateways", method="post", payload=payload, status_codes=201)
+    _base_api(
+        request="/v1/gateways",
+        client="fabric_sp",
+        method="post",
+        payload=payload,
+        status_codes=201,
+    )
 
     print(
         f"{icons.green_dot} The '{name}' gateway was created within the '{capacity}' capacity."
@@ -322,6 +353,8 @@ def update_on_premises_gateway(
 
     This is a wrapper function for the following API: `Gateways - Update Gateway <https://learn.microsoft.com/rest/api/fabric/core/gateways/update-gateway>`_.
 
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
     Parameters
     ----------
     gateway : str | uuid.UUID
@@ -352,7 +385,12 @@ def update_on_premises_gateway(
 
     payload["type"] = "OnPremises"
 
-    _base_api(request=f"/v1/gateways/{gateway_id}", method="patch", payload=payload)
+    _base_api(
+        request=f"/v1/gateways/{gateway_id}",
+        client="fabric_sp",
+        method="patch",
+        payload=payload,
+    )
 
     print(f"{icons.green_dot} The '{gateway}' has been updated accordingly.")
 
@@ -368,6 +406,8 @@ def update_vnet_gateway(
 
     This is a wrapper function for the following API: `Gateways - Update Gateway <https://learn.microsoft.com/rest/api/fabric/core/gateways/update-gateway>`_.
 
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
     Parameters
     ----------
     gateway : str | uuid.UUID
@@ -399,7 +439,12 @@ def update_vnet_gateway(
 
     payload["type"] = "VirtualNetwork"
 
-    _base_api(request=f"/v1/gateways/{gateway_id}", method="patch", payload=payload)
+    _base_api(
+        request=f"/v1/gateways/{gateway_id}",
+        client="fabric_sp",
+        method="patch",
+        payload=payload,
+    )
     print(f"{icons.green_dot} The '{gateway}' has been updated accordingly.")
 
 
@@ -411,6 +456,8 @@ def bind_semantic_model_to_gateway(
 
     This is a wrapper function for the following API: `Datasets - Bind To Gateway In Group <https://learn.microsoft.com/rest/api/power-bi/datasets/bind-to-gateway-in-group>`_.
 
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
     Parameters
     ----------
     dataset : str | uuid.UUID
@@ -435,6 +482,7 @@ def bind_semantic_model_to_gateway(
 
     _base_api(
         request=f"/v1.0/myorg/groups/{workspace_id}/datasets/{dataset_id}/Default.BindToGateway",
+        client="fabric_sp",
         method="post",
         payload=payload,
     )

sempy_labs/_helper_functions.py

@@ -31,7 +31,9 @@ def _build_url(url: str, params: dict) -> str:
 
 
 def create_abfss_path(
-    lakehouse_id: UUID, lakehouse_workspace_id: UUID, delta_table_name: str
+    lakehouse_id: UUID,
+    lakehouse_workspace_id: UUID,
+    delta_table_name: Optional[str] = None,
 ) -> str:
     """
     Creates an abfss path for a delta table in a Fabric lakehouse.
@@ -42,18 +44,22 @@ def create_abfss_path(
         ID of the Fabric lakehouse.
     lakehouse_workspace_id : uuid.UUID
         ID of the Fabric workspace.
-    delta_table_name : str
+    delta_table_name : str, default=None
         Name of the delta table name.
 
     Returns
     -------
     str
-        An abfss path which can be used to save/reference a delta table in a Fabric lakehouse.
+        An abfss path which can be used to save/reference a delta table in a Fabric lakehouse or lakehouse.
     """
 
     fp = _get_default_file_path()
+    path = f"abfss://{lakehouse_workspace_id}@{fp}/{lakehouse_id}"
+
+    if delta_table_name is not None:
+        path += f"/Tables/{delta_table_name}"
 
-    return f"abfss://{lakehouse_workspace_id}@{fp}/{lakehouse_id}/Tables/{delta_table_name}"
+    return path
 
 
 def _get_default_file_path() -> str:
@@ -538,8 +544,9 @@ def save_as_delta_table(
             f"{icons.red_dot} Invalid 'delta_table_name'. Delta tables in the lakehouse cannot have spaces in their names."
         )
 
-    dataframe.columns = dataframe.columns.str.replace(" ", "_")
-    spark = SparkSession.builder.getOrCreate()
+    dataframe.columns = [col.replace(" ", "_") for col in dataframe.columns]
+
+    spark = _create_spark_session()
 
     type_mapping = {
         "string": StringType(),
@@ -1248,7 +1255,6 @@ def _get_column_aggregate(
     default_value: int = 0,
 ) -> int:
 
-    from pyspark.sql import SparkSession
     from pyspark.sql.functions import approx_count_distinct
     from pyspark.sql import functions as F
 
@@ -1257,7 +1263,7 @@ def _get_column_aggregate(
     lakehouse_id = resolve_lakehouse_id(lakehouse, workspace)
     path = create_abfss_path(lakehouse_id, workspace_id, table_name)
 
-    spark = SparkSession.builder.getOrCreate()
+    spark = _create_spark_session()
     df = spark.read.format("delta").load(path)
 
     if function in {"COUNTDISTINCT", "DISTINCTCOUNT"}:
@@ -1591,3 +1597,43 @@ def _print_success(item_name, item_type, workspace_name, action="created"):
         )
     else:
         raise NotImplementedError
+
+
+def _pure_python_notebook() -> bool:
+
+    from sempy.fabric._environment import _on_jupyter
+
+    return _on_jupyter()
+
+
+def _create_spark_session():
+
+    if _pure_python_notebook():
+        raise ValueError(
+            f"{icons.red_dot} This function is only available in a PySpark notebook."
+        )
+
+    from pyspark.sql import SparkSession
+
+    return SparkSession.builder.getOrCreate()
+
+
+def _read_delta_table(path: str):
+
+    spark = _create_spark_session()
+
+    return spark.read.format("delta").load(path)
+
+
+def _delta_table_row_count(table_name: str) -> int:
+
+    spark = _create_spark_session()
+
+    return spark.table(table_name).count()
+
+
+def _run_spark_sql_query(query):
+
+    spark = _create_spark_session()
+
+    return spark.sql(query)

sempy_labs/_list_functions.py

@@ -9,6 +9,7 @@ from sempy_labs._helper_functions import (
     _update_dataframe_datatypes,
     _base_api,
     _create_dataframe,
+    _run_spark_sql_query,
 )
 from sempy._utils._log import log
 import pandas as pd
@@ -584,14 +585,12 @@ def list_columns(
             query = f"{query} FROM {lakehouse}.{lakeTName}"
             sql_statements.append((table_name, query))
 
-            spark = SparkSession.builder.getOrCreate()
-
         for o in sql_statements:
             tName = o[0]
             query = o[1]
 
             # Run the query
-            df = spark.sql(query)
+            df = _run_spark_sql_query(query)
 
             for column in df.columns:
                 x = df.collect()[0][column]

sempy_labs/_ml_experiments.py

@@ -91,7 +91,7 @@ def create_ml_experiment(
     _base_api(
         request=f"/v1/workspaces/{workspace_id}/mlExperiments",
         method="post",
-        json=payload,
+        payload=payload,
         status_codes=[201, 202],
         lro_return_status_code=True,
     )

sempy_labs/_model_bpa.py

@@ -14,6 +14,7 @@ from sempy_labs._helper_functions import (
     get_language_codes,
     _get_column_aggregate,
     resolve_workspace_name_and_id,
+    _create_spark_session,
 )
 from sempy_labs.lakehouse import get_lakehouse_tables, lakehouse_attached
 from sempy_labs.tom import connect_semantic_model
@@ -181,7 +182,6 @@ def run_model_bpa(
             def translate_using_spark(rule_file):
 
                 from synapse.ml.services import Translate
-                from pyspark.sql import SparkSession
 
                 rules_temp = rule_file.copy()
                 rules_temp = rules_temp.drop(["Expression", "URL", "Severity"], axis=1)
@@ -195,7 +195,7 @@ def run_model_bpa(
                     ]
                 )
 
-                spark = SparkSession.builder.getOrCreate()
+                spark = _create_spark_session()
                 dfRules = spark.createDataFrame(rules_temp, schema)
 
                 columns = ["Category", "Rule Name", "Description"]

sempy_labs/_model_bpa_rules.py

@@ -416,7 +416,7 @@ def model_bpa_rules(
                 lambda obj, tom: any(
                     re.search(
                         r"USERELATIONSHIP\s*\(\s*.+?(?=])\]\s*,\s*'*"
-                        + obj.Name
+                        + re.escape(obj.Name)
                         + r"'*\[",
                         m.Expression,
                         flags=re.IGNORECASE,
@@ -455,7 +455,9 @@ def model_bpa_rules(
                 "Warning",
                 "The EVALUATEANDLOG function should not be used in production models",
                 lambda obj, tom: re.search(
-                    r"evaluateandlog\s*\(", obj.Expression, flags=re.IGNORECASE
+                    r"evaluateandlog\s*\(",
+                    obj.Expression,
+                    flags=re.IGNORECASE,
                 ),
                 "The EVALUATEANDLOG function is meant to be used only in development/test environments and should not be used in production models.",
                 "https://pbidax.wordpress.com/2022/08/16/introduce-the-dax-evaluateandlog-function",
@@ -592,13 +594,13 @@ def model_bpa_rules(
                 and not any(
                     re.search(
                         r"USERELATIONSHIP\s*\(\s*\'*"
-                        + obj.FromTable.Name
+                        + re.escape(obj.FromTable.Name)
                         + r"'*\["
-                        + obj.FromColumn.Name
+                        + re.escape(obj.FromColumn.Name)
                         + r"\]\s*,\s*'*"
-                        + obj.ToTable.Name
+                        + re.escape(obj.ToTable.Name)
                         + r"'*\["
-                        + obj.ToColumn.Name
+                        + re.escape(obj.ToColumn.Name)
                         + r"\]",
                         m.Expression,
                         flags=re.IGNORECASE,

sempy_labs/_translations.py

@@ -5,6 +5,7 @@ from sempy._utils._log import log
 import sempy_labs._icons as icons
 from sempy_labs._helper_functions import (
     get_language_codes,
+    _create_spark_session,
 )
 from uuid import UUID
 
@@ -40,7 +41,6 @@ def translate_semantic_model(
 
     from synapse.ml.services import Translate
     from pyspark.sql.functions import col, flatten
-    from pyspark.sql import SparkSession
     from sempy_labs.tom import connect_semantic_model
 
     icons.sll_tags.append("TranslateSemanticModel")
@@ -145,7 +145,7 @@ def translate_semantic_model(
                 [df_prep, pd.DataFrame(new_data, index=[0])], ignore_index=True
             )
 
-        spark = SparkSession.builder.getOrCreate()
+        spark = _create_spark_session()
         df = spark.createDataFrame(df_prep)
 
         columns = ["Name", "Description", "Display Folder"]

sempy_labs/_vertipaq.py

@@ -6,7 +6,6 @@ import os
 import shutil
 import datetime
 import warnings
-from pyspark.sql import SparkSession
 from sempy_labs._helper_functions import (
     format_dax_object_name,
     resolve_lakehouse_name,
@@ -15,6 +14,7 @@ from sempy_labs._helper_functions import (
     _get_column_aggregate,
     resolve_workspace_name_and_id,
     resolve_dataset_name_and_id,
+    _create_spark_session,
 )
 from sempy_labs._list_functions import list_relationships, list_tables
 from sempy_labs.lakehouse import lakehouse_attached, get_lakehouse_tables
@@ -197,7 +197,7 @@ def vertipaq_analyzer(
                 )
 
             sql_statements = []
-            spark = SparkSession.builder.getOrCreate()
+            spark = _create_spark_session()
             # Loop through tables
             for lakeTName in dfC_flt["Query"].unique():
                 query = "SELECT "
@@ -275,7 +275,7 @@ def vertipaq_analyzer(
             dfR.rename(columns={"Source": "To Lake Column"}, inplace=True)
             dfR.drop(columns=["Column Object"], inplace=True)
 
-            spark = SparkSession.builder.getOrCreate()
+            spark = _create_spark_session()
             for i, r in dfR.iterrows():
                 fromTable = r["From Lake Table"]
                 fromColumn = r["From Lake Column"]

sempy_labs/_warehouses.py

@@ -93,7 +93,7 @@ def list_warehouses(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
 
     responses = _base_api(
-        reqeust=f"/v1/workspaces/{workspace_id}/warehouses", uses_pagination=True
+        request=f"/v1/workspaces/{workspace_id}/warehouses", uses_pagination=True
     )
 
     for r in responses:

sempy_labs/admin/_domains.py

@@ -413,8 +413,6 @@ def assign_domain_workspaces(domain: str | UUID, workspace_names: str | List[str
         request=f"/v1/admin/domains/{domain_id}/assignWorkspaces",
         method="post",
         payload=payload,
-        lro_return_status_code=True,
-        status_codes=200,
     )
 
     print(

sempy_labs/admin/_scanner.py

@@ -4,10 +4,13 @@ from uuid import UUID
 from sempy.fabric.exceptions import FabricHTTPException
 import numpy as np
 import time
+import sempy_labs._icons as icons
 from sempy_labs.admin._basic_functions import list_workspaces
 from sempy._utils._log import log
 from sempy_labs._helper_functions import (
     _base_api,
+    _is_valid_uuid,
+    _build_url,
 )
 
 
@@ -39,22 +42,17 @@ def scan_workspaces(
     dataset_expressions : bool, default=False
         Whether to return data source details.
     lineage : bool, default=False
-        Whether to return lineage info (upstream dataflows, tiles, data source IDs).
+        Whether to return lineage info (upstream dataflows, tiles, data source IDs)
     artifact_users : bool, default=False
         Whether to return user details for a Power BI item (such as a report or a dashboard).
-    workspace : str | List[str] | uuid.UUID | List[uuid.UUID], default=None
-        The required workspace name(s) or id(s) to be scanned
+    workspace : str | List[str] | UUID | List[UUID], default=None
+        The required workspace name(s) or id(s) to be scanned. It supports a limit of 100 workspaces and only IDs in GUID format.
 
     Returns
     -------
     dict
         A json object with the scan result.
     """
-    scan_result = {
-        "workspaces": [],
-        "datasourceInstances": [],
-        "misconfiguredDatasourceInstances": [],
-    }
 
     if workspace is None:
         workspace = fabric.resolve_workspace_name()
@@ -62,55 +60,62 @@ def scan_workspaces(
     if isinstance(workspace, str):
         workspace = [workspace]
 
+    if len(workspace) > 100:
+        print(
+            f"{icons.yellow_dot} More than 100 workspaces where provided. Truncating to the fist 100."
+        )
+        workspace = workspace[:100]
+
     workspace_list = []
 
-    dfW = list_workspaces()
-    workspace_list = dfW[dfW["Name"].isin(workspace)]["Id"].tolist()
-    workspace_list = workspace_list + dfW[dfW["Id"].isin(workspace)]["Id"].tolist()
+    for w in workspace:
+        if _is_valid_uuid(w):
+            workspace_list.append(w)
+        else:
+            dfW = list_workspaces(workspace=w)
+            workspace_list = (
+                workspace_list + dfW[dfW["Name"].isin(workspace)]["Id"].tolist()
+            )
 
-    workspaces = np.array(workspace_list)
-    batch_size = 99
-    for i in range(0, len(workspaces), batch_size):
-        batch = workspaces[i : i + batch_size].tolist()
-        payload = {"workspaces": batch}
+    url = "/v1.0/myorg/admin/workspaces/getInfo"
+    params = {}
+    params["lineage"] = lineage
+    params["datasourceDetails"] = data_source_details
+    params["datasetSchema"] = dataset_schema
+    params["datasetExpressions"] = dataset_expressions
+    params["getArtifactUsers"] = artifact_users
 
-        url = f"/v1.0/myorg/admin/workspaces/getInfo?lineage={lineage}&datasourceDetails={data_source_details}&datasetSchema={dataset_schema}&datasetExpressions={dataset_expressions}&getArtifactUsers={artifact_users}"
-        response = _base_api(
-            request=url,
-            method="post",
-            payload=payload,
-            status_codes=202,
-            client="fabric_sp",
-        )
+    url = _build_url(url, params)
 
-        scan_id = response.json()["id"]
-        scan_status = response.json().get("status")
-        while scan_status not in ["Succeeded", "Failed"]:
-            time.sleep(1)
-            response = _base_api(
-                request=f"/v1.0/myorg/admin/workspaces/scanStatus/{scan_id}",
-                client="fabric_sp",
-            )
-            scan_status = response.json().get("status")
-        if scan_status == "Failed":
-            raise FabricHTTPException(response)
+    payload = {"workspaces": workspace_list}
+
+    response = _base_api(
+        request=url,
+        method="post",
+        payload=payload,
+        status_codes=202,
+        client="fabric_sp",
+    )
+
+    scan_id = response.json()["id"]
+    scan_status = response.json().get("status")
+
+    while scan_status not in ["Succeeded", "Failed"]:
+        time.sleep(1)
         response = _base_api(
-            request=f"/v1.0/myorg/admin/workspaces/scanResult/{scan_id}",
+            request=f"/v1.0/myorg/admin/workspaces/scanStatus/{scan_id}",
             client="fabric_sp",
         )
-        responseJson = response.json()
+        scan_status = response.json().get("status")
 
-        if "workspaces" in responseJson:
-            scan_result["workspaces"].extend(responseJson["workspaces"])
+    if scan_status == "Failed":
+        raise FabricHTTPException(response)
 
-        if "datasourceInstances" in responseJson:
-            scan_result["datasourceInstances"].extend(
-                responseJson["datasourceInstances"]
-            )
+    response = _base_api(
+        request=f"/v1.0/myorg/admin/workspaces/scanResult/{scan_id}",
+        client="fabric_sp",
+    )
 
-        if "misconfiguredDatasourceInstances" in responseJson:
-            scan_result["misconfiguredDatasourceInstances"].extend(
-                responseJson["misconfiguredDatasourceInstances"]
-            )
+    print(f"{icons.green_dot} Status: {scan_status}")
 
-    return scan_result
+    return response.json()

sempy_labs/lakehouse/_get_lakehouse_columns.py

@@ -1,10 +1,10 @@
 import pandas as pd
-from pyspark.sql import SparkSession
 from sempy_labs._helper_functions import (
     format_dax_object_name,
     resolve_workspace_name_and_id,
     resolve_lakehouse_name_and_id,
     _create_dataframe,
+    _create_spark_session,
 )
 from typing import Optional
 from sempy._utils._log import log
@@ -51,7 +51,7 @@ def get_lakehouse_columns(
         lakehouse=lakehouse, workspace=workspace_id
     )
 
-    spark = SparkSession.builder.getOrCreate()
+    spark = _create_spark_session()
 
     tables = get_lakehouse_tables(
         lakehouse=lakehouse_id, workspace=workspace_id, extended=False, count_rows=False

sempy_labs/lakehouse/_get_lakehouse_tables.py

@@ -1,6 +1,5 @@
 import sempy.fabric as fabric
 import pandas as pd
-from pyspark.sql import SparkSession
 import pyarrow.parquet as pq
 import datetime
 from sempy_labs._helper_functions import (
@@ -10,6 +9,7 @@ from sempy_labs._helper_functions import (
     save_as_delta_table,
     _base_api,
     _create_dataframe,
+    _create_spark_session,
 )
 from sempy_labs.directlake._guardrails import (
     get_sku_size,
@@ -112,7 +112,7 @@ def get_lakehouse_tables(
     if extended:
         sku_value = get_sku_size(workspace_id)
         guardrail = get_directlake_guardrails_for_sku(sku_value)
-        spark = SparkSession.builder.getOrCreate()
+        spark = _create_spark_session()
         df["Files"] = None
         df["Row Groups"] = None
         df["Table Size"] = None