semantic-link-labs 0.8.11__py3-none-any.whl → 0.9.0__py3-none-any.whl

sempy_labs/admin/_scanner.py

@@ -6,6 +6,7 @@ import numpy as np
 import time
 from sempy_labs.admin._basic_functions import list_workspaces
 from sempy._utils._log import log
+import sempy_labs._authentication as auth
 
 
 @log
@@ -25,6 +26,8 @@ def scan_workspaces(
         `Admin - WorkspaceInfo GetScanStatus <https://learn.microsoft.com/rest/api/power-bi/admin/workspace-info-get-scan-status>`_.
         `Admin - WorkspaceInfo GetScanResult <https://learn.microsoft.com/rest/api/power-bi/admin/workspace-info-get-scan-result>`_.
 
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
     Parameters
     ----------
     data_source_details : bool, default=False
@@ -32,11 +35,11 @@ def scan_workspaces(
     dataset_schema: bool = False
         Whether to return dataset schema (tables, columns and measures). If you set this parameter to true, you must fully enable metadata scanning in order for data to be returned. For more information, see Enable tenant settings for metadata scanning.
     dataset_expressions : bool, default=False
-        Whether to return data source details
+        Whether to return data source details.
     lineage : bool, default=False
-        Whether to return lineage info (upstream dataflows, tiles, data source IDs)
+        Whether to return lineage info (upstream dataflows, tiles, data source IDs).
     artifact_users : bool, default=False
-        Whether to return user details for a Power BI item (such as a report or a dashboard)
+        Whether to return user details for a Power BI item (such as a report or a dashboard).
     workspace : str | List[str] | uuid.UUID | List[uuid.UUID], default=None
         The required workspace name(s) or id(s) to be scanned
 
@@ -51,7 +54,7 @@ def scan_workspaces(
         "misconfiguredDatasourceInstances": [],
     }
 
-    client = fabric.FabricRestClient()
+    client = fabric.FabricRestClient(token_provider=auth.token_provider.get())
 
     if workspace is None:
         workspace = fabric.resolve_workspace_name()

sempy_labs/directlake/_directlake_schema_compare.py

@@ -50,7 +50,6 @@ def direct_lake_schema_compare(
     artifact_type, lakehouse_name, lakehouse_id, lakehouse_workspace_id = (
         get_direct_lake_source(dataset=dataset_id, workspace=workspace_id)
     )
-    lakehouse_workspace = fabric.resolve_workspace_name(lakehouse_workspace_id)
 
     if artifact_type == "Warehouse":
         raise ValueError(
@@ -59,11 +58,17 @@ def direct_lake_schema_compare(
 
     dfP = fabric.list_partitions(dataset=dataset_id, workspace=workspace_id)
 
-    if not any(r["Mode"] == "DirectLake" for i, r in dfP.iterrows()):
+    if not any(r["Mode"] == "DirectLake" for _, r in dfP.iterrows()):
         raise ValueError(
             f"{icons.red_dot} The '{dataset_name}' semantic model within the '{workspace_name}' workspace is not in Direct Lake mode."
         )
 
+    if artifact_type is None:
+        raise ValueError(
+            f"{icons.red_dot} This function only supports Direct Lake semantic models where the source lakehouse resides in the same workpace as the semantic model."
+        )
+
+    lakehouse_workspace = fabric.resolve_workspace_name(lakehouse_workspace_id)
     dfT = fabric.list_tables(dataset=dataset_id, workspace=workspace_id)
     dfC = fabric.list_columns(dataset=dataset_id, workspace=workspace_id)
     lc = get_lakehouse_columns(lakehouse_name, lakehouse_workspace)

sempy_labs/directlake/_directlake_schema_sync.py

@@ -61,6 +61,12 @@ def direct_lake_schema_sync(
         raise ValueError(
             f"{icons.red_dot} This function is only valid for Direct Lake semantic models which source from Fabric lakehouses (not warehouses)."
         )
+
+    if artifact_type is None:
+        raise ValueError(
+            f"{icons.red_dot} This function only supports Direct Lake semantic models where the source lakehouse resides in the same workpace as the semantic model."
+        )
+
     lakehouse_workspace = fabric.resolve_workspace_name(lakehouse_workspace_id)
 
     lc = get_lakehouse_columns(lakehouse_name, lakehouse_workspace)

sempy_labs/directlake/_dl_helper.py

@@ -205,7 +205,7 @@ def get_direct_lake_source(
     dataset: str | UUID, workspace: Optional[str | UUID] = None
 ) -> Tuple[str, str, UUID, UUID]:
     """
-    Obtains the source information for a direct lake semantic model.
+    Obtains the source information for a direct lake semantic model (if the source is located in the same workspace as the semantic model).
 
     Parameters
     ----------
@@ -224,34 +224,54 @@ def get_direct_lake_source(
         If the semantic model is not a Direct Lake semantic model, it will return None, None, None.
     """
 
-    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
-    (dataset_name, dataset_id) = resolve_dataset_name_and_id(dataset, workspace_id)
-    client = fabric.PowerBIRestClient()
-    request_body = {
-        "artifacts": [
-            {
-                "objectId": dataset_id,
-                "type": "dataset",
-            }
-        ]
-    }
-    response = client.post(
-        "metadata/relations/upstream?apiVersion=3", json=request_body
-    )
-
-    artifacts = response.json().get("artifacts", [])
-    sql_id, sql_object_name, sql_workspace_id, artifact_type = None, None, None, None
+    from sempy_labs._helper_functions import get_direct_lake_sql_endpoint
 
-    for artifact in artifacts:
-        object_type = artifact.get("typeName")
-        display_name = artifact.get("displayName")
-        if object_type in ["Datawarehouse", "Lakewarehouse"]:
-            artifact_type = (
-                "Warehouse" if object_type == "Datawarehouse" else "Lakehouse"
-            )
-            sql_id = artifact.get("objectId")
-            sql_workspace_id = artifact.get("workspace", {}).get("objectId")
-            sql_object_name = display_name
-            break
-
-    return artifact_type, sql_object_name, sql_id, sql_workspace_id
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    sql_endpoint_id = get_direct_lake_sql_endpoint(dataset=dataset, workspace=workspace)
+    dfI = fabric.list_items(workspace=workspace)
+    dfI_filt = dfI[(dfI["Id"] == sql_endpoint_id) & (dfI["Type"] == "SQLEndpoint")]
+
+    artifact_type, artifact_name, artifact_id = None, None, None
+
+    if not dfI_filt.empty:
+        artifact_name = dfI_filt["Display Name"].iloc[0]
+        artifact_id = dfI[
+            (dfI["Display Name"] == artifact_name)
+            & (dfI["Type"].isin(["Lakehouse", "Warehouse"]))
+        ]["Id"].iloc[0]
+        artifact_type = dfI[
+            (dfI["Display Name"] == artifact_name)
+            & (dfI["Type"].isin(["Lakehouse", "Warehouse"]))
+        ]["Type"].iloc[0]
+
+    return artifact_type, artifact_name, artifact_id, workspace_id
+
+    # client = fabric.PowerBIRestClient()
+    # request_body = {
+    #    "artifacts": [
+    #        {
+    #            "objectId": dataset_id,
+    #            "type": "dataset",
+    #        }
+    #    ]
+    # }
+    # response = client.post(
+    #    "metadata/relations/upstream?apiVersion=3", json=request_body
+    # )
+
+    # artifacts = response.json().get("artifacts", [])
+    # sql_id, sql_object_name, sql_workspace_id, artifact_type = None, None, None, None
+
+    # for artifact in artifacts:
+    #    object_type = artifact.get("typeName")
+    #    display_name = artifact.get("displayName")
+    #    if object_type in ["Datawarehouse", "Lakewarehouse"]:
+    #        artifact_type = (
+    #            "Warehouse" if object_type == "Datawarehouse" else "Lakehouse"
+    #        )
+    #        sql_id = artifact.get("objectId")
+    #        sql_workspace_id = artifact.get("workspace", {}).get("objectId")
+    #        sql_object_name = display_name
+    #        break
+
+    # return artifact_type, sql_object_name, sql_id, sql_workspace_id

sempy_labs/directlake/_get_directlake_lakehouse.py

@@ -8,6 +8,7 @@ from sempy_labs._helper_functions import (
 )
 from typing import Optional, Tuple
 from uuid import UUID
+import sempy_labs._icons as icons
 
 
 def get_direct_lake_lakehouse(
@@ -41,30 +42,22 @@ def get_direct_lake_lakehouse(
         The lakehouse name and lakehouse ID.
     """
 
-    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
-    (dataset_name, dataset_id) = resolve_dataset_name_and_id(dataset, workspace_id)
-
-    if lakehouse_workspace is None:
-        lakehouse_workspace = workspace_name
-
-    if lakehouse is None:
-        lakehouse_id = fabric.get_lakehouse_id()
-        lakehouse = resolve_lakehouse_name(lakehouse_id, lakehouse_workspace)
-
-    # dfP = fabric.list_partitions(dataset=dataset, workspace=workspace)
-    # dfP_filt = dfP[dfP["Mode"] == "DirectLake"]
-
-    # if len(dfP_filt) == 0:
-    #    raise ValueError(
-    #        f"{icons.red_dot} The '{dataset}' semantic model within the '{workspace}' workspace is not in Direct Lake mode."
-    #    )
-
-    sqlEndpointId = get_direct_lake_sql_endpoint(dataset_id, workspace_id)
-
-    dfI = fabric.list_items(workspace=lakehouse_workspace, type="SQLEndpoint")
-    dfI_filt = dfI[dfI["Id"] == sqlEndpointId]
-    lakehouseName = dfI_filt["Display Name"].iloc[0]
-
-    lakehouseId = resolve_lakehouse_id(lakehouseName, lakehouse_workspace)
-
-    return lakehouseName, lakehouseId
+    from sempy_labs.directlake._dl_helper import get_direct_lake_source
+
+    artifact_type, artifact_name, artifact_id, workspace_id = get_direct_lake_source(
+        dataset=dataset, workspace=workspace
+    )
+
+    if artifact_type in ["Lakehouse", "Warehouse"]:
+        return artifact_name, artifact_id
+    else:
+        dfP = fabric.list_partitions(dataset=dataset, workspace=workspace)
+        dfP_filt = dfP[dfP["Mode"] == "DirectLake"]
+        if dfP_filt.empty:
+            raise ValueError(
+                f"{icons.red_dot} The '{dataset}' semantic model within the '{workspace}' workspace is not in Direct Lake mode."
+            )
+        lakehouse_id = resolve_lakehouse_id(
+            lakehouse=lakehouse, workspace=lakehouse_workspace
+        )
+        return lakehouse, lakehouse_id

sempy_labs/directlake/_update_directlake_partition_entity.py

@@ -125,6 +125,11 @@ def add_table_to_direct_lake_semantic_model(
             f"{icons.red_dot} This function is only valid for Direct Lake semantic models which source from Fabric lakehouses (not warehouses)."
         )
 
+    if artifact_type is None:
+        raise ValueError(
+            f"{icons.red_dot} This function only supports Direct Lake semantic models where the source lakehouse resides in the same workpace as the semantic model."
+        )
+
     lakehouse_workspace = fabric.resolve_workspace_name(lakehouse_workspace_id)
 
     with connect_semantic_model(

sempy_labs/lakehouse/_get_lakehouse_columns.py

@@ -1,11 +1,9 @@
-import sempy.fabric as fabric
 import pandas as pd
 from pyspark.sql import SparkSession
 from sempy_labs._helper_functions import (
-    resolve_lakehouse_name,
     format_dax_object_name,
-    resolve_lakehouse_id,
     resolve_workspace_name_and_id,
+    resolve_lakehouse_name_and_id,
 )
 from typing import Optional
 from sempy._utils._log import log
@@ -14,15 +12,15 @@ from uuid import UUID
 
 @log
 def get_lakehouse_columns(
-    lakehouse: Optional[str] = None, workspace: Optional[str | UUID] = None
+    lakehouse: Optional[str | UUID] = None, workspace: Optional[str | UUID] = None
 ) -> pd.DataFrame:
     """
     Shows the tables and columns of a lakehouse and their respective properties.
 
     Parameters
     ----------
-    lakehouse : str, default=None
-        The Fabric lakehouse.
+    lakehouse : str | uuid.UUID, default=None
+        The Fabric lakehouse name or ID.
         Defaults to None which resolves to the lakehouse attached to the notebook.
     lakehouse_workspace : str | uuid.UUID, default=None
         The Fabric workspace name or ID used by the lakehouse.
@@ -49,34 +47,31 @@ def get_lakehouse_columns(
     )
 
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
-
-    if lakehouse is None:
-        lakehouse_id = fabric.get_lakehouse_id()
-        lakehouse = resolve_lakehouse_name(lakehouse_id, workspace_id)
-    else:
-        lakehouse_id = resolve_lakehouse_id(lakehouse, workspace_id)
+    (lakehouse_name, lakehouse_id) = resolve_lakehouse_name_and_id(
+        lakehouse=lakehouse, workspace=workspace_id
+    )
 
     spark = SparkSession.builder.getOrCreate()
 
     tables = get_lakehouse_tables(
-        lakehouse=lakehouse, workspace=workspace_id, extended=False, count_rows=False
+        lakehouse=lakehouse_id, workspace=workspace_id, extended=False, count_rows=False
     )
     tables_filt = tables[tables["Format"] == "delta"]
 
-    for i, r in tables_filt.iterrows():
-        tName = r["Table Name"]
-        tPath = r["Location"]
-        delta_table = DeltaTable.forPath(spark, tPath)
+    for _, r in tables_filt.iterrows():
+        table_name = r["Table Name"]
+        path = r["Location"]
+        delta_table = DeltaTable.forPath(spark, path)
         sparkdf = delta_table.toDF()
 
-        for cName, data_type in sparkdf.dtypes:
-            tc = format_dax_object_name(tName, cName)
+        for col_name, data_type in sparkdf.dtypes:
+            full_column_name = format_dax_object_name(table_name, col_name)
             new_data = {
                 "Workspace Name": workspace_name,
                 "Lakehouse Name": lakehouse,
-                "Table Name": tName,
-                "Column Name": cName,
-                "Full Column Name": tc,
+                "Table Name": table_name,
+                "Column Name": col_name,
+                "Full Column Name": full_column_name,
                 "Data Type": data_type,
             }
             df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)

sempy_labs/lakehouse/_get_lakehouse_tables.py

@@ -4,10 +4,11 @@ from pyspark.sql import SparkSession
 import pyarrow.parquet as pq
 import datetime
 from sempy_labs._helper_functions import (
-    resolve_lakehouse_id,
-    resolve_lakehouse_name,
+    _get_column_aggregate,
     resolve_workspace_name_and_id,
+    resolve_lakehouse_name_and_id,
     pagination,
+    save_as_delta_table,
 )
 from sempy_labs.directlake._guardrails import (
     get_sku_size,
@@ -23,7 +24,7 @@ from uuid import UUID
 
 @log
 def get_lakehouse_tables(
-    lakehouse: Optional[str] = None,
+    lakehouse: Optional[str | UUID] = None,
     workspace: Optional[str | UUID] = None,
     extended: bool = False,
     count_rows: bool = False,
@@ -36,8 +37,8 @@ def get_lakehouse_tables(
 
     Parameters
     ----------
-    lakehouse : str, default=None
-        The Fabric lakehouse.
+    lakehouse : str | uuid.UUID, default=None
+        The Fabric lakehouse name or ID.
         Defaults to None which resolves to the lakehouse attached to the notebook.
     workspace : str | uuid.UUID, default=None
         The Fabric workspace name or ID used by the lakehouse.
@@ -68,12 +69,9 @@ def get_lakehouse_tables(
     )
 
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
-
-    if lakehouse is None:
-        lakehouse_id = fabric.get_lakehouse_id()
-        lakehouse = resolve_lakehouse_name(lakehouse_id, workspace_id)
-    else:
-        lakehouse_id = resolve_lakehouse_id(lakehouse, workspace_id)
+    (lakehouse_name, lakehouse_id) = resolve_lakehouse_name_and_id(
+        lakehouse=lakehouse, workspace=workspace_id
+    )
 
     if count_rows:  # Setting countrows defaults to extended=True
         extended = True
@@ -106,7 +104,7 @@ def get_lakehouse_tables(
         for i in r.get("data", []):
             new_data = {
                 "Workspace Name": workspace_name,
-                "Lakehouse Name": lakehouse,
+                "Lakehouse Name": lakehouse_name,
                 "Table Name": i.get("name"),
                 "Format": i.get("format"),
                 "Type": i.get("type"),
@@ -179,23 +177,17 @@ def get_lakehouse_tables(
                 f"{icons.red_dot} In order to save the report.json file, a lakehouse must be attached to the notebook. Please attach a lakehouse to this notebook."
             )
 
-        spark = SparkSession.builder.getOrCreate()
-
-        lakehouse_id = fabric.get_lakehouse_id()
-        lakehouse = resolve_lakehouse_name(
-            lakehouse_id=lakehouse_id, workspace=workspace_id
-        )
+        (current_lakehouse_name, current_lakehouse_id) = resolve_lakehouse_name_and_id()
         lakeTName = "lakehouse_table_details"
         lakeT_filt = df[df["Table Name"] == lakeTName]
 
-        query = f"SELECT MAX(RunId) FROM {lakehouse}.{lakeTName}"
-
         if len(lakeT_filt) == 0:
-            runId = 1
+            run_id = 1
         else:
-            dfSpark = spark.sql(query)
-            maxRunId = dfSpark.collect()[0][0]
-            runId = maxRunId + 1
+            max_run_id = _get_column_aggregate(
+                lakehouse=current_lakehouse_name, table_name=lakeTName
+            )
+            run_id = max_run_id + 1
 
         export_df = df.copy()
 
@@ -240,15 +232,11 @@ def get_lakehouse_tables(
         print(
             f"{icons.in_progress} Saving Lakehouse table properties to the '{lakeTName}' table in the lakehouse...\n"
         )
-        now = datetime.datetime.now()
-        export_df["Timestamp"] = now
-        export_df["RunId"] = runId
+        export_df["Timestamp"] = datetime.datetime.now()
+        export_df["RunId"] = run_id
 
-        export_df.columns = export_df.columns.str.replace(" ", "_")
-        spark_df = spark.createDataFrame(export_df)
-        spark_df.write.mode("append").format("delta").saveAsTable(lakeTName)
-        print(
-            f"{icons.bullet} Lakehouse table properties have been saved to the '{lakeTName}' delta table."
+        save_as_delta_table(
+            dataframe=export_df, delta_table_name=lakeTName, write_mode="append"
         )
 
     return df

sempy_labs/lakehouse/_lakehouse.py

@@ -1,9 +1,4 @@
-import sempy.fabric as fabric
 from tqdm.auto import tqdm
-from sempy_labs._helper_functions import (
-    resolve_lakehouse_name,
-    resolve_workspace_name_and_id,
-)
 from typing import List, Optional, Union
 from sempy._utils._log import log
 from uuid import UUID
@@ -56,13 +51,7 @@ def optimize_lakehouse_tables(
     from sempy_labs.lakehouse._get_lakehouse_tables import get_lakehouse_tables
     from delta import DeltaTable
 
-    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
-
-    if lakehouse is None:
-        lakehouse_id = fabric.get_lakehouse_id()
-        lakehouse = resolve_lakehouse_name(lakehouse_id, workspace_id)
-
-    lakeTables = get_lakehouse_tables(lakehouse=lakehouse, workspace=workspace_id)
+    lakeTables = get_lakehouse_tables(lakehouse=lakehouse, workspace=workspace)
     lakeTablesDelta = lakeTables[lakeTables["Format"] == "delta"]
 
     if isinstance(tables, str):
@@ -115,13 +104,7 @@ def vacuum_lakehouse_tables(
     from sempy_labs.lakehouse._get_lakehouse_tables import get_lakehouse_tables
     from delta import DeltaTable
 
-    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
-
-    if lakehouse is None:
-        lakehouse_id = fabric.get_lakehouse_id()
-        lakehouse = resolve_lakehouse_name(lakehouse_id, workspace_id)
-
-    lakeTables = get_lakehouse_tables(lakehouse=lakehouse, workspace=workspace_id)
+    lakeTables = get_lakehouse_tables(lakehouse=lakehouse, workspace=workspace)
     lakeTablesDelta = lakeTables[lakeTables["Format"] == "delta"]
 
     if isinstance(tables, str):

sempy_labs/report/_generate_report.py

@@ -8,11 +8,13 @@ from sempy_labs._helper_functions import (
     _conv_b64,
     resolve_report_id,
     resolve_dataset_name_and_id,
+    resolve_item_name_and_id,
     lro,
 )
 import sempy_labs._icons as icons
 from sempy._utils._log import log
 from uuid import UUID
+from sempy.fabric.exceptions import FabricHTTPException
 
 
 def create_report_from_reportjson(
@@ -371,3 +373,46 @@ def _create_report(
             report_workspace=report_workspace,
             dataset_workspace=dataset_workspace,
         )
+
+
+def _get_report(
+    report: str | UUID, workspace: Optional[str | UUID] = None
+) -> pd.DataFrame:
+
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    (report_name, report_id) = resolve_item_name_and_id(
+        item=report, type="Report", workspace=workspace
+    )
+
+    client = fabric.FabricRestClient()
+    response = client.get(f"/v1.0/myorg/groups/{workspace_id}/reports/{report_id}")
+
+    if response.status_code != 200:
+        raise FabricHTTPException(response)
+
+    result = response.json()
+
+    new_data = {
+        "Id": result.get("id"),
+        "Report Type": result.get("reportType"),
+        "Name": result.get("name"),
+        "Web Url": result.get("webUrl"),
+        "Embed Url": result.get("embedUrl"),
+        "Is From Pbix": result.get("isFromPbix"),
+        "Is Owned By Me": result.get("isOwnedByMe"),
+        "Dataset Id": result.get("datasetId"),
+        "Dataset Workspace Id": result.get("datasetWorkspaceId"),
+        "Users": result.get("users") if result.get("users") is not None else [],
+        "Subscriptions": (
+            result.get("subscriptions")
+            if result.get("subscriptions") is not None
+            else []
+        ),
+    }
+
+    df = pd.DataFrame([new_data])
+
+    bool_cols = ["Is From Pbix", "Is Owned By Me"]
+    df[bool_cols] = df[bool_cols].astype(bool)
+
+    return df

sempy_labs/report/_report_bpa.py

@@ -10,7 +10,7 @@ from sempy_labs._helper_functions import (
     resolve_report_id,
     resolve_lakehouse_name,
     resolve_workspace_capacity,
-    _get_max_run_id,
+    _get_column_aggregate,
     resolve_workspace_name_and_id,
 )
 from sempy_labs.lakehouse import get_lakehouse_tables, lakehouse_attached
@@ -217,7 +217,7 @@ def run_report_bpa(
         if len(lakeT_filt) == 0:
             runId = 1
         else:
-            max_run_id = _get_max_run_id(
+            max_run_id = _get_column_aggregate(
                 lakehouse=lakehouse, table_name=delta_table_name
             )
             runId = max_run_id + 1