semantic-link-labs 0.10.0__py3-none-any.whl → 0.11.0__py3-none-any.whl

sempy_labs/admin/_users.py

@@ -5,8 +5,10 @@ from sempy_labs._helper_functions import (
 )
 from uuid import UUID
 import pandas as pd
+from sempy._utils._log import log
 
 
+@log
 def list_access_entities(
     user_email_address: str,
 ) -> pd.DataFrame:
@@ -43,6 +45,7 @@ def list_access_entities(
         uses_pagination=True,
     )
 
+    dfs = []
     for r in responses:
         for v in r.get("accessEntities", []):
             new_data = {
@@ -54,11 +57,15 @@ def list_access_entities(
                     "additionalPermissions"
                 ),
             }
-            df = pd.concat([df, pd.DataFrame([new_data])], ignore_index=True)
+            dfs.append(pd.DataFrame(new_data, index=[0]))
+
+    if dfs:
+        df = pd.concat(dfs, ignore_index=True)
 
     return df
 
 
+@log
 def list_user_subscriptions(user: str | UUID) -> pd.DataFrame:
     """
     Shows a list of subscriptions for the specified user. This is a preview API call.
@@ -127,7 +134,6 @@ def list_user_subscriptions(user: str | UUID) -> pd.DataFrame:
 
     if rows:
         df = pd.DataFrame(rows, columns=list(columns.keys()))
-
-    _update_dataframe_datatypes(dataframe=df, column_map=columns)
+        _update_dataframe_datatypes(dataframe=df, column_map=columns)
 
     return df

sempy_labs/admin/_workspaces.py

@@ -2,15 +2,21 @@ from sempy_labs._helper_functions import (
     _base_api,
     _build_url,
     _encode_user,
+    _update_dataframe_datatypes,
+    _create_dataframe,
 )
+
 from uuid import UUID
 from typing import Optional
 from sempy_labs.admin._basic_functions import (
     _resolve_workspace_name_and_id,
 )
 import sempy_labs._icons as icons
+import pandas as pd
+from sempy._utils._log import log
 
 
+@log
 def add_user_to_workspace(
     user: str | UUID,
     role: str = "Member",
@@ -68,6 +74,7 @@ def add_user_to_workspace(
     )
 
 
+@log
 def delete_user_from_workspace(
     user: str | UUID,
     workspace: Optional[str | UUID] = None,
@@ -116,6 +123,7 @@ def delete_user_from_workspace(
     )
 
 
+@log
 def restore_deleted_workspace(workspace_id: UUID, name: str, email_address: str):
     """
     Restores a deleted workspace.
@@ -146,3 +154,83 @@ def restore_deleted_workspace(workspace_id: UUID, name: str, email_address: str)
     print(
         f"{icons.green_dot} The '{workspace_id}' workspace has been restored as '{name}'."
     )
+
+
+@log
+def list_orphaned_workspaces(top: int = 100) -> pd.DataFrame:
+    """
+    Shows a list of orphaned workspaces (those with no users or no admins).
+
+    This is a wrapper function for the following API:
+    `Admin - Groups ListGroupsAsAdmin <https://learn.microsoft.com/rest/api/power-bi/admin/groups-get-groups-as-admin>`_.
+
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
+    Parameters
+    ----------
+    top : int, default=100
+        The maximum number of results to return.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing a list of orphaned workspaces.
+    """
+
+    # column structure with proper data types
+    columns = {
+        "Workspace Name": "string",
+        "Workspace Id": "string",
+        "Type": "string",
+        "State": "string",
+        "Is Read Only": "bool",
+        "Is On Dedicated Capacity": "bool",
+        "Capacity Migration Status": "string",
+        "Has Workspace Level Settings": "bool",
+        "Users": "list",
+    }
+
+    df = _create_dataframe(columns=columns)
+
+    url = (
+        "/v1.0/myorg/admin/groups?"
+        "$expand=users&"
+        "$filter=(not users/any()) or "
+        "(not users/any(u: u/groupUserAccessRight eq Microsoft.PowerBI.ServiceContracts.Api.GroupUserAccessRight'Admin'))&"
+        f"$top={top}"
+    )
+
+    response = _base_api(request=url, client="fabric_sp")
+    values = response.json().get("value", [])
+    df_raw = pd.json_normalize(values)
+
+    # friendly names and reorder
+    if not df_raw.empty:
+        df_raw = df_raw.rename(
+            columns={
+                "name": "Workspace Name",
+                "id": "Workspace Id",
+                "type": "Type",
+                "state": "State",
+                "isReadOnly": "Is Read Only",
+                "isOnDedicatedCapacity": "Is On Dedicated Capacity",
+                "capacityMigrationStatus": "Capacity Migration Status",
+                "hasWorkspaceLevelSettings ": "Has Workspace Level Settings",  # Note the space in original
+                "users": "Users",
+            }
+        )
+
+        df = df_raw[list(columns.keys())].copy()
+
+        # Convert empty lists to a more readable format for Users column
+        if "Users" in df.columns:
+            df["Users"] = df["Users"].apply(
+                lambda x: x if (x is not None and len(x) > 0) else []
+            )
+    else:
+        df = _create_dataframe(columns=columns)
+
+    # proper data types
+    _update_dataframe_datatypes(dataframe=df, column_map=columns)
+
+    return df

sempy_labs/directlake/_dl_helper.py

@@ -13,6 +13,7 @@ from sempy_labs._helper_functions import (
 )
 
 
+@log
 def check_fallback_reason(
     dataset: str | UUID, workspace: Optional[str | UUID] = None
 ) -> pd.DataFrame:
@@ -195,6 +196,7 @@ def generate_direct_lake_semantic_model(
         refresh_semantic_model(dataset=dataset, workspace=workspace_id)
 
 
+@log
 def get_direct_lake_source(
     dataset: str | UUID, workspace: Optional[str | UUID] = None
 ) -> Tuple[str, str, UUID, UUID]:

sempy_labs/directlake/_generate_shared_expression.py

@@ -8,8 +8,10 @@ from sempy_labs._helper_functions import (
 from typing import Optional
 import sempy_labs._icons as icons
 from uuid import UUID
+from sempy._utils._log import log
 
 
+@log
 def generate_shared_expression(
     item_name: Optional[str] = None,
     item_type: str = "Lakehouse",

sempy_labs/directlake/_get_directlake_lakehouse.py

@@ -1,16 +1,14 @@
 import sempy.fabric as fabric
 from sempy_labs._helper_functions import (
     resolve_lakehouse_id,
-    resolve_lakehouse_name,
-    get_direct_lake_sql_endpoint,
-    resolve_workspace_name_and_id,
-    resolve_dataset_name_and_id,
 )
 from typing import Optional, Tuple
 from uuid import UUID
 import sempy_labs._icons as icons
+from sempy._utils._log import log
 
 
+@log
 def get_direct_lake_lakehouse(
     dataset: str | UUID,
     workspace: Optional[str | UUID] = None,

sempy_labs/directlake/_get_shared_expression.py

@@ -1,7 +1,9 @@
 from typing import Optional
 from uuid import UUID
+from sempy._utils._log import log
 
 
+@log
 def get_shared_expression(
     lakehouse: Optional[str] = None, workspace: Optional[str | UUID] = None
 ) -> str:

sempy_labs/directlake/_guardrails.py

@@ -6,8 +6,10 @@ from uuid import UUID
 from sempy_labs._helper_functions import (
     resolve_workspace_name_and_id,
 )
+from sempy._utils._log import log
 
 
+@log
 def get_direct_lake_guardrails() -> pd.DataFrame:
     """
     Shows the guardrails for when Direct Lake semantic models will fallback to Direct Query

sempy_labs/directlake/_update_directlake_model_lakehouse_connection.py

@@ -13,6 +13,7 @@ from uuid import UUID
 import re
 
 
+@log
 def _extract_expression_list(expression):
     """
     Finds the pattern for DL/SQL & DL/OL expressions in the semantic model.
@@ -37,6 +38,7 @@ def _extract_expression_list(expression):
     return result
 
 
+@log
 def _get_direct_lake_expressions(
     dataset: str | UUID, workspace: Optional[str | UUID] = None
 ) -> dict:
@@ -111,9 +113,9 @@ def update_direct_lake_model_connection(
 
     Parameters
     ----------
-    dataset : str | UUID
+    dataset : str | uuid.UUID
         Name or ID of the semantic model.
-    workspace : str | UUID, default=None
+    workspace : str | uuid.UUID, default=None
         The Fabric workspace name or ID in which the semantic model exists.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
@@ -122,7 +124,7 @@ def update_direct_lake_model_connection(
         Defaults to None which resolves to the lakehouse attached to the notebook.
     source_type : str, default="Lakehouse"
         The type of source for the Direct Lake semantic model. Valid options: "Lakehouse", "Warehouse".
-    source_workspace : str | UUID, default=None
+    source_workspace : str | uuid.UUID, default=None
         The Fabric workspace name or ID used by the lakehouse/warehouse.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.

sempy_labs/directlake/_warm_cache.py

@@ -182,6 +182,7 @@ def warm_direct_lake_cache_isresident(
     )
 
 
+@log
 def _put_columns_into_memory(dataset, workspace, col_df, return_dataframe: bool = True):
 
     row_limit = 1000000

sempy_labs/graph/_groups.py

@@ -11,6 +11,7 @@ import sempy_labs._icons as icons
 from typing import List, Literal
 
 
+@log
 def resolve_group_id(group: str | UUID) -> UUID:
     """
     Resolves the group ID from the group name or ID.
@@ -74,6 +75,7 @@ def list_groups() -> pd.DataFrame:
 
     df = _create_dataframe(columns=columns)
 
+    dfs = []
     for v in result.get("value"):
         new_data = {
             "Group Id": v.get("id"),
@@ -90,14 +92,16 @@ def list_groups() -> pd.DataFrame:
             "Visibility": v.get("visibility"),
             "Security Identifier": v.get("securityIdentifier"),
         }
+        dfs.append(pd.DataFrame(new_data, index=[0]))
 
-        df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
-
-    _update_dataframe_datatypes(dataframe=df, column_map=columns)
+    if dfs:
+        df = pd.concat(dfs, ignore_index=True)
+        _update_dataframe_datatypes(dataframe=df, column_map=columns)
 
     return df
 
 
+@log
 def _get_group(group_id: UUID) -> pd.DataFrame:
     """
     Shows a list of groups and their properties.
@@ -136,6 +140,7 @@ def _get_group(group_id: UUID) -> pd.DataFrame:
     }
     df = _create_dataframe(columns=columns)
 
+    dfs = []
     for v in result.get("value"):
         new_data = {
             "Group Id": v.get("id"),
@@ -153,9 +158,11 @@ def _get_group(group_id: UUID) -> pd.DataFrame:
             "Security Identifier": v.get("securityIdentifier"),
         }
 
-        df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
+        dfs.append(pd.DataFrame(new_data, index=[0]))
 
-    _update_dataframe_datatypes(dataframe=df, column_map=columns)
+    if dfs:
+        df = pd.concat(dfs, ignore_index=True)
+        _update_dataframe_datatypes(dataframe=df, column_map=columns)
 
     return df
 
@@ -200,6 +207,7 @@ def list_group_members(group: str | UUID) -> pd.DataFrame:
 
     df = _create_dataframe(columns=columns)
 
+    dfs = []
     for v in result.get("value"):
         new_data = {
             "Member Id": v.get("id"),
@@ -214,8 +222,10 @@ def list_group_members(group: str | UUID) -> pd.DataFrame:
             "Given Name": v.get("givenName"),
             "Surname": v.get("surname"),
         }
+        dfs.append(pd.DataFrame(new_data, index=[0]))
 
-        df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
+    if dfs:
+        df = pd.concat(dfs, ignore_index=True)
 
     return df
 
@@ -260,6 +270,7 @@ def list_group_owners(group: str | UUID) -> pd.DataFrame:
 
     df = _create_dataframe(columns=columns)
 
+    dfs = []
     for v in result.get("value"):
         new_data = {
             "Owner Id": v.get("id"),
@@ -274,12 +285,15 @@ def list_group_owners(group: str | UUID) -> pd.DataFrame:
             "Given Name": v.get("givenName"),
             "Surname": v.get("surname"),
         }
+        dfs.append(pd.DataFrame(new_data, index=[0]))
 
-        df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
+    if dfs:
+        df = pd.concat(dfs, ignore_index=True)
 
     return df
 
 
+@log
 def _base_add_to_group(
     group: str | UUID,
     object: str | UUID,
@@ -359,6 +373,7 @@ def add_group_members(
     _base_add_to_group(group=group, object=user, object_type="members")
 
 
+@log
 def add_group_owners(
     group: str | UUID,
     user: str | UUID | List[str | UUID],

sempy_labs/graph/_teams.py

@@ -42,6 +42,7 @@ def list_teams() -> pd.DataFrame:
 
     df = _create_dataframe(columns=columns)
 
+    dfs = []
     for v in result.get("value"):
         new_data = {
             "Team Id": v.get("id"),
@@ -58,13 +59,16 @@ def list_teams() -> pd.DataFrame:
             "Member Count": v.get("memberCount"),
         }
 
-        df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
+        dfs.append(pd.DataFrame(new_data, index=[0]))
 
-    _update_dataframe_datatypes(dataframe=df, column_map=columns)
+    if dfs:
+        df = pd.concat(dfs, ignore_index=True)
+        _update_dataframe_datatypes(dataframe=df, column_map=columns)
 
     return df
 
 
+@log
 def list_chats(user: str | UUID) -> pd.DataFrame:
     """
     In progress...
@@ -95,6 +99,7 @@ def list_chats(user: str | UUID) -> pd.DataFrame:
     return df
 
 
+@log
 def send_teams_message(chat_id: str, message: str):
     """
     In progress...

sempy_labs/graph/_users.py

@@ -10,6 +10,7 @@ from sempy_labs._helper_functions import (
 from sempy._utils._log import log
 
 
+@log
 def resolve_user_id(user: str | UUID) -> UUID:
     """
     Resolves the user ID from the user principal name or ID.

sempy_labs/lakehouse/_blobs.py

@@ -14,6 +14,7 @@ import pandas as pd
 from sempy.fabric.exceptions import FabricHTTPException
 
 
+@log
 def _request_blob_api(
     request: str,
     method: str = "get",

sempy_labs/lakehouse/_get_lakehouse_tables.py

@@ -40,6 +40,8 @@ def get_lakehouse_tables(
     This function can be executed in either a PySpark or pure Python notebook.
 
     This is a wrapper function for the following API: `Tables - List Tables <https://learn.microsoft.com/rest/api/fabric/lakehouse/tables/list-tables>`_ plus extended capabilities.
+    However, the above mentioned API does not support Lakehouse schemas (Preview) until it is in GA (General Availability). This version also supports schema
+    enabled Lakehouses.
 
     Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
 
@@ -68,6 +70,7 @@ def get_lakehouse_tables(
     columns = {
         "Workspace Name": "string",
         "Lakehouse Name": "string",
+        "Schema Name": "string",
         "Table Name": "string",
         "Format": "string",
         "Type": "string",
@@ -83,27 +86,57 @@ def get_lakehouse_tables(
     if count_rows:  # Setting countrows defaults to extended=True
         extended = True
 
-    responses = _base_api(
-        request=f"v1/workspaces/{workspace_id}/lakehouses/{lakehouse_id}/tables",
-        uses_pagination=True,
-        client="fabric_sp",
-    )
+    API_called = True
+    try:
+        responses = _base_api(
+            request=f"v1/workspaces/{workspace_id}/lakehouses/{lakehouse_id}/tables",
+            uses_pagination=True,
+            client="fabric_sp",
+        )
 
-    if not responses[0].get("data"):
-        return df
+    except Exception as e:
+        API_called = False
 
     dfs = []
-    for r in responses:
-        for i in r.get("data", []):
-            new_data = {
-                "Workspace Name": workspace_name,
-                "Lakehouse Name": lakehouse_name,
-                "Table Name": i.get("name"),
-                "Format": i.get("format"),
-                "Type": i.get("type"),
-                "Location": i.get("location"),
-            }
-            dfs.append(pd.DataFrame(new_data, index=[0]))
+    local_path = None
+    if API_called:
+        if not responses[0].get("data"):
+            return df
+
+        for r in responses:
+            for i in r.get("data", []):
+                new_data = {
+                    "Workspace Name": workspace_name,
+                    "Lakehouse Name": lakehouse_name,
+                    "Schema Name": "",
+                    "Table Name": i.get("name"),
+                    "Format": i.get("format"),
+                    "Type": i.get("type"),
+                    "Location": i.get("location"),
+                }
+                dfs.append(pd.DataFrame(new_data, index=[0]))
+    else:
+        local_path = _mount(lakehouse=lakehouse_id, workspace=workspace_id)
+        tables_path = os.path.join(local_path, "Tables")
+        list_schema = os.listdir(tables_path)
+
+        for schema_name in list_schema:
+            schema_table_path = os.path.join(local_path, "Tables", schema_name)
+            list_tables = os.listdir(schema_table_path)
+            for table_name in list_tables:
+                location_path = create_abfss_path(
+                    lakehouse_id, workspace_id, table_name, schema_name
+                )
+                new_data = {
+                    "Workspace Name": workspace_name,
+                    "Lakehouse Name": lakehouse_name,
+                    "Schema Name": schema_name,
+                    "Table Name": table_name,
+                    "Format": "delta",
+                    "Type": "Managed",
+                    "Location": location_path,
+                }
+                dfs.append(pd.DataFrame(new_data, index=[0]))
 
     if dfs:
         df = pd.concat(dfs, ignore_index=True)
@@ -111,17 +144,25 @@ def get_lakehouse_tables(
     if extended:
         sku_value = get_sku_size(workspace_id)
         guardrail = get_directlake_guardrails_for_sku(sku_value)
-        local_path = _mount(lakehouse=lakehouse_id, workspace=workspace_id)
+        # Avoid mounting the lakehouse if is already mounted
+        if not local_path:
+            local_path = _mount(lakehouse=lakehouse_id, workspace=workspace_id)
 
         df["Files"], df["Row Groups"], df["Table Size"] = None, None, None
         if count_rows:
             df["Row Count"] = None
 
         for i, r in df.iterrows():
+            use_schema = True
+            schema_name = r["Schema Name"]
             table_name = r["Table Name"]
             if r["Type"] == "Managed" and r["Format"] == "delta":
-                delta_table_path = create_abfss_path(
-                    lakehouse_id, workspace_id, table_name
+                delta_table_path = (
+                    create_abfss_path(
+                        lakehouse_id, workspace_id, table_name, schema_name
+                    )
+                    .replace("//", "/")  # When schema_name = ""
+                    .replace("abfss:/", "abfss://")  # Put back the // after abfss:
                 )
 
                 if _pure_python_notebook():
@@ -135,29 +176,46 @@ def get_lakehouse_tables(
                     size_in_bytes = 0
                     for f in latest_files:
                         local_file_path = os.path.join(
-                            local_path, "Tables", table_name, os.path.basename(f)
+                            local_path, "Tables", schema_name, table_name, f
                         )
+
                         if os.path.exists(local_file_path):
                             size_in_bytes += os.path.getsize(local_file_path)
                     num_latest_files = len(latest_files)
                 else:
                     delta_table = _get_delta_table(delta_table_path)
+
                     latest_files = _read_delta_table(delta_table_path).inputFiles()
                     table_df = delta_table.toDF()
                     table_details = delta_table.detail().collect()[0].asDict()
-                    num_latest_files = table_details.get("numFiles", 0)
                     size_in_bytes = table_details.get("sizeInBytes", 0)
+                    num_latest_files = table_details.get("numFiles", 0)
+
+                table_path = os.path.join(local_path, "Tables", schema_name, table_name)
 
-                table_path = os.path.join(local_path, "Tables", table_name)
-                file_paths = [os.path.basename(f) for f in latest_files]
+                file_paths = []
+                for file in latest_files:
+                    if _pure_python_notebook():
+                        file_paths.append(file)
+                    else:
+                        # Append the <Partition folder>/<filename> or <filename>
+                        find_table = file.find(table_name)
+                        len_file = len(file)
+                        len_table = len(table_name)
+                        last_chars = len_file - (find_table + len_table + 1)
+                        file_paths.append(file[-last_chars:])
 
                 num_rowgroups = 0
                 for filename in file_paths:
-                    parquet_file = pq.ParquetFile(f"{table_path}/{filename}")
-                    num_rowgroups += parquet_file.num_row_groups
+                    parquet_file_path = f"{table_path}/{filename}"
+                    if os.path.exists(parquet_file_path):
+                        parquet_file = pq.ParquetFile(parquet_file_path)
+                        num_rowgroups += parquet_file.num_row_groups
+
                 df.at[i, "Files"] = num_latest_files
                 df.at[i, "Row Groups"] = num_rowgroups
                 df.at[i, "Table Size"] = size_in_bytes
+
             if count_rows:
                 if _pure_python_notebook():
                     row_count = delta_table.to_pyarrow_table().num_rows
@@ -165,6 +223,9 @@ def get_lakehouse_tables(
                     row_count = table_df.count()
                 df.at[i, "Row Count"] = row_count
 
+            # Set "Schema Name" = "dbo" when it is ""
+            df.loc[df["Schema Name"] == "", "Schema Name"] = "dbo"
+
     if extended:
         intColumns = ["Files", "Row Groups", "Table Size"]
         df[intColumns] = df[intColumns].astype(int)

sempy_labs/lakehouse/_helper.py

@@ -54,6 +54,7 @@ def is_v_ordered(
     return any(b"vorder" in key for key in ds_schema.keys())
 
 
+@log
 def delete_lakehouse(
     lakehouse: str | UUID, workspace: Optional[str | UUID] = None
 ) -> None:
@@ -77,6 +78,7 @@ def delete_lakehouse(
     delete_item(item=lakehouse, item_type="lakehouse", workspace=workspace)
 
 
+@log
 def update_lakehouse(
     name: Optional[str] = None,
     description: Optional[str] = None,