semantic-link-labs 0.10.1__py3-none-any.whl → 0.11.0__py3-none-any.whl

sempy_labs/graph/_groups.py

@@ -11,6 +11,7 @@ import sempy_labs._icons as icons
 from typing import List, Literal
 
 
+@log
 def resolve_group_id(group: str | UUID) -> UUID:
     """
     Resolves the group ID from the group name or ID.
@@ -74,6 +75,7 @@ def list_groups() -> pd.DataFrame:
 
     df = _create_dataframe(columns=columns)
 
+    dfs = []
     for v in result.get("value"):
         new_data = {
             "Group Id": v.get("id"),
@@ -90,14 +92,16 @@ def list_groups() -> pd.DataFrame:
             "Visibility": v.get("visibility"),
             "Security Identifier": v.get("securityIdentifier"),
         }
+        dfs.append(pd.DataFrame(new_data, index=[0]))
 
-        df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
-
-    _update_dataframe_datatypes(dataframe=df, column_map=columns)
+    if dfs:
+        df = pd.concat(dfs, ignore_index=True)
+        _update_dataframe_datatypes(dataframe=df, column_map=columns)
 
     return df
 
 
+@log
 def _get_group(group_id: UUID) -> pd.DataFrame:
     """
     Shows a list of groups and their properties.
@@ -136,6 +140,7 @@ def _get_group(group_id: UUID) -> pd.DataFrame:
     }
     df = _create_dataframe(columns=columns)
 
+    dfs = []
     for v in result.get("value"):
         new_data = {
             "Group Id": v.get("id"),
@@ -153,9 +158,11 @@ def _get_group(group_id: UUID) -> pd.DataFrame:
             "Security Identifier": v.get("securityIdentifier"),
         }
 
-        df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
+        dfs.append(pd.DataFrame(new_data, index=[0]))
 
-    _update_dataframe_datatypes(dataframe=df, column_map=columns)
+    if dfs:
+        df = pd.concat(dfs, ignore_index=True)
+        _update_dataframe_datatypes(dataframe=df, column_map=columns)
 
     return df
 
@@ -200,6 +207,7 @@ def list_group_members(group: str | UUID) -> pd.DataFrame:
 
     df = _create_dataframe(columns=columns)
 
+    dfs = []
     for v in result.get("value"):
         new_data = {
             "Member Id": v.get("id"),
@@ -214,8 +222,10 @@ def list_group_members(group: str | UUID) -> pd.DataFrame:
             "Given Name": v.get("givenName"),
             "Surname": v.get("surname"),
         }
+        dfs.append(pd.DataFrame(new_data, index=[0]))
 
-        df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
+    if dfs:
+        df = pd.concat(dfs, ignore_index=True)
 
     return df
 
@@ -260,6 +270,7 @@ def list_group_owners(group: str | UUID) -> pd.DataFrame:
 
     df = _create_dataframe(columns=columns)
 
+    dfs = []
     for v in result.get("value"):
         new_data = {
             "Owner Id": v.get("id"),
@@ -274,12 +285,15 @@ def list_group_owners(group: str | UUID) -> pd.DataFrame:
             "Given Name": v.get("givenName"),
             "Surname": v.get("surname"),
         }
+        dfs.append(pd.DataFrame(new_data, index=[0]))
 
-        df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
+    if dfs:
+        df = pd.concat(dfs, ignore_index=True)
 
     return df
 
 
+@log
 def _base_add_to_group(
     group: str | UUID,
     object: str | UUID,
@@ -359,6 +373,7 @@ def add_group_members(
     _base_add_to_group(group=group, object=user, object_type="members")
 
 
+@log
 def add_group_owners(
     group: str | UUID,
     user: str | UUID | List[str | UUID],

sempy_labs/graph/_teams.py

@@ -42,6 +42,7 @@ def list_teams() -> pd.DataFrame:
 
     df = _create_dataframe(columns=columns)
 
+    dfs = []
     for v in result.get("value"):
         new_data = {
             "Team Id": v.get("id"),
@@ -58,13 +59,16 @@ def list_teams() -> pd.DataFrame:
             "Member Count": v.get("memberCount"),
         }
 
-        df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
+        dfs.append(pd.DataFrame(new_data, index=[0]))
 
-    _update_dataframe_datatypes(dataframe=df, column_map=columns)
+    if dfs:
+        df = pd.concat(dfs, ignore_index=True)
+        _update_dataframe_datatypes(dataframe=df, column_map=columns)
 
     return df
 
 
+@log
 def list_chats(user: str | UUID) -> pd.DataFrame:
     """
     In progress...
@@ -95,6 +99,7 @@ def list_chats(user: str | UUID) -> pd.DataFrame:
     return df
 
 
+@log
 def send_teams_message(chat_id: str, message: str):
     """
     In progress...

sempy_labs/graph/_users.py

@@ -10,6 +10,7 @@ from sempy_labs._helper_functions import (
 from sempy._utils._log import log
 
 
+@log
 def resolve_user_id(user: str | UUID) -> UUID:
     """
     Resolves the user ID from the user principal name or ID.

sempy_labs/lakehouse/_blobs.py

@@ -14,6 +14,7 @@ import pandas as pd
 from sempy.fabric.exceptions import FabricHTTPException
 
 
+@log
 def _request_blob_api(
     request: str,
     method: str = "get",

sempy_labs/lakehouse/_get_lakehouse_tables.py

@@ -40,6 +40,8 @@ def get_lakehouse_tables(
     This function can be executed in either a PySpark or pure Python notebook.
 
     This is a wrapper function for the following API: `Tables - List Tables <https://learn.microsoft.com/rest/api/fabric/lakehouse/tables/list-tables>`_ plus extended capabilities.
+    However, the above mentioned API does not support Lakehouse schemas (Preview) until it is in GA (General Availability). This version also supports schema
+    enabled Lakehouses.
 
     Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
 
@@ -68,6 +70,7 @@ def get_lakehouse_tables(
     columns = {
         "Workspace Name": "string",
         "Lakehouse Name": "string",
+        "Schema Name": "string",
         "Table Name": "string",
         "Format": "string",
         "Type": "string",
@@ -83,27 +86,57 @@ def get_lakehouse_tables(
     if count_rows:  # Setting countrows defaults to extended=True
         extended = True
 
-    responses = _base_api(
-        request=f"v1/workspaces/{workspace_id}/lakehouses/{lakehouse_id}/tables",
-        uses_pagination=True,
-        client="fabric_sp",
-    )
+    API_called = True
+    try:
+        responses = _base_api(
+            request=f"v1/workspaces/{workspace_id}/lakehouses/{lakehouse_id}/tables",
+            uses_pagination=True,
+            client="fabric_sp",
+        )
 
-    if not responses[0].get("data"):
-        return df
+    except Exception as e:
+        API_called = False
 
     dfs = []
-    for r in responses:
-        for i in r.get("data", []):
-            new_data = {
-                "Workspace Name": workspace_name,
-                "Lakehouse Name": lakehouse_name,
-                "Table Name": i.get("name"),
-                "Format": i.get("format"),
-                "Type": i.get("type"),
-                "Location": i.get("location"),
-            }
-            dfs.append(pd.DataFrame(new_data, index=[0]))
+    local_path = None
+    if API_called:
+        if not responses[0].get("data"):
+            return df
+
+        for r in responses:
+            for i in r.get("data", []):
+                new_data = {
+                    "Workspace Name": workspace_name,
+                    "Lakehouse Name": lakehouse_name,
+                    "Schema Name": "",
+                    "Table Name": i.get("name"),
+                    "Format": i.get("format"),
+                    "Type": i.get("type"),
+                    "Location": i.get("location"),
+                }
+                dfs.append(pd.DataFrame(new_data, index=[0]))
+    else:
+        local_path = _mount(lakehouse=lakehouse_id, workspace=workspace_id)
+        tables_path = os.path.join(local_path, "Tables")
+        list_schema = os.listdir(tables_path)
+
+        for schema_name in list_schema:
+            schema_table_path = os.path.join(local_path, "Tables", schema_name)
+            list_tables = os.listdir(schema_table_path)
+            for table_name in list_tables:
+                location_path = create_abfss_path(
+                    lakehouse_id, workspace_id, table_name, schema_name
+                )
+                new_data = {
+                    "Workspace Name": workspace_name,
+                    "Lakehouse Name": lakehouse_name,
+                    "Schema Name": schema_name,
+                    "Table Name": table_name,
+                    "Format": "delta",
+                    "Type": "Managed",
+                    "Location": location_path,
+                }
+                dfs.append(pd.DataFrame(new_data, index=[0]))
 
     if dfs:
         df = pd.concat(dfs, ignore_index=True)
@@ -111,17 +144,25 @@ def get_lakehouse_tables(
     if extended:
         sku_value = get_sku_size(workspace_id)
         guardrail = get_directlake_guardrails_for_sku(sku_value)
-        local_path = _mount(lakehouse=lakehouse_id, workspace=workspace_id)
+        # Avoid mounting the lakehouse if is already mounted
+        if not local_path:
+            local_path = _mount(lakehouse=lakehouse_id, workspace=workspace_id)
 
         df["Files"], df["Row Groups"], df["Table Size"] = None, None, None
         if count_rows:
             df["Row Count"] = None
 
         for i, r in df.iterrows():
+            use_schema = True
+            schema_name = r["Schema Name"]
             table_name = r["Table Name"]
             if r["Type"] == "Managed" and r["Format"] == "delta":
-                delta_table_path = create_abfss_path(
-                    lakehouse_id, workspace_id, table_name
+                delta_table_path = (
+                    create_abfss_path(
+                        lakehouse_id, workspace_id, table_name, schema_name
+                    )
+                    .replace("//", "/")  # When schema_name = ""
+                    .replace("abfss:/", "abfss://")  # Put back the // after abfss:
                 )
 
                 if _pure_python_notebook():
@@ -135,29 +176,46 @@ def get_lakehouse_tables(
                     size_in_bytes = 0
                     for f in latest_files:
                         local_file_path = os.path.join(
-                            local_path, "Tables", table_name, os.path.basename(f)
+                            local_path, "Tables", schema_name, table_name, f
                         )
+
                         if os.path.exists(local_file_path):
                             size_in_bytes += os.path.getsize(local_file_path)
                     num_latest_files = len(latest_files)
                 else:
                     delta_table = _get_delta_table(delta_table_path)
+
                     latest_files = _read_delta_table(delta_table_path).inputFiles()
                     table_df = delta_table.toDF()
                     table_details = delta_table.detail().collect()[0].asDict()
-                    num_latest_files = table_details.get("numFiles", 0)
                     size_in_bytes = table_details.get("sizeInBytes", 0)
+                    num_latest_files = table_details.get("numFiles", 0)
+
+                table_path = os.path.join(local_path, "Tables", schema_name, table_name)
 
-                table_path = os.path.join(local_path, "Tables", table_name)
-                file_paths = [os.path.basename(f) for f in latest_files]
+                file_paths = []
+                for file in latest_files:
+                    if _pure_python_notebook():
+                        file_paths.append(file)
+                    else:
+                        # Append the <Partition folder>/<filename> or <filename>
+                        find_table = file.find(table_name)
+                        len_file = len(file)
+                        len_table = len(table_name)
+                        last_chars = len_file - (find_table + len_table + 1)
+                        file_paths.append(file[-last_chars:])
 
                 num_rowgroups = 0
                 for filename in file_paths:
-                    parquet_file = pq.ParquetFile(f"{table_path}/{filename}")
-                    num_rowgroups += parquet_file.num_row_groups
+                    parquet_file_path = f"{table_path}/{filename}"
+                    if os.path.exists(parquet_file_path):
+                        parquet_file = pq.ParquetFile(parquet_file_path)
+                        num_rowgroups += parquet_file.num_row_groups
+
                 df.at[i, "Files"] = num_latest_files
                 df.at[i, "Row Groups"] = num_rowgroups
                 df.at[i, "Table Size"] = size_in_bytes
+
             if count_rows:
                 if _pure_python_notebook():
                     row_count = delta_table.to_pyarrow_table().num_rows
@@ -165,6 +223,9 @@ def get_lakehouse_tables(
                     row_count = table_df.count()
                 df.at[i, "Row Count"] = row_count
 
+            # Set "Schema Name" = "dbo" when it is ""
+            df.loc[df["Schema Name"] == "", "Schema Name"] = "dbo"
+
     if extended:
         intColumns = ["Files", "Row Groups", "Table Size"]
         df[intColumns] = df[intColumns].astype(int)

sempy_labs/lakehouse/_helper.py

@@ -54,6 +54,7 @@ def is_v_ordered(
     return any(b"vorder" in key for key in ds_schema.keys())
 
 
+@log
 def delete_lakehouse(
     lakehouse: str | UUID, workspace: Optional[str | UUID] = None
 ) -> None:
@@ -77,6 +78,7 @@ def delete_lakehouse(
     delete_item(item=lakehouse, item_type="lakehouse", workspace=workspace)
 
 
+@log
 def update_lakehouse(
     name: Optional[str] = None,
     description: Optional[str] = None,

sempy_labs/lakehouse/_lakehouse.py

@@ -11,8 +11,14 @@ from sempy_labs._helper_functions import (
 )
 import sempy_labs._icons as icons
 import re
+import time
+import pandas as pd
+from sempy_labs._job_scheduler import (
+    _get_item_job_instance,
+)
 
 
+@log
 def lakehouse_attached() -> bool:
     """
     Identifies if a lakehouse is attached to the notebook.
@@ -33,6 +39,7 @@ def lakehouse_attached() -> bool:
         return False
 
 
+@log
 def _optimize_table(path):
 
     if _pure_python_notebook():
@@ -46,6 +53,7 @@ def _optimize_table(path):
         DeltaTable.forPath(spark, path).optimize().executeCompaction()
 
 
+@log
 def _vacuum_table(path, retain_n_hours):
 
     if _pure_python_notebook():
@@ -145,6 +153,7 @@ def vacuum_lakehouse_tables(
         _vacuum_table(path=path, retain_n_hours=retain_n_hours)
 
 
+@log
 def run_table_maintenance(
     table_name: str,
     optimize: bool = False,
@@ -154,7 +163,7 @@ def run_table_maintenance(
     schema: Optional[str] = None,
     lakehouse: Optional[str | UUID] = None,
     workspace: Optional[str | UUID] = None,
-):
+) -> pd.DataFrame:
     """
     Runs table maintenance operations on the specified table within the lakehouse.
 
@@ -181,6 +190,11 @@ def run_table_maintenance(
         The Fabric workspace name or ID used by the lakehouse.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A DataFrame containing the job instance details of the table maintenance operation.
     """
 
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
@@ -223,13 +237,32 @@ def run_table_maintenance(
     if vacuum and retention_period is not None:
         payload["executionData"]["vacuumSettings"]["retentionPeriod"] = retention_period
 
-    _base_api(
+    response = _base_api(
         request=f"/v1/workspaces/{workspace_id}/lakehouses/{lakehouse_id}/jobs/instances?jobType=TableMaintenance",
         method="post",
         payload=payload,
         status_codes=202,
     )
 
-    print(
-        f"{icons.green_dot} The table maintenance job for the '{table_name}' table in the '{lakehouse_name}' lakehouse within the '{workspace_name}' workspace has been initiated."
-    )
+    f"{icons.in_progress} The table maintenance job for the '{table_name}' table in the '{lakehouse_name}' lakehouse within the '{workspace_name}' workspace has been initiated."
+
+    status_url = response.headers.get("Location").split("fabric.microsoft.com")[1]
+    status = None
+    while status not in ["Completed", "Failed"]:
+        response = _base_api(request=status_url)
+        status = response.json().get("status")
+        time.sleep(10)
+
+    df = _get_item_job_instance(url=status_url)
+
+    if status == "Completed":
+        print(
+            f"{icons.green_dot} The table maintenance job for the '{table_name}' table in the '{lakehouse_name}' lakehouse within the '{workspace_name}' workspace has succeeded."
+        )
+    else:
+        print(status)
+        print(
+            f"{icons.red_dot} The table maintenance job for the '{table_name}' table in the '{lakehouse_name}' lakehouse within the '{workspace_name}' workspace has failed."
+        )
+
+    return df

sempy_labs/lakehouse/_livy_sessions.py

@@ -8,8 +8,10 @@ from sempy_labs._helper_functions import (
 import pandas as pd
 from typing import Optional
 from uuid import UUID
+from sempy._utils._log import log
 
 
+@log
 def list_livy_sessions(
     lakehouse: Optional[str | UUID] = None, workspace: Optional[str | UUID] = None
 ) -> pd.DataFrame:
@@ -85,7 +87,6 @@ def list_livy_sessions(
     )
 
     dfs = []
-
     for r in responses:
         for v in r.get("value", []):
             queued_duration = v.get("queuedDuration", {})

sempy_labs/lakehouse/_shortcuts.py

@@ -216,6 +216,7 @@ def create_shortcut(
     )
 
 
+@log
 def delete_shortcut(
     shortcut_name: str,
     shortcut_path: str = "Tables",
@@ -260,6 +261,7 @@ def delete_shortcut(
     )
 
 
+@log
 def reset_shortcut_cache(workspace: Optional[str | UUID] = None):
     """
     Deletes any cached files that were stored while reading from shortcuts.
@@ -369,6 +371,7 @@ def list_shortcuts(
         "S3Compatible": "s3Compatible",
     }
 
+    dfs = []
     for r in responses:
         for i in r.get("value", []):
             tgt = i.get("target", {})
@@ -415,6 +418,9 @@ def list_shortcuts(
                 "SubPath": sub_path,
                 "Source Properties Raw": str(tgt),
             }
-            df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
+            dfs.append(pd.DataFrame(new_data, index=[0]))
+
+    if dfs:
+        df = pd.concat(dfs, ignore_index=True)
 
     return df

sempy_labs/migration/_direct_lake_to_import.py

@@ -2,8 +2,10 @@ import sempy
 from uuid import UUID
 import sempy_labs._icons as icons
 from typing import Optional
+from sempy._utils._log import log
 
 
+@log
 def migrate_direct_lake_to_import(
     dataset: str | UUID,
     workspace: Optional[str | UUID] = None,

sempy_labs/mirrored_azure_databricks_catalog/_discover.py

@@ -6,8 +6,10 @@ from sempy_labs._helper_functions import (
     _create_dataframe,
 )
 import pandas as pd
+from sempy._utils._log import log
 
 
+@log
 def discover_catalogs(
     databricks_workspace_connection_id: UUID,
     workspace: Optional[str | UUID] = None,
@@ -70,6 +72,7 @@ def discover_catalogs(
     return df
 
 
+@log
 def discover_schemas(
     catalog: str,
     databricks_workspace_connection_id: UUID,
@@ -135,6 +138,7 @@ def discover_schemas(
     return df
 
 
+@log
 def discover_tables(
     catalog: str,
     schema: str,

sempy_labs/mirrored_azure_databricks_catalog/_refresh_catalog_metadata.py

@@ -6,8 +6,10 @@ from sempy_labs._helper_functions import (
     _base_api,
 )
 import sempy_labs._icons as icons
+from sempy._utils._log import log
 
 
+@log
 def refresh_catalog_metadata(
     mirrored_azure_databricks_catalog: str | UUID,
     workspace: Optional[str | UUID] = None,

sempy_labs/report/_download_report.py

@@ -1,4 +1,3 @@
-import sempy.fabric as fabric
 import sempy_labs._icons as icons
 from typing import Optional
 from sempy_labs._helper_functions import (
@@ -11,8 +10,10 @@ from sempy_labs._helper_functions import (
 )
 from sempy_labs.lakehouse._lakehouse import lakehouse_attached
 from uuid import UUID
+from sempy._utils._log import log
 
 
+@log
 def download_report(
     report: str | UUID,
     file_name: Optional[str] = None,

sempy_labs/report/_generate_report.py

@@ -18,6 +18,7 @@ from sempy._utils._log import log
 from uuid import UUID
 
 
+@log
 def create_report_from_reportjson(
     report: str,
     dataset: str | UUID,
@@ -119,6 +120,7 @@ def create_report_from_reportjson(
     )
 
 
+@log
 def update_report_from_reportjson(
     report: str | UUID, report_json: dict, workspace: Optional[str | UUID] = None
 ):

sempy_labs/report/_paginated.py

@@ -7,8 +7,10 @@ from sempy_labs._helper_functions import (
     resolve_item_id,
     _create_dataframe,
 )
+from sempy._utils._log import log
 
 
+@log
 def get_report_datasources(
     report: str | UUID,
     workspace: Optional[str | UUID] = None,