semantic-link-labs 0.9.6__py3-none-any.whl → 0.9.8__py3-none-any.whl

sempy_labs/graph/_users.py

@@ -7,6 +7,7 @@ from sempy_labs._helper_functions import (
     _base_api,
     _create_dataframe,
 )
+from sempy._utils._log import log
 
 
 def resolve_user_id(user: str | UUID) -> UUID:
@@ -33,6 +34,7 @@ def resolve_user_id(user: str | UUID) -> UUID:
         return result.get("id")
 
 
+@log
 def get_user(user: str | UUID) -> pd.DataFrame:
     """
     Shows properties of a given user.
@@ -70,6 +72,7 @@ def get_user(user: str | UUID) -> pd.DataFrame:
     return pd.DataFrame([new_data])
 
 
+@log
 def list_users() -> pd.DataFrame:
     """
     Shows a list of users and their properties.
@@ -120,6 +123,7 @@ def list_users() -> pd.DataFrame:
     return df
 
 
+@log
 def send_mail(
     user: UUID | str,
     subject: str,

sempy_labs/lakehouse/__init__.py

@@ -1,12 +1,15 @@
-from sempy_labs.lakehouse._get_lakehouse_columns import get_lakehouse_columns
-from sempy_labs.lakehouse._get_lakehouse_tables import get_lakehouse_tables
+from sempy_labs.lakehouse._get_lakehouse_columns import (
+    get_lakehouse_columns,
+)
+from sempy_labs.lakehouse._get_lakehouse_tables import (
+    get_lakehouse_tables,
+)
 from sempy_labs.lakehouse._lakehouse import (
     lakehouse_attached,
     optimize_lakehouse_tables,
     vacuum_lakehouse_tables,
     run_table_maintenance,
 )
-
 from sempy_labs.lakehouse._shortcuts import (
     # create_shortcut,
     create_shortcut_onelake,
@@ -14,6 +17,10 @@ from sempy_labs.lakehouse._shortcuts import (
     reset_shortcut_cache,
     list_shortcuts,
 )
+from sempy_labs.lakehouse._blobs import (
+    recover_lakehouse_object,
+    list_blobs,
+)
 
 __all__ = [
     "get_lakehouse_columns",
@@ -27,4 +34,6 @@ __all__ = [
     "reset_shortcut_cache",
     "run_table_maintenance",
     "list_shortcuts",
+    "recover_lakehouse_object",
+    "list_blobs",
 ]

sempy_labs/lakehouse/_blobs.py

@@ -0,0 +1,231 @@
+from sempy_labs._helper_functions import (
+    resolve_workspace_id,
+    resolve_lakehouse_id,
+    _xml_to_dict,
+    _create_dataframe,
+    _update_dataframe_datatypes,
+)
+from sempy._utils._log import log
+from uuid import UUID
+from typing import Optional, List
+import sempy_labs._icons as icons
+import xml.etree.ElementTree as ET
+import pandas as pd
+
+
+def _request_blob_api(
+    request: str,
+    method: str = "get",
+    payload: Optional[dict] = None,
+    status_codes: int | List[int] = 200,
+):
+
+    import requests
+    import notebookutils
+    from sempy.fabric.exceptions import FabricHTTPException
+
+    if isinstance(status_codes, int):
+        status_codes = [status_codes]
+
+    token = notebookutils.credentials.getToken("storage")
+
+    headers = {
+        "Authorization": f"Bearer {token}",
+        "Content-Type": "application/json",
+        "x-ms-version": "2025-05-05",
+    }
+
+    response = requests.request(
+        method.upper(),
+        f"https://onelake.blob.fabric.microsoft.com/{request}",
+        headers=headers,
+        json=payload,
+    )
+
+    if response.status_code not in status_codes:
+        raise FabricHTTPException(response)
+
+    return response
+
+
+@log
+def list_blobs(
+    lakehouse: Optional[str | UUID] = None,
+    workspace: Optional[str | UUID] = None,
+    container: Optional[str] = None,
+) -> pd.DataFrame:
+    """
+    Returns a list of blobs for a given lakehouse.
+
+    This function leverages the following API: `List Blobs <https://learn.microsoft.com/rest/api/storageservices/list-blobs?tabs=microsoft-entra-id>`_.
+
+    Parameters
+    ----------
+    lakehouse : str | uuid.UUID, default=None
+        The Fabric lakehouse name or ID.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID used by the lakehouse.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    container : str, default=None
+        The container name to list blobs from. If None, lists all blobs in the lakehouse.
+        Valid values are "Tables" or "Files". If not specified, the function will list all blobs in the lakehouse.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing a list of blobs in the lakehouse.
+    """
+
+    workspace_id = resolve_workspace_id(workspace)
+    lakehouse_id = resolve_lakehouse_id(lakehouse, workspace_id)
+
+    if container is None:
+        path_prefix = f"{workspace_id}/{lakehouse_id}"
+    else:
+        if container not in ["Tables", "Files"]:
+            raise ValueError(
+                f"{icons.red_dot} Invalid container '{container}' within the file_path parameter. Expected 'Tables' or 'Files'."
+            )
+        path_prefix = f"{workspace_id}/{lakehouse_id}/{container}"
+
+    response = _request_blob_api(
+        request=f"{path_prefix}?restype=container&comp=list&include=deleted"
+    )
+    root = ET.fromstring(response.content)
+    response_json = _xml_to_dict(root)
+
+    columns = {
+        "Blob Name": "str",
+        "Is Deleted": "bool",
+        "Deletion Id": "str",
+        "Creation Time": "datetime",
+        "Expiry Time": "datetime",
+        "Etag": "str",
+        "Resource Type": "str",
+        "Content Length": "int",
+        "Content Type": "str",
+        "Content Encoding": "str",
+        "Content Language": "str",
+        "Content CRC64": "str",
+        "Content MD5": "str",
+        "Cache Control": "str",
+        "Content Disposition": "str",
+        "Blob Type": "str",
+        "Access Tier": "str",
+        "Access Tier Inferred": "str",
+        "Server Encrypted": "bool",
+        "Deleted Time": "str",
+        "Remaining Retention Days": "str",
+    }
+
+    df = _create_dataframe(columns=columns)
+
+    for blob in (
+        response_json.get("EnumerationResults", {}).get("Blobs", {}).get("Blob", {})
+    ):
+        p = blob.get("Properties", {})
+        new_data = {
+            "Blob Name": blob.get("Name"),
+            "Is Deleted": blob.get("Deleted", False),
+            "Deletion Id": blob.get("DeletionId"),
+            "Creation Time": p.get("Creation-Time"),
+            "Expiry Time": p.get("Expiry-Time"),
+            "Etag": p.get("Etag"),
+            "Resource Type": p.get("ResourceType"),
+            "Content Length": p.get("Content-Length"),
+            "Content Type": p.get("Content-Type"),
+            "Content Encoding": p.get("Content-Encoding"),
+            "Content Language": p.get("Content-Language"),
+            "Content CRC64": p.get("Content-CRC64"),
+            "Content MD5": p.get("Content-MD5"),
+            "Cache Control": p.get("Cache-Control"),
+            "Content Disposition": p.get("Content-Disposition"),
+            "Blob Type": p.get("BlobType"),
+            "Access Tier": p.get("AccessTier"),
+            "Access Tier Inferred": p.get("AccessTierInferred"),
+            "Server Encrypted": p.get("ServerEncrypted"),
+            "Deleted Time": p.get("DeletedTime"),
+            "Remaining Retention Days": p.get("RemainingRetentionDays"),
+        }
+
+        df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
+
+    _update_dataframe_datatypes(dataframe=df, column_map=columns)
+
+    return df
+
+
+@log
+def recover_lakehouse_object(
+    file_path: str,
+    lakehouse: Optional[str | UUID] = None,
+    workspace: Optional[str | UUID] = None,
+):
+    """
+    Recovers an object (i.e. table, file, folder) in a lakehouse from a deleted state. Only `soft-deleted objects <https://learn.microsoft.com/fabric/onelake/onelake-disaster-recovery#soft-delete-for-onelake-files>`_ can be recovered (deleted for less than 7 days).
+
+    Parameters
+    ----------
+    file_path : str
+        The file path of the object to restore. For example: "Tables/my_delta_table".
+    lakehouse : str | uuid.UUID, default=None
+        The Fabric lakehouse name or ID.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID used by the lakehouse.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    workspace_id = resolve_workspace_id(workspace)
+    lakehouse_id = resolve_lakehouse_id(lakehouse, workspace_id)
+
+    blob_path_prefix = f"{lakehouse_id}/{file_path}"
+
+    container = file_path.split("/")[0]
+    if container not in ["Tables", "Files"]:
+        raise ValueError(
+            f"{icons.red_dot} Invalid container '{container}' within the file_path parameter. Expected 'Tables' or 'Files'."
+        )
+
+    df = list_blobs(lakehouse=lakehouse, workspace=workspace, container=container)
+
+    for _, r in df.iterrows():
+        blob_name = r.get("Blob Name")
+        is_deleted = r.get("Is Deleted")
+        if blob_name.startswith(blob_path_prefix) and is_deleted:
+            print(f"{icons.in_progress} Restoring the '{blob_name}' blob...")
+            _request_blob_api(
+                request=f"{workspace_id}/{lakehouse_id}/{file_path}?comp=undelete",
+                method="put",
+            )
+            print(f"{icons.green_dot} The '{blob_name}' blob has been restored.")
+
+
+def _get_user_delegation_key():
+
+    # https://learn.microsoft.com/rest/api/storageservices/get-user-delegation-key
+
+    from datetime import datetime, timedelta, timezone
+
+    utc_now = datetime.now(timezone.utc)
+    start_time = utc_now + timedelta(minutes=2)
+    expiry_time = start_time + timedelta(minutes=45)
+    start_str = start_time.strftime("%Y-%m-%dT%H:%M:%SZ")
+    expiry_str = expiry_time.strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    payload = f"""<?xml version="1.0" encoding="utf-8"?>
+    <KeyInfo>
+        <Start>{start_str}</Start>
+        <Expiry>{expiry_str}</Expiry>
+    </KeyInfo>"""
+
+    response = _request_blob_api(
+        request="restype=service&comp=userdelegationkey",
+        method="post",
+        payload=payload,
+    )
+
+    return response.content

sempy_labs/lakehouse/_shortcuts.py

@@ -24,12 +24,15 @@ def create_shortcut_onelake(
     shortcut_name: Optional[str] = None,
     source_path: str = "Tables",
     destination_path: str = "Tables",
+    shortcut_conflict_policy: Optional[str] = None,
 ):
     """
     Creates a `shortcut <https://learn.microsoft.com/fabric/onelake/onelake-shortcuts>`_ to a delta table in OneLake.
 
     This is a wrapper function for the following API: `OneLake Shortcuts - Create Shortcut <https://learn.microsoft.com/rest/api/fabric/core/onelake-shortcuts/create-shortcut>`_.
 
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
     Parameters
     ----------
     table_name : str
@@ -51,6 +54,8 @@ def create_shortcut_onelake(
         A string representing the full path to the table/file in the source lakehouse, including either "Files" or "Tables". Examples: Tables/FolderName/SubFolderName; Files/FolderName/SubFolderName.
     destination_path: str, default="Tables"
         A string representing the full path where the shortcut is created, including either "Files" or "Tables". Examples: Tables/FolderName/SubFolderName; Files/FolderName/SubFolderName.
+    shortcut_conflict_policy : str, default=None
+        When provided, it defines the action to take when a shortcut with the same name and path already exists. The default action is 'Abort'. Additional ShortcutConflictPolicy types may be added over time.
     """
 
     if not (source_path.startswith("Files") or source_path.startswith("Tables")):
@@ -103,7 +108,8 @@ def create_shortcut_onelake(
     # Check if the shortcut already exists
     try:
         response = _base_api(
-            request=f"/v1/workspaces/{destination_workspace_id}/items/{destination_lakehouse_id}/shortcuts/{destination_path}/{actual_shortcut_name}"
+            request=f"/v1/workspaces/{destination_workspace_id}/items/{destination_lakehouse_id}/shortcuts/{destination_path}/{actual_shortcut_name}",
+            client="fabric_sp",
         )
         response_json = response.json()
         del response_json["target"]["type"]
@@ -119,11 +125,21 @@ def create_shortcut_onelake(
     except FabricHTTPException:
         pass
 
+    url = f"/v1/workspaces/{destination_workspace_id}/items/{destination_lakehouse_id}/shortcuts"
+
+    if shortcut_conflict_policy:
+        if shortcut_conflict_policy not in ["Abort", "GenerateUniqueName"]:
+            raise ValueError(
+                f"{icons.red_dot} The 'shortcut_conflict_policy' parameter must be either 'Abort' or 'GenerateUniqueName'."
+            )
+        url += f"?shortcutConflictPolicy={shortcut_conflict_policy}"
+
     _base_api(
-        request=f"/v1/workspaces/{destination_workspace_id}/items/{destination_lakehouse_id}/shortcuts",
+        request=url,
         payload=payload,
         status_codes=201,
         method="post",
+        client="fabric_sp",
     )
 
     print(
@@ -211,6 +227,8 @@ def delete_shortcut(
 
     This is a wrapper function for the following API: `OneLake Shortcuts - Delete Shortcut <https://learn.microsoft.com/rest/api/fabric/core/onelake-shortcuts/delete-shortcut>`_.
 
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
     Parameters
     ----------
     shortcut_name : str
@@ -234,6 +252,7 @@ def delete_shortcut(
     _base_api(
         request=f"/v1/workspaces/{workspace_id}/items/{lakehouse_id}/shortcuts/{shortcut_path}/{shortcut_name}",
         method="delete",
+        client="fabric_sp",
     )
 
     print(
@@ -288,7 +307,7 @@ def list_shortcuts(
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
     path: str, default=None
-        The path within lakehouse where to look for shortcuts. If provied, must start with either "Files" or "Tables". Examples: Tables/FolderName/SubFolderName; Files/FolderName/SubFolderName.
+        The path within lakehouse where to look for shortcuts. If provided, must start with either "Files" or "Tables". Examples: Tables/FolderName/SubFolderName; Files/FolderName/SubFolderName.
         Defaults to None which will retun all shortcuts on the given lakehouse
 
     Returns

sempy_labs/migration/_direct_lake_to_import.py

@@ -1,11 +1,16 @@
 import sempy
 from uuid import UUID
 import sempy_labs._icons as icons
+from typing import Optional
 
 
-def migrate_direct_lake_to_import(dataset: str | UUID, workspace: str | UUID):
+def migrate_direct_lake_to_import(
+    dataset: str | UUID,
+    workspace: Optional[str | UUID] = None,
+    mode: str = "import",
+):
     """
-    Migrates a semantic model from Direct Lake mode to import mode. After running this function, you must go to the semantic model settings and update the cloud connection. Not doing so will result in an inablity to refresh/use the semantic model.
+    Migrates a semantic model or specific table(s) from a Direct Lake mode to import or DirectQuery mode. After running this function, you must go to the semantic model settings and update the cloud connection. Not doing so will result in an inablity to refresh/use the semantic model.
 
     Parameters
     ----------
@@ -15,12 +20,29 @@ def migrate_direct_lake_to_import(dataset: str | UUID, workspace: str | UUID):
         The Fabric workspace name or ID.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
+    mode : str, default="import"
+        The mode to migrate to. Can be either "import" or "directquery".
     """
 
     sempy.fabric._client._utils._init_analysis_services()
     import Microsoft.AnalysisServices.Tabular as TOM
     from sempy_labs.tom import connect_semantic_model
 
+    modes = {
+        "import": "Import",
+        "directquery": "DirectQuery",
+        "dq": "DirectQuery",
+    }
+
+    # Resolve mode
+    mode = mode.lower()
+    actual_mode = modes.get(mode)
+    if actual_mode is None:
+        raise ValueError(f"Invalid mode '{mode}'. Must be one of {list(modes.keys())}.")
+
+    # if isinstance(tables, str):
+    #     tables = [tables]
+
     with connect_semantic_model(
         dataset=dataset, workspace=workspace, readonly=False
     ) as tom:
@@ -31,7 +53,14 @@ def migrate_direct_lake_to_import(dataset: str | UUID, workspace: str | UUID):
             )
             return
 
-        for t in tom.model.Tables:
+        # if tables is None:
+        table_list = [t for t in tom.model.Tables]
+        # else:
+        #    table_list = [t for t in tom.model.Tables if t.Name in tables]
+        # if not table_list:
+        #    raise ValueError(f"{icons.red_dot} No tables found to migrate.")
+
+        for t in table_list:
             table_name = t.Name
             if t.Partitions.Count == 1 and all(
                 p.Mode == TOM.ModeType.DirectLake for p in t.Partitions
@@ -51,16 +80,24 @@ def migrate_direct_lake_to_import(dataset: str | UUID, workspace: str | UUID):
                     table_name=table_name,
                     partition_name=partition_name,
                     expression=expression,
-                    mode="Import",
+                    mode=actual_mode,
                 )
                 # Remove Direct Lake partition
                 tom.remove_object(object=p)
+                # if tables is not None:
+                #    print(
+                #        f"{icons.green_dot} The '{table_name}' table has been migrated to '{actual_mode}' mode."
+                #    )
 
         tom.model.Model.DefaultMode = TOM.ModeType.Import
+    # if tables is None:
+    print(
+        f"{icons.green_dot} All tables which were in Direct Lake mode have been migrated to '{actual_mode}' mode."
+    )
 
-        # Check
-        # for t in tom.model.Tables:
-        #    if t.Partitions.Count == 1 and all(p.Mode == TOM.ModeType.Import for p in t.Partitions) and t.CalculationGroup is None:
-        #        p = next(p for p in t.Partitions)
-        #        print(p.Name)
-        #        print(p.Source.Expression)
+    # Check
+    # for t in tom.model.Tables:
+    #    if t.Partitions.Count == 1 and all(p.Mode == TOM.ModeType.Import for p in t.Partitions) and t.CalculationGroup is None:
+    #        p = next(p for p in t.Partitions)
+    #        print(p.Name)
+    #        print(p.Source.Expression)

sempy_labs/report/__init__.py

@@ -1,3 +1,6 @@
+from sempy_labs.report._save_report import (
+    save_report_as_pbip,
+)
 from sempy_labs.report._reportwrapper import (
     ReportWrapper,
 )
@@ -46,4 +49,5 @@ __all__ = [
     "run_report_bpa",
     "get_report_datasources",
     "download_report",
+    "save_report_as_pbip",
 ]

sempy_labs/report/_report_functions.py

@@ -116,9 +116,9 @@ def report_dependency_tree(workspace: Optional[str | UUID] = None):
     dfR.rename(columns={"Name": "Report Name"}, inplace=True)
     dfR = dfR[["Report Name", "Dataset Name"]]
 
-    report_icon = "\U0001F4F6"
-    dataset_icon = "\U0001F9CA"
-    workspace_icon = "\U0001F465"
+    report_icon = "\U0001f4f6"
+    dataset_icon = "\U0001f9ca"
+    workspace_icon = "\U0001f465"
 
     node_dict = {}
     rootNode = Node(workspace_name)

sempy_labs/report/_report_helper.py

@@ -236,15 +236,27 @@ def find_entity_property_pairs(data, result=None, keys_path=None):
         keys_path = []
 
     if isinstance(data, dict):
+        expression = data.get("Expression", {})
+        source_ref = (
+            expression.get("SourceRef", {}) if isinstance(expression, dict) else {}
+        )
+
         if (
-            "Entity" in data.get("Expression", {}).get("SourceRef", {})
+            isinstance(source_ref, dict)
+            and "Entity" in source_ref
             and "Property" in data
         ):
-            entity = data.get("Expression", {}).get("SourceRef", {}).get("Entity", {})
-            property_value = data.get("Property")
-            object_type = keys_path[-1].replace("HierarchyLevel", "Hierarchy")
+            entity = source_ref.get("Entity", "")
+            property_value = data.get("Property", "")
+
+            object_type = (
+                keys_path[-1].replace("HierarchyLevel", "Hierarchy")
+                if keys_path
+                else "Unknown"
+            )
             result[property_value] = (entity, object_type)
-            keys_path.pop()
+            if keys_path:
+                keys_path.pop()
 
         # Recursively search the rest of the dictionary
         for key, value in data.items():

sempy_labs/report/_reportwrapper.py

@@ -966,17 +966,25 @@ class ReportWrapper:
                 keys_path = []
 
             if isinstance(data, dict):
+                expression = data.get("Expression", {})
+                source_ref = (
+                    expression.get("SourceRef", {})
+                    if isinstance(expression, dict)
+                    else {}
+                )
                 if (
-                    "Entity" in data.get("Expression", {}).get("SourceRef", {})
+                    isinstance(source_ref, dict)
+                    and "Entity" in source_ref
                     and "Property" in data
                 ):
-                    entity = (
-                        data.get("Expression", {})
-                        .get("SourceRef", {})
-                        .get("Entity", {})
+                    entity = source_ref.get("Entity", "")
+                    property_value = data.get("Property", "")
+
+                    object_type = (
+                        keys_path[-1].replace("HierarchyLevel", "Hierarchy")
+                        if keys_path
+                        else "Unknown"
                     )
-                    property_value = data.get("Property", {})
-                    object_type = keys_path[-1].replace("HierarchyLevel", "Hierarchy")
                     is_agg = keys_path[-3] == "Aggregation"
                     is_viz_calc = keys_path[-3] == "NativeVisualCalculation"
                     is_sparkline = keys_path[-3] == "SparklineData"
@@ -987,7 +995,8 @@ class ReportWrapper:
                         is_viz_calc,
                         is_sparkline,
                     )
-                    keys_path.pop()
+                    if keys_path:
+                        keys_path.pop()
 
                 # Recursively search the rest of the dictionary
                 for key, value in data.items():