semantic-link-labs 0.12.8__py3-none-any.whl

sempy_labs/graph_model/_items.py

@@ -0,0 +1,149 @@
+import pandas as pd
+from uuid import UUID
+from typing import Optional
+from sempy._utils._log import log
+from sempy_labs._helper_functions import (
+    _base_api,
+    _create_dataframe,
+    resolve_item_id,
+    resolve_workspace_id,
+    resolve_item_name_and_id,
+    resolve_workspace_name_and_id,
+)
+import sempy_labs._icons as icons
+
+
+@log
+def list_graph_models(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
+    """
+    Shows the graph models within a workspace.
+
+    This is a wrapper function for the following API: `Items - List Graph Models <https://learn.microsoft.com/rest/api/fabric/graphmodel/items/list-graph-models>`_.
+
+    Parameters
+    ----------
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing the graph models within a workspace.
+    """
+
+    columns = {
+        "Graph Model Name": "string",
+        "Graph Model Id": "string",
+        "Description": "string",
+        "OneLake Root Path": "string",
+    }
+    df = _create_dataframe(columns=columns)
+
+    workspace_id = resolve_workspace_id(workspace)
+
+    responses = _base_api(
+        request=f"/v1/workspaces/{workspace_id}/GraphModels",
+        uses_pagination=True,
+    )
+
+    rows = []
+    for r in responses:
+        for v in r.get("value", []):
+            rows.append(
+                {
+                    "Graph Model Name": v.get("displayName"),
+                    "Graph Model Id": v.get("id"),
+                    "Description": v.get("description"),
+                    "OneLake Root Path": v.get("properties", {}).get("oneLakeRootPath"),
+                }
+            )
+
+    if rows:
+        df = pd.DataFrame(rows, columns=list(columns.keys()))
+
+    return df
+
+
+@log
+def execute_query(
+    graph_model: str | UUID, query: str, workspace: Optional[str | UUID] = None
+) -> dict:
+    """
+    Executes a query on the specified graph model.
+
+    This is a wrapper function for the following API: `Items - ExecuteQuery <https://learn.microsoft.com/rest/api/fabric/graphmodel/items/execute-query(preview)>`_.
+
+    Parameters
+    ----------
+    graph_model : str | uuid.UUID
+        The graph model name or ID.
+    query : str
+        The query string.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    dict
+        The response from the API.
+    """
+
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    (item_name, item_id) = resolve_item_name_and_id(
+        item=graph_model, type="GraphModel", workspace=workspace_id
+    )
+
+    payload = {
+        "query": query,
+    }
+    response = _base_api(
+        request=f"/v1/workspaces/{workspace_id}/GraphModels/{item_id}/executeQuery?preview=True",
+        method="post",
+        payload=payload,
+    )
+
+    print(
+        f"{icons.green_dot} Executed query on Graph Model '{item_name}' in workspace '{workspace_name}' successfully."
+    )
+
+    return response.json()
+
+
+@log
+def get_queryable_graph_type(
+    graph_model: str | UUID, workspace: Optional[str | UUID] = None
+) -> dict:
+    """
+    Gets the current queryable graph type.
+
+    This is a wrapper function for the following API: `Items - GetQueryableGraphType <https://learn.microsoft.com/rest/api/fabric/graphmodel/items/get-queryable-graph-type(preview)>`_.
+
+    Parameters
+    ----------
+    graph_model : str | uuid.UUID
+        The graph model name or ID.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    dict
+        A dictionary showing the current queryable graph type.
+    """
+
+    workspace_id = resolve_workspace_id(workspace)
+    item_id = resolve_item_id(
+        item=graph_model, type="GraphModel", workspace=workspace_id
+    )
+
+    response = _base_api(
+        request=f"/v1/workspaces/{workspace_id}/GraphModels/{item_id}/getQueryableGraphType?preview=True"
+    )
+
+    return response.json()

sempy_labs/lakehouse/__init__.py

@@ -0,0 +1,67 @@
+from ._get_lakehouse_columns import (
+    get_lakehouse_columns,
+)
+from ._get_lakehouse_tables import (
+    get_lakehouse_tables,
+)
+from ._lakehouse import (
+    lakehouse_attached,
+    optimize_lakehouse_tables,
+    vacuum_lakehouse_tables,
+    run_table_maintenance,
+    list_lakehouses,
+)
+from ._shortcuts import (
+    # create_shortcut,
+    create_shortcut_onelake,
+    delete_shortcut,
+    reset_shortcut_cache,
+    list_shortcuts,
+)
+from ._blobs import (
+    recover_lakehouse_object,
+    list_blobs,
+)
+from ._livy_sessions import (
+    list_livy_sessions,
+)
+from ._helper import (
+    is_v_ordered,
+    delete_lakehouse,
+    update_lakehouse,
+    load_table,
+)
+from ._materialized_lake_views import (
+    refresh_materialized_lake_views,
+)
+from ._schemas import (
+    list_schemas,
+    schema_exists,
+    is_schema_enabled,
+)
+
+__all__ = [
+    "get_lakehouse_columns",
+    "get_lakehouse_tables",
+    "lakehouse_attached",
+    "optimize_lakehouse_tables",
+    # create_shortcut,
+    "create_shortcut_onelake",
+    "delete_shortcut",
+    "vacuum_lakehouse_tables",
+    "reset_shortcut_cache",
+    "run_table_maintenance",
+    "list_shortcuts",
+    "recover_lakehouse_object",
+    "list_blobs",
+    "list_livy_sessions",
+    "is_v_ordered",
+    "delete_lakehouse",
+    "update_lakehouse",
+    "load_table",
+    "refresh_materialized_lake_views",
+    "list_lakehouses",
+    "list_schemas",
+    "schema_exists",
+    "is_schema_enabled",
+]

sempy_labs/lakehouse/_blobs.py

@@ -0,0 +1,247 @@
+from sempy_labs._helper_functions import (
+    resolve_workspace_id,
+    resolve_lakehouse_id,
+    _xml_to_dict,
+    _create_dataframe,
+    _update_dataframe_datatypes,
+)
+from sempy._utils._log import log
+from uuid import UUID
+from typing import Optional, List
+import sempy_labs._icons as icons
+import xml.etree.ElementTree as ET
+import pandas as pd
+from sempy.fabric.exceptions import FabricHTTPException
+
+
+@log
+def _request_blob_api(
+    request: str,
+    method: str = "get",
+    payload: Optional[dict] = None,
+    status_codes: int | List[int] = 200,
+    uses_pagination: bool = False,
+):
+
+    import requests
+    import notebookutils
+    from sempy.fabric.exceptions import FabricHTTPException
+
+    if isinstance(status_codes, int):
+        status_codes = [status_codes]
+
+    token = notebookutils.credentials.getToken("storage")
+
+    headers = {
+        "Authorization": f"Bearer {token}",
+        "Content-Type": "application/xml",
+        "x-ms-version": "2025-05-05",
+    }
+
+    base_url = "https://onelake.blob.fabric.microsoft.com/"
+    full_url = f"{base_url}{request}"
+    results = []
+
+    while True:
+        response = requests.request(
+            method.upper(),
+            full_url,
+            headers=headers,
+            data=payload if method.lower() != "get" else None,
+        )
+
+        if response.status_code not in status_codes:
+            raise FabricHTTPException(response)
+
+        if not uses_pagination:
+            return response
+
+        # Parse XML to find blobs and NextMarker
+        root = ET.fromstring(response.content)
+        results.append(root)
+
+        next_marker = root.findtext(".//NextMarker")
+        if not next_marker:
+            break  # No more pages
+
+        # Append the marker to the original request (assuming query string format)
+        delimiter = "&" if "?" in request else "?"
+        full_url = f"{base_url}{request}{delimiter}marker={next_marker}"
+
+    return results
+
+
+@log
+def list_blobs(
+    lakehouse: Optional[str | UUID] = None,
+    workspace: Optional[str | UUID] = None,
+    container: Optional[str] = None,
+) -> pd.DataFrame:
+    """
+    Returns a list of blobs for a given lakehouse.
+
+    This function leverages the following API: `List Blobs <https://learn.microsoft.com/rest/api/storageservices/list-blobs?tabs=microsoft-entra-id>`_.
+
+    Parameters
+    ----------
+    lakehouse : str | uuid.UUID, default=None
+        The Fabric lakehouse name or ID.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID used by the lakehouse.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    container : str, default=None
+        The container name to list blobs from. If None, lists all blobs in the lakehouse.
+        Valid values are "Tables" or "Files". If not specified, the function will list all blobs in the lakehouse.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing a list of blobs in the lakehouse.
+    """
+
+    workspace_id = resolve_workspace_id(workspace)
+    lakehouse_id = resolve_lakehouse_id(lakehouse, workspace_id)
+
+    if container is None:
+        path_prefix = f"{workspace_id}/{lakehouse_id}"
+    else:
+        if container not in ["Tables", "Files"]:
+            raise ValueError(
+                f"{icons.red_dot} Invalid container '{container}' within the file_path parameter. Expected 'Tables' or 'Files'."
+            )
+        path_prefix = f"{workspace_id}/{lakehouse_id}/{container}"
+
+    columns = {
+        "Blob Name": "str",
+        "Is Deleted": "bool",
+        "Deletion Id": "str",
+        "Creation Time": "datetime",
+        "Expiry Time": "datetime",
+        "Etag": "str",
+        "Resource Type": "str",
+        "Content Length": "int",
+        "Content Type": "str",
+        "Content Encoding": "str",
+        "Content Language": "str",
+        "Content CRC64": "str",
+        "Content MD5": "str",
+        "Cache Control": "str",
+        "Content Disposition": "str",
+        "Blob Type": "str",
+        "Access Tier": "str",
+        "Access Tier Inferred": "str",
+        "Server Encrypted": "bool",
+        "Deleted Time": "str",
+        "Remaining Retention Days": "str",
+    }
+
+    df = _create_dataframe(columns=columns)
+
+    url = f"{path_prefix}?restype=container&comp=list&include=deleted"
+
+    responses = _request_blob_api(
+        request=url,
+        uses_pagination=True,
+    )
+
+    rows = []
+    for root in responses:
+        response_json = _xml_to_dict(root)
+
+        blobs = (
+            response_json.get("EnumerationResults", {}).get("Blobs", {}).get("Blob", [])
+        )
+
+        if isinstance(blobs, dict):
+            blobs = [blobs]
+
+        for blob in blobs:
+            p = blob.get("Properties", {})
+            rows.append(
+                {
+                    "Blob Name": blob.get("Name"),
+                    "Is Deleted": blob.get("Deleted", False),
+                    "Deletion Id": blob.get("DeletionId"),
+                    "Creation Time": p.get("Creation-Time"),
+                    "Expiry Time": p.get("Expiry-Time"),
+                    "Etag": p.get("Etag"),
+                    "Resource Type": p.get("ResourceType"),
+                    "Content Length": p.get("Content-Length"),
+                    "Content Type": p.get("Content-Type"),
+                    "Content Encoding": p.get("Content-Encoding"),
+                    "Content Language": p.get("Content-Language"),
+                    "Content CRC64": p.get("Content-CRC64"),
+                    "Content MD5": p.get("Content-MD5"),
+                    "Cache Control": p.get("Cache-Control"),
+                    "Content Disposition": p.get("Content-Disposition"),
+                    "Blob Type": p.get("BlobType"),
+                    "Access Tier": p.get("AccessTier"),
+                    "Access Tier Inferred": p.get("AccessTierInferred"),
+                    "Server Encrypted": p.get("ServerEncrypted"),
+                    "Deleted Time": p.get("DeletedTime"),
+                    "Remaining Retention Days": p.get("RemainingRetentionDays"),
+                }
+            )
+
+    if rows:
+        df = pd.DataFrame(rows, columns=list(columns.keys()))
+        _update_dataframe_datatypes(dataframe=df, column_map=columns)
+
+    return df
+
+
+@log
+def recover_lakehouse_object(
+    file_path: str,
+    lakehouse: Optional[str | UUID] = None,
+    workspace: Optional[str | UUID] = None,
+):
+    """
+    Recovers an object (i.e. table, file, folder) in a lakehouse from a deleted state. Only `soft-deleted objects <https://learn.microsoft.com/fabric/onelake/onelake-disaster-recovery#soft-delete-for-onelake-files>`_ can be recovered (deleted for less than 7 days).
+
+    Parameters
+    ----------
+    file_path : str
+        The file path of the object to restore. For example: "Tables/my_delta_table".
+    lakehouse : str | uuid.UUID, default=None
+        The Fabric lakehouse name or ID.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID used by the lakehouse.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    workspace_id = resolve_workspace_id(workspace)
+    lakehouse_id = resolve_lakehouse_id(lakehouse, workspace_id)
+
+    blob_name = f"{lakehouse_id}/{file_path}"
+
+    container = file_path.split("/")[0]
+    if container not in ["Tables", "Files"]:
+        raise ValueError(
+            f"{icons.red_dot} Invalid container '{container}' within the file_path parameter. Expected 'Tables' or 'Files'."
+        )
+
+    # Undelete the blob
+    print(f"{icons.in_progress} Attempting to recover the '{blob_name}' blob...")
+
+    try:
+        _request_blob_api(
+            request=f"{workspace_id}/{lakehouse_id}/{file_path}?comp=undelete",
+            method="put",
+        )
+        print(
+            f"{icons.green_dot} The '{blob_name}' blob recover attempt was successful."
+        )
+    except FabricHTTPException as e:
+        if e.status_code == 404:
+            print(
+                f"{icons.warning} The '{blob_name}' blob was not found. No action taken."
+            )
+        else:
+            print(
+                f"{icons.red_dot} An error occurred while recovering the '{blob_name}' blob: {e}"
+            )

sempy_labs/lakehouse/_get_lakehouse_columns.py

@@ -0,0 +1,102 @@
+import pandas as pd
+import re
+from sempy_labs._helper_functions import (
+    format_dax_object_name,
+    resolve_workspace_name_and_id,
+    resolve_lakehouse_name_and_id,
+    _create_dataframe,
+    _get_delta_table,
+    _pure_python_notebook,
+)
+from typing import Optional
+from sempy._utils._log import log
+from uuid import UUID
+import sempy_labs._icons as icons
+
+
+@log
+def get_lakehouse_columns(
+    lakehouse: Optional[str | UUID] = None, workspace: Optional[str | UUID] = None
+) -> pd.DataFrame:
+    """
+    Shows the tables and columns of a lakehouse and their respective properties. This function can be executed in either a PySpark or pure Python notebook. Note that data types may show differently when using PySpark vs pure Python.
+
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
+    Parameters
+    ----------
+    lakehouse : str | uuid.UUID, default=None
+        The Fabric lakehouse name or ID.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    lakehouse_workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID used by the lakehouse.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    pandas.DataFrame
+        Shows the tables/columns within a lakehouse and their properties.
+    """
+    from ._get_lakehouse_tables import get_lakehouse_tables
+
+    columns = {
+        "Workspace Name": "string",
+        "Lakehouse Name": "string",
+        "Table Name": "string",
+        "Column Name": "string",
+        "Full Column Name": "string",
+        "Data Type": "string",
+    }
+    df = _create_dataframe(columns=columns)
+
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    (lakehouse_name, lakehouse_id) = resolve_lakehouse_name_and_id(
+        lakehouse=lakehouse, workspace=workspace_id
+    )
+
+    tables = get_lakehouse_tables(
+        lakehouse=lakehouse_id, workspace=workspace_id, extended=False, count_rows=False
+    )
+    tables_filt = tables[tables["Format"] == "delta"]
+
+    def add_column_metadata(table_name, col_name, data_type):
+        new_rows.append(
+            {
+                "Workspace Name": workspace_name,
+                "Lakehouse Name": lakehouse_name,
+                "Table Name": table_name,
+                "Column Name": col_name,
+                "Full Column Name": format_dax_object_name(table_name, col_name),
+                "Data Type": data_type,
+            }
+        )
+
+    new_rows = []
+
+    for _, r in tables_filt.iterrows():
+        table_name = r["Table Name"]
+        path = r["Location"]
+
+        if _pure_python_notebook():
+            from deltalake import DeltaTable
+
+            table_schema = DeltaTable(path).schema()
+
+            for field in table_schema.fields:
+                col_name = field.name
+                match = re.search(r'"(.*?)"', str(field.type))
+                if not match:
+                    raise ValueError(
+                        f"{icons.red_dot} Could not find data type for column {col_name}."
+                    )
+                data_type = match.group(1)
+                add_column_metadata(table_name, col_name, data_type)
+        else:
+            delta_table = _get_delta_table(path=path)
+            table_df = delta_table.toDF()
+
+            for col_name, data_type in table_df.dtypes:
+                add_column_metadata(table_name, col_name, data_type)
+
+    return pd.concat([df, pd.DataFrame(new_rows)], ignore_index=True)