semantic-link-labs 0.12.8__py3-none-any.whl

sempy_labs/lakehouse/_get_lakehouse_tables.py

@@ -0,0 +1,274 @@
+import os
+import pandas as pd
+import pyarrow.parquet as pq
+from datetime import datetime
+from sempy_labs._helper_functions import (
+    _get_column_aggregate,
+    resolve_lakehouse_name_and_id,
+    save_as_delta_table,
+    resolve_workspace_id,
+    _read_delta_table,
+    _get_delta_table,
+    _mount,
+    create_abfss_path,
+    _pure_python_notebook,
+)
+from sempy_labs.directlake._guardrails import (
+    get_sku_size,
+    get_directlake_guardrails_for_sku,
+)
+from sempy_labs.lakehouse._lakehouse import lakehouse_attached
+from typing import Optional
+import sempy_labs._icons as icons
+from sempy._utils._log import log
+from uuid import UUID
+from sempy_labs.lakehouse._schemas import list_tables
+
+
+@log
+def get_lakehouse_tables(
+    lakehouse: Optional[str | UUID] = None,
+    workspace: Optional[str | UUID] = None,
+    extended: bool = False,
+    count_rows: bool = False,
+    export: bool = False,
+    exclude_shortcuts: bool = False,
+) -> pd.DataFrame:
+    """
+    Shows the tables of a lakehouse and their respective properties. Option to include additional properties relevant to Direct Lake guardrails.
+
+    This function can be executed in either a PySpark or pure Python notebook.
+
+    This is a wrapper function for the following API: `Tables - List Tables <https://learn.microsoft.com/rest/api/fabric/lakehouse/tables/list-tables>`_ plus extended capabilities.
+    However, the above mentioned API does not support Lakehouse schemas (Preview) until it is in GA (General Availability). This version also supports schema
+    enabled Lakehouses.
+
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
+    Parameters
+    ----------
+    lakehouse : str | uuid.UUID, default=None
+        The Fabric lakehouse name or ID.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID used by the lakehouse.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    extended : bool, default=False
+        Obtains additional columns relevant to the size of each table.
+    count_rows : bool, default=False
+        Obtains a row count for each lakehouse table.
+    export : bool, default=False
+        Exports the resulting dataframe to a delta table in the lakehouse.
+    exclude_shortcuts : bool, default=False
+        If True, excludes shortcuts.
+
+    Returns
+    -------
+    pandas.DataFrame
+        Shows the tables/columns within a lakehouse and their properties.
+    """
+
+    workspace_id = resolve_workspace_id(workspace)
+    (lakehouse_name, lakehouse_id) = resolve_lakehouse_name_and_id(
+        lakehouse=lakehouse, workspace=workspace_id
+    )
+
+    df = list_tables(lakehouse=lakehouse, workspace=workspace)
+
+    local_path = _mount(lakehouse=lakehouse_id, workspace=workspace_id)
+
+    if extended:
+        sku_value = get_sku_size(workspace_id)
+        guardrail = get_directlake_guardrails_for_sku(sku_value)
+        # Avoid mounting the lakehouse if is already mounted
+        if not local_path:
+            local_path = _mount(lakehouse=lakehouse_id, workspace=workspace_id)
+
+        df["Files"], df["Row Groups"], df["Table Size"] = None, None, None
+        if count_rows:
+            df["Row Count"] = None
+
+        for i, r in df.iterrows():
+            schema_name = r["Schema Name"]
+            table_name = r["Table Name"]
+            if r["Type"] == "Managed" and r["Format"] == "delta":
+                delta_table_path = (
+                    create_abfss_path(
+                        lakehouse_id, workspace_id, table_name, schema_name
+                    )
+                    .replace("//", "/")  # When schema_name = ""
+                    .replace("abfss:/", "abfss://")  # Put back the // after abfss:
+                )
+
+                if _pure_python_notebook():
+                    from deltalake import DeltaTable
+
+                    delta_table = DeltaTable(delta_table_path)
+                    latest_files = [
+                        file["path"]
+                        for file in delta_table.get_add_actions().to_pylist()
+                    ]
+                    size_in_bytes = 0
+                    for f in latest_files:
+                        local_file_path = os.path.join(
+                            local_path, "Tables", schema_name, table_name, f
+                        )
+
+                        if os.path.exists(local_file_path):
+                            size_in_bytes += os.path.getsize(local_file_path)
+                    num_latest_files = len(latest_files)
+                else:
+                    delta_table = _get_delta_table(delta_table_path)
+
+                    latest_files = _read_delta_table(delta_table_path).inputFiles()
+                    table_df = delta_table.toDF()
+                    table_details = delta_table.detail().collect()[0].asDict()
+                    size_in_bytes = table_details.get("sizeInBytes", 0)
+                    num_latest_files = table_details.get("numFiles", 0)
+
+                table_path = os.path.join(local_path, "Tables", schema_name, table_name)
+
+                file_paths = []
+                for file in latest_files:
+                    if _pure_python_notebook():
+                        file_paths.append(file)
+                    else:
+                        # Append the <Partition folder>/<filename> or <filename>
+                        find_table = file.find(table_name)
+                        len_file = len(file)
+                        len_table = len(table_name)
+                        last_chars = len_file - (find_table + len_table + 1)
+                        file_paths.append(file[-last_chars:])
+
+                num_rowgroups = 0
+                for filename in file_paths:
+                    parquet_file_path = f"{table_path}/{filename}"
+                    if os.path.exists(parquet_file_path):
+                        parquet_file = pq.ParquetFile(parquet_file_path)
+                        num_rowgroups += parquet_file.num_row_groups
+
+                df.at[i, "Files"] = num_latest_files
+                df.at[i, "Row Groups"] = num_rowgroups
+                df.at[i, "Table Size"] = size_in_bytes
+
+            if count_rows:
+                if _pure_python_notebook():
+                    row_count = delta_table.to_pyarrow_table().num_rows
+                else:
+                    row_count = table_df.count()
+                df.at[i, "Row Count"] = row_count
+
+            # Set "Schema Name" = "dbo" when it is ""
+            df.loc[df["Schema Name"] == "", "Schema Name"] = "dbo"
+
+    if extended:
+        intColumns = ["Files", "Row Groups", "Table Size"]
+        df[intColumns] = df[intColumns].astype(int)
+
+        col_name = guardrail.columns[0]
+        df["SKU"] = guardrail[col_name].iloc[0]
+        df["Parquet File Guardrail"] = guardrail["Parquet files per table"].iloc[0]
+        df["Row Group Guardrail"] = guardrail["Row groups per table"].iloc[0]
+        df["Row Count Guardrail"] = (
+            guardrail["Rows per table (millions)"].iloc[0] * 1000000
+        )
+
+        df["Parquet File Guardrail Hit"] = df["Files"] > df["Parquet File Guardrail"]
+        df["Row Group Guardrail Hit"] = df["Row Groups"] > df["Row Group Guardrail"]
+    if count_rows:
+        df["Row Count"] = df["Row Count"].astype(int)
+        df["Row Count Guardrail Hit"] = df["Row Count"] > df["Row Count Guardrail"]
+
+    if exclude_shortcuts:
+        from sempy_labs.lakehouse._shortcuts import list_shortcuts
+
+        # Exclude shortcuts
+        shortcuts = (
+            list_shortcuts(lakehouse=lakehouse, workspace=workspace)
+            .query("`Shortcut Path`.str.startswith('/Tables')", engine="python")
+            .assign(
+                FullPath=lambda df: df["Shortcut Path"].str.rstrip("/")
+                + "/"
+                + df["Shortcut Name"]
+            )["FullPath"]
+            .tolist()
+        )
+
+        df["FullPath"] = df.apply(
+            lambda x: (
+                f"/Tables/{x['Table Name']}"
+                if pd.isna(x["Schema Name"]) or x["Schema Name"] == ""
+                else f"/Tables/{x['Schema Name']}/{x['Table Name']}"
+            ),
+            axis=1,
+        )
+
+        df = df[~df["FullPath"].isin(shortcuts)].reset_index(drop=True)
+
+    if export:
+        if not lakehouse_attached():
+            raise ValueError(
+                f"{icons.red_dot} In order to save the dataframe, a lakehouse must be attached to the notebook. Please attach a lakehouse to this notebook."
+            )
+
+        lake_table_name = "lakehouse_table_details"
+        df_filt = df[df["Table Name"] == lake_table_name]
+
+        if df_filt.empty:
+            run_id = 1
+        else:
+            max_run_id = _get_column_aggregate(table_name=lake_table_name)
+            run_id = max_run_id + 1
+
+        export_df = df.copy()
+
+        cols = [
+            "Files",
+            "Row Groups",
+            "Row Count",
+            "Table Size",
+            "SKU",
+            "Parquet File Guardrail",
+            "Row Group Guardrail",
+            "Row Count Guardrail",
+            "Parquet File Guardrail Hit",
+            "Row Group Guardrail Hit",
+            "Row Count Guardrail Hit",
+        ]
+
+        for c in cols:
+            if c not in export_df:
+                if c in [
+                    "Files",
+                    "Row Groups",
+                    "Row Count",
+                    "Table Size",
+                    "Parquet File Guardrail",
+                    "Row Group Guardrail",
+                    "Row Count Guardrail",
+                ]:
+                    export_df[c] = 0
+                    export_df[c] = export_df[c].astype(int)
+                elif c in ["SKU"]:
+                    export_df[c] = None
+                    export_df[c] = export_df[c].astype(str)
+                elif c in [
+                    "Parquet File Guardrail Hit",
+                    "Row Group Guardrail Hit",
+                    "Row Count Guardrail Hit",
+                ]:
+                    export_df[c] = False
+                    export_df[c] = export_df[c].astype(bool)
+
+        print(
+            f"{icons.in_progress} Saving Lakehouse table properties to the '{lake_table_name}' table in the lakehouse...\n"
+        )
+        export_df["Timestamp"] = datetime.now()
+        export_df["RunId"] = run_id
+
+        save_as_delta_table(
+            dataframe=export_df, delta_table_name=lake_table_name, write_mode="append"
+        )
+
+    return df

sempy_labs/lakehouse/_helper.py

@@ -0,0 +1,250 @@
+from uuid import UUID
+from typing import Optional, Literal
+import pyarrow.dataset as ds
+from sempy_labs._helper_functions import (
+    _mount,
+    delete_item,
+    _base_api,
+    resolve_workspace_name_and_id,
+    resolve_lakehouse_name_and_id,
+)
+from sempy._utils._log import log
+import sempy_labs._icons as icons
+import os
+import json
+
+
+@log
+def is_v_ordered(
+    table_name: str,
+    lakehouse: Optional[str | UUID] = None,
+    workspace: Optional[str | UUID] = None,
+    schema: Optional[str] = None,
+) -> bool:
+    """
+    Checks if a delta table is v-ordered.
+
+    Parameters
+    ----------
+    table_name : str
+        The name of the table to check.
+    lakehouse : str | uuid.UUID, default=None
+        The Fabric lakehouse name or ID.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID used by the lakehouse.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    schema : str, optional
+        The schema of the table to check. If not provided, the default schema is used.
+
+    Returns
+    -------
+    bool
+        True if the table is v-ordered, False otherwise.
+    """
+
+    local_path = _mount(lakehouse=lakehouse, workspace=workspace)
+    table_path = (
+        f"{local_path}/Tables/{schema}/{table_name}"
+        if schema
+        else f"{local_path}/Tables/{table_name}"
+    )
+    ds_schema = ds.dataset(table_path).schema.metadata
+
+    if ds_schema:
+        return any(b"vorder" in key for key in ds_schema.keys())
+
+    delta_log_path = os.path.join(table_path, "_delta_log")
+
+    def read_vorder_tag(delta_log_path):
+        json_files = sorted(
+            [f for f in os.listdir(delta_log_path) if f.endswith(".json")], reverse=True
+        )
+
+        if not json_files:
+            return False
+
+        latest_file = os.path.join(delta_log_path, json_files[0])
+
+        with open(latest_file, "r") as f:
+            all_data = [
+                json.loads(line) for line in f if line.strip()
+            ]  # one dict per line
+            for data in all_data:
+                if "metaData" in data:
+                    return (
+                        data.get("metaData", {})
+                        .get("configuration", {})
+                        .get("delta.parquet.vorder.enabled", "false")
+                        == "true"
+                    )
+
+            # If no metaData, fall back to commitInfo
+            for data in all_data:
+                if "commitInfo" in data:
+                    tags = data["commitInfo"].get("tags", {})
+                    return tags.get("VORDER", "false").lower() == "true"
+
+        return False  # Default if not found
+
+    return read_vorder_tag(delta_log_path)
+
+
+@log
+def delete_lakehouse(
+    lakehouse: str | UUID, workspace: Optional[str | UUID] = None
+) -> None:
+    """
+    Deletes a lakehouse.
+
+    This is a wrapper function for the following API: `Items - Delete Lakehouse <https://learn.microsoft.com/rest/api/fabric/lakehouse/items/delete-lakehouse>`_.
+
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
+    Parameters
+    ----------
+    lakehouse : str | uuid.UUID
+        The name or ID of the lakehouse to delete.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID used by the lakehouse.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    delete_item(item=lakehouse, item_type="lakehouse", workspace=workspace)
+
+
+@log
+def update_lakehouse(
+    name: Optional[str] = None,
+    description: Optional[str] = None,
+    lakehouse: Optional[str | UUID] = None,
+    workspace: Optional[str | UUID] = None,
+):
+    """
+    Updates a lakehouse.
+
+    This is a wrapper function for the following API: `Items - Update Lakehouse <https://learn.microsoft.com/rest/api/fabric/lakehouse/items/update-lakehouse>`_.
+
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
+    Parameters
+    ----------
+    name: str, default=None
+        The new name of the lakehouse.
+        Defaults to None which does not update the name.
+    description: str, default=None
+        The new description of the lakehouse.
+        Defaults to None which does not update the description.
+    lakehouse : str | uuid.UUID, default=None
+        The name or ID of the lakehouse to update.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID used by the lakehouse.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    if not name and not description:
+        raise ValueError(
+            f"{icons.red_dot} Either name or description must be provided."
+        )
+
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    (lakehouse_name, lakehouse_id) = resolve_lakehouse_name_and_id(
+        lakehouse, workspace_id
+    )
+
+    payload = {}
+    if name:
+        payload["displayName"] = name
+    if description:
+        payload["description"] = description
+
+    _base_api(
+        request=f"/v1/workspaces/{workspace_id}/lakehouses/{lakehouse_id}",
+        method="patch",
+        client="fabric_sp",
+        payload=payload,
+    )
+
+    print(
+        f"{icons.green_dot} The '{lakehouse_name}' lakehouse within the '{workspace_name}' workspace has been updated accordingly."
+    )
+
+
+@log
+def load_table(
+    table_name: str,
+    file_path: str,
+    mode: Literal["Overwrite", "Append"],
+    lakehouse: Optional[str | UUID] = None,
+    workspace: Optional[str | UUID] = None,
+):
+    """
+    Loads a table into a lakehouse. Currently only files are supported, not folders.
+
+    This is a wrapper function for the following API: `Tables - Load Table <https://learn.microsoft.com/rest/api/fabric/lakehouse/tables/load-table>`_.
+
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
+    Parameters
+    ----------
+    table_name : str
+        The name of the table to load.
+    file_path : str
+        The path to the data to load.
+    mode : Literal["Overwrite", "Append"]
+        The mode to use when loading the data.
+        "Overwrite" will overwrite the existing data.
+        "Append" will append the data to the existing data.
+    lakehouse : str | uuid.UUID, default=None
+        The name or ID of the lakehouse to load the table into.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID used by the lakehouse.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    (lakehouse_name, lakehouse_id) = resolve_lakehouse_name_and_id(
+        lakehouse, workspace_id
+    )
+
+    file_extension = os.path.splitext(file_path)[1]
+
+    payload = {
+        "relativePath": file_path,
+        "pathType": "File",
+        "mode": mode,
+        "formatOptions": {},
+    }
+
+    if file_extension == ".csv":
+        payload["formatOptions"] = {"format": "Csv", "header": True, "delimiter": ","}
+    elif file_extension == ".parquet":
+        payload["formatOptions"] = {
+            "format": "Parquet",
+            "header": True,
+        }
+    # Solve for loading folders
+    # elif file_extension == '':
+    #    payload['pathType'] = "Folder"
+    #    payload["recursive"] = recursive
+    #    payload['formatOptions']
+    else:
+        raise NotImplementedError()
+
+    _base_api(
+        request=f"/v1/workspaces/{workspace_id}/lakehouses/{lakehouse_id}/tables/{table_name}/load",
+        client="fabric_sp",
+        method="post",
+        status_codes=202,
+        lro_return_status_code=True,
+    )
+
+    print(
+        f"{icons.green_dot} The '{table_name}' table has been loaded into the '{lakehouse_name}' lakehouse within the '{workspace_name}' workspace."
+    )