semantic-link-labs 0.9.3__py3-none-any.whl → 0.9.4__py3-none-any.whl

sempy_labs/_delta_analyzer.py

@@ -1,5 +1,8 @@
 import pandas as pd
-import datetime
+import re
+from datetime import datetime
+import os
+from uuid import UUID
 from typing import Dict, Optional
 import pyarrow.dataset as ds
 import pyarrow.parquet as pq
@@ -12,20 +15,43 @@ from sempy_labs._helper_functions import (
     resolve_workspace_name_and_id,
     resolve_lakehouse_name_and_id,
     _read_delta_table,
-    _delta_table_row_count,
+    _mount,
+    _create_spark_session,
 )
+from sempy._utils._log import log
 from sempy_labs.lakehouse._get_lakehouse_tables import get_lakehouse_tables
 from sempy_labs.lakehouse._lakehouse import lakehouse_attached
 import sempy_labs._icons as icons
-from uuid import UUID
+from tqdm.auto import tqdm
+
 
+def get_parquet_file_infos(path):
+
+    import notebookutils
 
+    files = []
+    items = notebookutils.fs.ls(path)
+    for item in items:
+        if item.isDir:
+            # Ignore the _delta_log directory
+            if "_delta_log" not in item.path:
+                files.extend(get_parquet_file_infos(item.path))
+        else:
+            # Filter out non-Parquet files and files with size 0
+            if item.path.endswith(".parquet") and item.size > 0:
+                files.append((item.path, item.size))
+    return files
+
+
+@log
 def delta_analyzer(
     table_name: str,
     approx_distinct_count: bool = True,
     export: bool = False,
     lakehouse: Optional[str | UUID] = None,
     workspace: Optional[str | UUID] = None,
+    column_stats: bool = True,
+    skip_cardinality: bool = True,
 ) -> Dict[str, pd.DataFrame]:
     """
     Analyzes a delta table and shows the results in dictionary containing a set of 5 dataframes. If 'export' is set to True, the results will be saved to delta tables in the lakehouse attached to the notebook.
@@ -55,13 +81,20 @@ def delta_analyzer(
         The Fabric workspace name or ID used by the lakehouse.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
+    column_stats : bool, default=True
+        If True, collects data about column chunks and columns. If False, skips that step and only returns the other 3 dataframes.
+    skip_cardinality : bool, default=True
+        If True, skips the cardinality calculation for each column. If False, calculates the cardinality for each column.
 
     Returns
     -------
     Dict[str, pandas.DataFrame]
         A dictionary of pandas dataframes showing semantic model objects which violated the best practice analyzer rules.
     """
-    import notebookutils
+
+    # Must calculate column stats if calculating cardinality
+    if not skip_cardinality:
+        column_stats = True
 
     # display_toggle = notebookutils.common.configs.pandas_display
 
@@ -70,70 +103,60 @@ def delta_analyzer(
     #    notebookutils.common.configs.pandas_display = False
 
     prefix = "SLL_DeltaAnalyzer_"
-    now = datetime.datetime.now()
+    now = datetime.now()
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace=workspace)
     (lakehouse_name, lakehouse_id) = resolve_lakehouse_name_and_id(
         lakehouse=lakehouse, workspace=workspace
     )
     path = create_abfss_path(lakehouse_id, workspace_id, table_name)
-    lake_path = create_abfss_path(lakehouse_id, workspace_id)
-    mounts = notebookutils.fs.mounts()
-    mount_point = f"/{workspace_name.replace(' ', '')}{lakehouse_name.replace(' ', '')}"
-    if not any(i.get("source") == lake_path for i in mounts):
-        # Mount lakehouse if not mounted
-        notebookutils.fs.mount(lake_path, mount_point)
-        print(
-            f"{icons.green_dot} Mounted the '{lakehouse_name}' lakehouse within the '{workspace_name}' to the notebook."
-        )
-
-    mounts = notebookutils.fs.mounts()
-    local_path = next(
-        i.get("localPath") for i in mounts if i.get("source") == lake_path
-    )
+    local_path = _mount(lakehouse=lakehouse, workspace=workspace)
     table_path = f"{local_path}/Tables/{table_name}"
+    delta_table_path = create_abfss_path(lakehouse_id, workspace_id, table_name)
 
     # Set back to original value
     # notebookutils.common.configs.pandas_display = display_toggle
 
     parquet_file_df_columns = {
-        "ParquetFile": "string",
-        "RowCount": "int",
-        "RowGroups": "int",
+        # "Dataset": "string",
+        "Parquet File": "string",
+        "Row Count": "int",
+        "Row Groups": "int",
+        "Created By": "string",
+        "Total Table Rows": "int",
+        "Total Table Row Groups": "int",
     }
     row_group_df_columns = {
-        "ParquetFile": "string",
-        "RowGroupID": "int",
-        "RowCount": "int",
-        "CompressedSize": "int",
-        "UncompressedSize": "int",
-        "CompressionRatio": "float",
+        # "Dataset": "string",
+        "Parquet File": "string",
+        "Row Group ID": "int",
+        "Row Count": "int",
+        "Compressed Size": "int",
+        "Uncompressed Size": "int",
+        "Compression Ratio": "float",
+        "Total Table Rows": "int",
+        "Ratio Of Total Table Rows": "float",
+        "Total Table Row Groups": "int",
     }
     column_chunk_df_columns = {
-        "ParquetFile": "string",
-        "ColumnID": "int",
-        "ColumnName": "string",
-        "ColumnType": "string",
-        "CompressedSize": "int",
-        "UncompressedSize": "int",
-        "HasDict": "bool",
-        "DictOffset": "int_fillna",
-        "ValueCount": "int",
+        # "Dataset": "string",
+        "Parquet File": "string",
+        "Column ID": "int",
+        "Column Name": "string",
+        "Column Type": "string",
+        "Compressed Size": "int",
+        "Uncompressed Size": "int",
+        "Has Dict": "bool",
+        "Dict Offset": "int_fillna",
+        "Value Count": "int",
         "Encodings": "string",
+        "Statistics": "string",
+        "Primative Type": "string",
     }
 
     parquet_file_df = _create_dataframe(columns=parquet_file_df_columns)
     row_group_df = _create_dataframe(columns=row_group_df_columns)
     column_chunk_df = _create_dataframe(columns=column_chunk_df_columns)
 
-    # delta_table = DeltaTable.forPath(spark, path)
-    # detail_df = spark.sql(f"DESCRIBE DETAIL `{table_name}`").collect()[0]
-
-    # num_files = detail_df.numFiles
-    # size_in_bytes = detail_df.sizeInBytes
-
-    latest_files = _read_delta_table(path).inputFiles()
-    file_paths = [f.split("/")[-1] for f in latest_files]
-    row_count = _delta_table_row_count(table_name)
     row_groups = 0
     max_rows_per_row_group = 0
     min_rows_per_row_group = float("inf")
@@ -141,21 +164,67 @@ def delta_analyzer(
     schema = ds.dataset(table_path).schema.metadata
     is_vorder = any(b"vorder" in key for key in schema.keys())
 
-    for file_name in file_paths:
-        parquet_file = pq.ParquetFile(f"{table_path}/{file_name}")
+    # Get the common details of the Delta table
+    spark = _create_spark_session()
+
+    from delta import DeltaTable
+
+    delta_table = DeltaTable.forPath(spark, delta_table_path)
+    table_df = delta_table.toDF()
+    # total_partition_count = table_df.rdd.getNumPartitions()
+    row_count = table_df.count()
+    table_details = delta_table.detail().collect()[0].asDict()
+    # created_at = table_details.get("createdAt")
+    # last_modified = table_details.get("lastModified")
+    # partition_columns = table_details.get("partitionColumns")
+    # clustering_columns = table_details.get("clusteringColumns")
+    num_latest_files = table_details.get("numFiles", 0)
+    # size_in_bytes = table_details.get("sizeInBytes")
+    # min_reader_version = table_details.get("minReaderVersion")
+    # min_writer_version = table_details.get("minWriterVersion")
+
+    latest_files = _read_delta_table(path).inputFiles()
+    # file_paths = [f.split("/")[-1] for f in latest_files]
+    all_parquet_files = get_parquet_file_infos(delta_table_path)
+    common_file_paths = set(
+        [file_info[0] for file_info in all_parquet_files]
+    ).intersection(set(latest_files))
+    latest_version_files = [
+        file_info
+        for file_info in all_parquet_files
+        if file_info[0] in common_file_paths
+    ]
+
+    for idx, (file_path, file_size) in enumerate(
+        bar := tqdm(latest_version_files), start=1
+    ):
+        file_name = os.path.basename(file_path)
+        bar.set_description(
+            f"Analyzing the '{file_name}' parquet file ({idx}/{num_latest_files})..."
+        )
+
+        relative_path = file_path.split("Tables/")[1]
+        file_system_path = f"{local_path}/Tables/{relative_path}"
+        parquet_file = pq.ParquetFile(file_system_path)
+
         row_groups += parquet_file.num_row_groups
 
         # Generate rowgroup dataframe
         new_data = {
-            "ParquetFile": file_name,
-            "RowCount": parquet_file.metadata.num_rows,
-            "RowGroups": parquet_file.num_row_groups,
+            # "Dataset": "Parquet Files",
+            "Parquet File": file_name,
+            "Row Count": parquet_file.metadata.num_rows,
+            "Row Groups": parquet_file.num_row_groups,
+            "Created By": parquet_file.metadata.created_by,
+            "Total Table Rows": -1,
+            "Total Table Row Groups": -1,
         }
 
         parquet_file_df = pd.concat(
             [parquet_file_df, pd.DataFrame(new_data, index=[0])], ignore_index=True
         )
 
+        # Loop through the row groups
         for i in range(parquet_file.num_row_groups):
             row_group = parquet_file.metadata.row_group(i)
             num_rows = row_group.num_rows
@@ -166,38 +235,50 @@ def delta_analyzer(
             total_compressed_size = 0
             total_uncompressed_size = 0
 
-            for j in range(row_group.num_columns):
-                column_chunk = row_group.column(j)
-                total_compressed_size += column_chunk.total_compressed_size
-                total_uncompressed_size += column_chunk.total_uncompressed_size
-
-                # Generate Column Chunk Dataframe
-                new_data = {
-                    "ParquetFile": file_name,
-                    "ColumnID": j,
-                    "ColumnName": column_chunk.path_in_schema,
-                    "ColumnType": column_chunk.physical_type,
-                    "CompressedSize": column_chunk.total_compressed_size,
-                    "UncompressedSize": column_chunk.total_uncompressed_size,
-                    "HasDict": column_chunk.has_dictionary_page,
-                    "DictOffset": column_chunk.dictionary_page_offset,
-                    "ValueCount": column_chunk.num_values,
-                    "Encodings": str(column_chunk.encodings),
-                }
-
-                column_chunk_df = pd.concat(
-                    [column_chunk_df, pd.DataFrame(new_data, index=[0])],
-                    ignore_index=True,
-                )
+            # Loop through the columns
+            if column_stats:
+                for j in range(row_group.num_columns):
+                    column_chunk = row_group.column(j)
+                    total_compressed_size += column_chunk.total_compressed_size
+                    total_uncompressed_size += column_chunk.total_uncompressed_size
+
+                    # Generate Column Chunk Dataframe
+                    new_data = {
+                        # "Dataset": "Column Chunks",
+                        "Parquet File": file_name,
+                        "Column ID": j,
+                        "Column Name": column_chunk.path_in_schema,
+                        "Column Type": column_chunk.physical_type,
+                        "Compressed Size": column_chunk.total_compressed_size,
+                        "Uncompressed Size": column_chunk.total_uncompressed_size,
+                        "Has Dict": column_chunk.has_dictionary_page,
+                        "Dict Offset": column_chunk.dictionary_page_offset,
+                        "Value Count": column_chunk.num_values,
+                        "Encodings": str(column_chunk.encodings),
+                        "Statistics": column_chunk.statistics,
+                        "PrimativeType": column_chunk.physical_type,
+                    }
+
+                    column_chunk_df = pd.concat(
+                        [column_chunk_df, pd.DataFrame(new_data, index=[0])],
+                        ignore_index=True,
+                    )
 
             # Generate rowgroup dataframe
             new_data = {
-                "ParquetFile": file_name,
-                "RowGroupID": i + 1,
-                "RowCount": num_rows,
-                "CompressedSize": total_compressed_size,
-                "UncompressedSize": total_uncompressed_size,
-                "CompressionRatio": total_compressed_size / total_uncompressed_size,
+                # "Dataset": "Row Groups",
+                "Parquet File": file_name,
+                "Row Group ID": i + 1,
+                "Row Count": num_rows,
+                "Compressed Size": total_compressed_size,
+                "Uncompressed Size": total_uncompressed_size,
+                "Compression Ratio": (
+                    total_compressed_size / total_uncompressed_size
+                    if column_stats
+                    else 0
+                ),
+                "Total Table Rows": -1,
+                "Total Table Row Groups": -1,
             }
 
             if not row_group_df.empty:
@@ -213,68 +294,89 @@ def delta_analyzer(
     summary_df = pd.DataFrame(
         [
             {
-                "RowCount": row_count,
-                "RowGroups": row_groups,
-                "ParquetFiles": len(file_paths),
-                "MaxRowsPerRowGroup": max_rows_per_row_group,
-                "MinRowsPerRowGroup": min_rows_per_row_group,
-                "AvgRowsPerRowGroup": avg_rows_per_row_group,
-                "VOrderEnabled": is_vorder,
+                # "Dataset": "Summary",
+                "Row Count": row_count,
+                "Row Groups": row_groups,
+                "Parquet Files": num_latest_files,
+                "Max Rows Per Row Group": max_rows_per_row_group,
+                "Min Rows Per Row Group": min_rows_per_row_group,
+                "Avg Rows Per Row Group": avg_rows_per_row_group,
+                "VOrder Enabled": is_vorder,
                 # "VOrderLevel": v_order_level,
             }
         ]
     )
 
     # Clean up data types
-    _update_dataframe_datatypes(
-        dataframe=column_chunk_df, column_map=column_chunk_df_columns
-    )
     _update_dataframe_datatypes(dataframe=row_group_df, column_map=row_group_df_columns)
     _update_dataframe_datatypes(
         dataframe=parquet_file_df, column_map=parquet_file_df_columns
     )
 
     # Generate column dataframe
-    column_df = column_chunk_df.groupby(
-        ["ColumnName", "ColumnType"], as_index=False
-    ).agg({"CompressedSize": "sum", "UncompressedSize": "sum"})
-
-    # Add distinct count to column_df
-    for ind, r in column_df.iterrows():
-        col_name = r["ColumnName"]
-        if approx_distinct_count:
-            dc = _get_column_aggregate(
-                table_name=table_name,
-                column_name=col_name,
-                function="approx",
-                lakehouse=lakehouse,
-                workspace=workspace,
-            )
-        else:
-            dc = _get_column_aggregate(
-                table_name=table_name,
-                column_name=col_name,
-                function="distinctcount",
-                lakehouse=lakehouse,
-                workspace=workspace,
-            )
+    if column_stats:
+        _update_dataframe_datatypes(
+            dataframe=column_chunk_df, column_map=column_chunk_df_columns
+        )
+        column_df = column_chunk_df.groupby(
+            ["Column Name", "Column Type"], as_index=False
+        ).agg({"Compressed Size": "sum", "Uncompressed Size": "sum"})
+
+        # Add distinct count to column_df
+        if not skip_cardinality:
+            for ind, r in column_df.iterrows():
+                col_name = r["Column Name"]
+                if approx_distinct_count:
+                    function = "approx"
+                else:
+                    function = "distinctcount"
+                dc = _get_column_aggregate(
+                    table_name=table_name,
+                    column_name=col_name,
+                    function=function,
+                    lakehouse=lakehouse,
+                    workspace=workspace,
+                )
+
+                if "Cardinality" not in column_df.columns:
+                    column_df["Cardinality"] = None
 
-        if "Cardinality" not in column_df.columns:
-            column_df["Cardinality"] = None
+                column_df.at[ind, "Cardinality"] = dc
 
-        column_df.at[ind, "Cardinality"] = dc
+        summary_df["Total Size"] = column_df["Compressed Size"].sum()
 
-    column_df["Cardinality"] = column_df["Cardinality"].astype(int)
-    summary_df["TotalSize"] = column_df["CompressedSize"].sum()
+    parquet_file_df["Total Table Rows"] = parquet_file_df["Row Count"].sum()
+    parquet_file_df["Total Table Row Groups"] = parquet_file_df["Row Groups"].sum()
+
+    row_group_df["Total Table Rows"] = parquet_file_df["Row Count"].sum()
+    row_group_df["Total Table Row Groups"] = parquet_file_df["Row Groups"].sum()
+    total_rows = row_group_df["Row Count"].sum()
+    row_group_df["Ratio Of Total Table Rows"] = (
+        row_group_df["Row Count"] / total_rows * 100.0
+    )
+
+    if column_stats:
+        column_df["Total Table Rows"] = parquet_file_df["Row Count"].sum()
+        column_df["Table Size"] = column_df["Compressed Size"].sum()
+        column_df["Size Percent Of Table"] = (
+            column_df["Compressed Size"] / column_df["Table Size"] * 100.0
+        )
+    if not skip_cardinality and column_stats:
+        column_df["Cardinality"] = column_df["Cardinality"].fillna(0).astype(int)
+        column_df["Cardinality Of Total Rows"] = (
+            column_df["Cardinality"] / column_df["Total Table Rows"] * 100.0
+        )
 
     dataframes = {
         "Summary": summary_df,
         "Parquet Files": parquet_file_df,
         "Row Groups": row_group_df,
-        "Column Chunks": column_chunk_df,
-        "Columns": column_df,
     }
 
+    if column_stats:
+        dataframes["Column Chunks"] = column_chunk_df
+        dataframes["Columns"] = column_df
+
     save_table = f"{prefix}Summary"
 
     if export:
@@ -295,11 +397,11 @@ def delta_analyzer(
     for name, df in dataframes.items():
         name = name.replace(" ", "")
         cols = {
-            "WorkspaceName": workspace_name,
-            "WorkspaceId": workspace_id,
-            "LakehouseName": lakehouse_name,
-            "LakehouseId": lakehouse_id,
-            "TableName": table_name,
+            "Workspace Name": workspace_name,
+            "Workspace Id": workspace_id,
+            "Lakehouse Name": lakehouse_name,
+            "Lakehouse Id": lakehouse_id,
+            "Table Name": table_name,
         }
         for i, (col, param) in enumerate(cols.items()):
             df[col] = param
@@ -309,8 +411,10 @@ def delta_analyzer(
         df["Timestamp"] = pd.to_datetime(df["Timestamp"])
 
         if export:
-            df["RunId"] = runId
-            df["RunId"] = df["RunId"].astype(int)
+            df["Run Id"] = runId
+            df["Run Id"] = df["Run Id"].astype(int)
+
+            df.columns = df.columns.str.replace(" ", "")
             save_as_delta_table(
                 dataframe=df,
                 delta_table_name=f"{prefix}{name}",
@@ -319,3 +423,51 @@ def delta_analyzer(
             )
 
     return dataframes
+
+
+@log
+def get_delta_table_history(
+    table_name: str,
+    lakehouse: Optional[str | UUID] = None,
+    workspace: Optional[str | UUID] = None,
+) -> pd.DataFrame:
+    """
+    Returns the history of a delta table as a pandas dataframe.
+
+    Parameters
+    ----------
+    table_name : str
+        The delta table name.
+    lakehouse : str | uuid.UUID, default=None
+        The Fabric lakehouse name or ID.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID used by the lakehouse.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A dataframe showing the history of the delta table.
+    """
+
+    def camel_to_title(text):
+        return re.sub(r"([a-z])([A-Z])", r"\1 \2", text).title()
+
+    spark = _create_spark_session()
+
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace=workspace)
+    (lakehouse_name, lakehouse_id) = resolve_lakehouse_name_and_id(
+        lakehouse=lakehouse, workspace=workspace
+    )
+    path = create_abfss_path(lakehouse_id, workspace_id, table_name)
+
+    from delta import DeltaTable
+
+    delta_table = DeltaTable.forPath(spark, path)
+    df = delta_table.history().toPandas()
+
+    df.rename(columns=lambda col: camel_to_title(col), inplace=True)
+
+    return df

sempy_labs/_eventhouses.py

@@ -7,12 +7,18 @@ from sempy_labs._helper_functions import (
     _print_success,
     resolve_item_id,
     _create_dataframe,
+    _conv_b64,
+    _decode_b64,
 )
 from uuid import UUID
+import sempy_labs._icons as icons
 
 
 def create_eventhouse(
-    name: str, description: Optional[str] = None, workspace: Optional[str | UUID] = None
+    name: str,
+    definition: Optional[dict],
+    description: Optional[str] = None,
+    workspace: Optional[str | UUID] = None,
 ):
     """
     Creates a Fabric eventhouse.
@@ -23,6 +29,8 @@ def create_eventhouse(
     ----------
     name: str
         Name of the eventhouse.
+    definition : dict
+        The definition (EventhouseProperties.json) of the eventhouse.
     description : str, default=None
         A description of the environment.
     workspace : str | uuid.UUID, default=None
@@ -38,6 +46,20 @@ def create_eventhouse(
     if description:
         payload["description"] = description
 
+    if definition is not None:
+        if not isinstance(definition, dict):
+            raise ValueError(f"{icons.red_dot} The definition must be a dictionary.")
+
+        payload["definition"] = {
+            "parts": [
+                {
+                    "path": "EventhouseProperties.json",
+                    "payload": _conv_b64(definition),
+                    "payloadType": "InlineBase64",
+                }
+            ]
+        }
+
     _base_api(
         request=f"/v1/workspaces/{workspace_id}/eventhouses",
         method="post",
@@ -123,3 +145,50 @@ def delete_eventhouse(name: str, workspace: Optional[str | UUID] = None):
         workspace_name=workspace_name,
         action="deleted",
     )
+
+
+def get_eventhouse_definition(
+    eventhouse: str | UUID,
+    workspace: Optional[str | UUID] = None,
+    return_dataframe: bool = False,
+) -> dict | pd.DataFrame:
+    """
+    Gets the eventhouse definition.
+
+    This is a wrapper function for the following API: `Items - Get Eventhouse Definition <https://learn.microsoft.com/rest/api/fabric/eventhouse/items/get-eventhouse-definition>`_.
+
+    Parameters
+    ----------
+    eventhouse : str
+        Name of the eventhouse.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID in which the eventhouse resides.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    return_dataframe : bool, default=False
+        If True, returns a dataframe. If False, returns a json dictionary.
+
+    Returns
+    -------
+    dict | pandas.DataFrame
+        The eventhouse definition in .json format or as a pandas dataframe.
+    """
+
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    item_id = resolve_item_id(item=eventhouse, type="Eventhouse", workspace=workspace)
+
+    result = _base_api(
+        request=f"/v1/workspaces/{workspace_id}/eventhouses/{item_id}/getDefinition",
+        method="post",
+        status_codes=None,
+        lro_return_json=True,
+    )
+
+    df = pd.json_normalize(result["definition"]["parts"])
+
+    if return_dataframe:
+        return df
+    else:
+        df_filt = df[df["path"] == "EventhouseProperties.json"]
+        payload = df_filt["payload"].iloc[0]
+        return _decode_b64(payload)