semantic-link-labs 0.9.0__py3-none-any.whl → 0.9.2__py3-none-any.whl

sempy_labs/_dax.py

@@ -4,12 +4,16 @@ from sempy_labs._helper_functions import (
     resolve_workspace_name_and_id,
     format_dax_object_name,
     resolve_dataset_name_and_id,
+    _base_api,
+    generate_guid,
 )
 from sempy_labs._model_dependencies import get_model_calc_dependencies
-from typing import Optional, List
+from typing import Optional, List, Tuple
 from sempy._utils._log import log
 from uuid import UUID
 from sempy_labs.directlake._warm_cache import _put_columns_into_memory
+import sempy_labs._icons as icons
+import time
 
 
 @log
@@ -47,15 +51,15 @@ def evaluate_dax_impersonation(
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
     (dataset_name, dataset_id) = resolve_dataset_name_and_id(dataset, workspace_id)
 
-    request_body = {
+    payload = {
         "queries": [{"query": dax_query}],
         "impersonatedUserName": user_name,
     }
 
-    client = fabric.PowerBIRestClient()
-    response = client.post(
-        f"/v1.0/myorg/groups/{workspace_id}/datasets/{dataset_id}/executeQueries",
-        json=request_body,
+    response = _base_api(
+        request=f"/v1.0/myorg/groups/{workspace_id}/datasets/{dataset_id}/executeQueries",
+        method="post",
+        payload=payload,
     )
     data = response.json()["results"][0]["tables"]
     column_names = data[0]["rows"][0].keys()
@@ -100,6 +104,8 @@ def get_dax_query_dependencies(
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
     (dataset_name, dataset_id) = resolve_dataset_name_and_id(dataset, workspace_id)
 
+    fabric.refresh_tom_cache(workspace=workspace)
+
     if isinstance(dax_string, str):
         dax_string = [dax_string]
 
@@ -257,3 +263,219 @@ def get_dax_query_memory_size(
     )
 
     return df["Total Size"].sum()
+
+
+@log
+def _dax_perf_test(
+    dataset: str,
+    dax_queries: dict,
+    clear_cache_before_run: bool = False,
+    refresh_type: Optional[str] = None,
+    rest_time: int = 2,
+    workspace: Optional[str] = None,
+) -> Tuple[pd.DataFrame, dict]:
+    """
+    Runs a performance test on a set of DAX queries.
+
+    Parameters
+    ----------
+    dataset : str
+        Name of the semantic model.
+    dax_queries : dict
+        The dax queries to run in a dictionary format. Here is an example:
+        {
+            "Sales Amount Test", """ """ EVALUATE SUMMARIZECOLUMNS("Sales Amount", [Sales Amount]) """ """,
+            "Order Quantity with Product", """ """ EVALUATE SUMMARIZECOLUMNS('Product'[Color], "Order Qty", [Order Qty]) """ """,
+        }
+    clear_cache_before_run : bool, default=False
+    refresh_type : str, default=None
+    rest_time : int, default=2
+        Rest time (in seconds) between the execution of each DAX query.
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    Tuple[pandas.DataFrame, dict]
+        A pandas dataframe showing the SQL profiler trace results of the DAX queries.
+        A dictionary of the query results in pandas dataframes.
+    """
+    from sempy_labs._refresh_semantic_model import refresh_semantic_model
+    from sempy_labs._clear_cache import clear_cache
+
+    event_schema = {
+        "QueryBegin": [
+            "EventClass",
+            "EventSubclass",
+            "CurrentTime",
+            "NTUserName",
+            "TextData",
+            "StartTime",
+            "ApplicationName",
+        ],
+        "QueryEnd": [
+            "EventClass",
+            "EventSubclass",
+            "CurrentTime",
+            "NTUserName",
+            "TextData",
+            "StartTime",
+            "EndTime",
+            "Duration",
+            "CpuTime",
+            "Success",
+            "ApplicationName",
+        ],
+        "VertiPaqSEQueryBegin": [
+            "EventClass",
+            "EventSubclass",
+            "CurrentTime",
+            "NTUserName",
+            "TextData",
+            "StartTime",
+        ],
+        "VertiPaqSEQueryEnd": [
+            "EventClass",
+            "EventSubclass",
+            "CurrentTime",
+            "NTUserName",
+            "TextData",
+            "StartTime",
+            "EndTime",
+            "Duration",
+            "CpuTime",
+            "Success",
+        ],
+        "VertiPaqSEQueryCacheMatch": [
+            "EventClass",
+            "EventSubclass",
+            "CurrentTime",
+            "NTUserName",
+            "TextData",
+        ],
+    }
+
+    # Add Execution Metrics
+    event_schema["ExecutionMetrics"] = ["EventClass", "ApplicationName", "TextData"]
+    # Add DAX Query Plan
+    # event_schema["DAXQueryPlan"] = ["EventClass", "EventSubclass", "CurrentTime", "StartTime", "EndTime", "Duration", "CpuTime", "ApplicationName", "TextData"]
+
+    query_results = {}
+
+    # Establish trace connection
+    with fabric.create_trace_connection(
+        dataset=dataset, workspace=workspace
+    ) as trace_connection:
+        with trace_connection.create_trace(event_schema) as trace:
+            trace.start()
+            print(f"{icons.in_progress} Starting performance testing...")
+            # Loop through DAX queries
+            for name, dax in dax_queries.items():
+
+                if clear_cache_before_run:
+                    clear_cache(dataset=dataset, workspace=workspace)
+                if refresh_type is not None:
+                    refresh_semantic_model(
+                        dataset=dataset, workspace=workspace, refresh_type=refresh_type
+                    )
+
+                # EVALUATE {1} is used to initate a warm cache
+                fabric.evaluate_dax(
+                    dataset=dataset, workspace=workspace, dax_string="""EVALUATE {1}"""
+                )
+                # Run DAX Query
+                result = fabric.evaluate_dax(
+                    dataset=dataset, workspace=workspace, dax_string=dax
+                )
+
+                # Add results to output
+                query_results[name] = result
+
+                time.sleep(rest_time)
+                print(f"{icons.green_dot} The '{name}' query has completed.")
+
+            df = trace.stop()
+            # Allow time to collect trace results
+            time.sleep(5)
+
+            # Step 1: Filter out unnecessary operations
+            query_names = list(dax_queries.keys())
+            df = df[
+                ~df["Application Name"].isin(["PowerBI", "PowerBIEIM"])
+                & (~df["Text Data"].str.startswith("EVALUATE {1}"))
+            ]
+            query_begin = df["Event Class"] == "QueryBegin"
+            temp_column_name = "QueryName_INT"
+            df = df.copy()
+            df[temp_column_name] = query_begin.cumsum()
+            df[temp_column_name] = (
+                df[temp_column_name]
+                .where(query_begin, None)  # Assign None to non-query begin rows
+                .ffill()  # Forward fill None values
+                .astype("Int64")  # Use pandas nullable integer type for numeric indices
+            )
+
+            df.loc[df[temp_column_name].notna(), "Query Name"] = (
+                df[temp_column_name]
+                .dropna()
+                .astype(int)
+                .map(lambda x: query_names[x - 1])
+            )
+            df = df[df[temp_column_name] != None]
+            df = df.drop(columns=[temp_column_name])
+
+            query_to_guid = {
+                name: generate_guid() for name in df["Query Name"].unique()
+            }
+            df["Query ID"] = df["Query Name"].map(query_to_guid)
+
+    df = df.reset_index(drop=True)
+
+    return df, query_results
+
+
+def _dax_perf_test_bulk(
+    mapping: dict,
+    clear_cache_before_run: bool = False,
+    refresh_type: Optional[str] = None,
+    rest_time: int = 2,
+):
+    """
+    mapping is something like this:
+
+    mapping = {
+        "Workspace1": {
+            "Dataset1": {
+                "Query1": "EVALUATE ...",
+                "Query2": "EVALUATE ...",
+            },
+            "Dataset2": {
+                "Query3": "EVALUATE ...",
+                "Query4": "EVALUATE ...",
+            }
+        },
+        "Workspace2": {
+            "Dataset3": {
+                "Query5": "EVALUATE ...",
+                "Query6": "EVALUATE ...",
+            },
+            "Dataset4": {
+                "Query7": "EVALUATE ...",
+                "Query8": "EVALUATE ...",
+            }
+        }
+    }
+    """
+
+    for workspace, datasets in mapping.items():
+        for dataset, queries in datasets.items():
+            _dax_perf_test(
+                dataset=dataset,
+                dax_queries=queries,
+                clear_cache_before_run=clear_cache_before_run,
+                refresh_type=refresh_type,
+                rest_time=rest_time,
+                workspace=workspace,
+            )

sempy_labs/_delta_analyzer.py

@@ -0,0 +1,303 @@
+import pandas as pd
+import datetime
+from typing import Dict
+import pyarrow.dataset as ds
+import pyarrow.parquet as pq
+from pyspark.sql import SparkSession
+from sempy_labs._helper_functions import (
+    create_abfss_path,
+    save_as_delta_table,
+    _get_column_aggregate,
+    _create_dataframe,
+    _update_dataframe_datatypes,
+    resolve_workspace_name_and_id,
+    resolve_lakehouse_name_and_id,
+)
+from sempy_labs.lakehouse._get_lakehouse_tables import get_lakehouse_tables
+from sempy_labs.lakehouse._lakehouse import lakehouse_attached
+import sempy_labs._icons as icons
+
+
+def delta_analyzer(
+    table_name: str,
+    approx_distinct_count: bool = True,
+    export: bool = False,
+) -> Dict[str, pd.DataFrame]:
+    """
+    Analyzes a delta table and shows the results in dictionary containing a set of 5 dataframes. The table being analyzed must be in the lakehouse attached to the notebook.
+
+    The 5 dataframes returned by this function are:
+
+    * Summary
+    * Parquet Files
+    * Row Groups
+    * Column Chunks
+    * Columns
+
+    Read more about Delta Analyzer `here <https://github.com/microsoft/Analysis-Services/tree/master/DeltaAnalyzer>`_.
+
+    Parameters
+    ----------
+    table_name : str
+        The delta table name.
+    approx_distinct_count: bool, default=True
+        If True, uses approx_count_distinct to calculate the cardinality of each column. If False, uses COUNT(DISTINCT) instead.
+    export : bool, default=False
+        If True, exports the resulting dataframes to delta tables in the lakehouse attached to the notebook.
+
+    Returns
+    -------
+    Dict[str, pandas.DataFrame]
+        A dictionary of pandas dataframes showing semantic model objects which violated the best practice analyzer rules.
+    """
+
+    if not lakehouse_attached():
+        raise ValueError(
+            f"{icons.red_dot} No lakehouse is attached to this notebook. Please attach a lakehouse to the notebook before running the Delta Analyzer."
+        )
+
+    prefix = "SLL_DeltaAnalyzer_"
+    now = datetime.datetime.now()
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace=None)
+    (lakehouse_name, lakehouse_id) = resolve_lakehouse_name_and_id(
+        lakehouse=None, workspace=None
+    )
+    path = create_abfss_path(lakehouse_id, workspace_id, table_name)
+    table_path = f"/lakehouse/default/Tables/{table_name}"
+
+    parquet_file_df_columns = {
+        "ParquetFile": "string",
+        "RowCount": "int",
+        "RowGroups": "int",
+    }
+    row_group_df_columns = {
+        "ParquetFile": "string",
+        "RowGroupID": "int",
+        "RowCount": "int",
+        "CompressedSize": "int",
+        "UncompressedSize": "int",
+        "CompressionRatio": "float",
+    }
+    column_chunk_df_columns = {
+        "ParquetFile": "string",
+        "ColumnID": "int",
+        "ColumnName": "string",
+        "ColumnType": "string",
+        "CompressedSize": "int",
+        "UncompressedSize": "int",
+        "HasDict": "bool",
+        "DictOffset": "int_fillna",
+        "ValueCount": "int",
+        "Encodings": "string",
+    }
+
+    parquet_file_df = _create_dataframe(columns=parquet_file_df_columns)
+    row_group_df = _create_dataframe(columns=row_group_df_columns)
+    column_chunk_df = _create_dataframe(columns=column_chunk_df_columns)
+
+    spark = SparkSession.builder.getOrCreate()
+    # delta_table = DeltaTable.forPath(spark, path)
+    # detail_df = spark.sql(f"DESCRIBE DETAIL `{table_name}`").collect()[0]
+
+    # num_files = detail_df.numFiles
+    # size_in_bytes = detail_df.sizeInBytes
+
+    latest_files = spark.read.format("delta").load(path).inputFiles()
+    file_paths = [f.split("/")[-1] for f in latest_files]
+    row_count = spark.table(table_name).count()
+    row_groups = 0
+    max_rows_per_row_group = 0
+    min_rows_per_row_group = float("inf")
+    # dt = DeltaTable.forPath(spark, path)
+    # schema = dt.toDF().schema
+    # is_vorder = False
+    # if (
+    #    dt.detail()
+    #    .collect()[0]
+    #    .asDict()
+    #    .get("properties")
+    #    .get("delta.parquet.vorder.enabled")
+    #    == "true"
+    # ):
+    #    is_vorder = True
+
+    schema = ds.dataset(table_path).schema.metadata
+    is_vorder = any(b"vorder" in key for key in schema.keys())
+    # v_order_level = (
+    #    int(schema.get(b"com.microsoft.parquet.vorder.level").decode("utf-8"))
+    #    if is_vorder
+    #    else None
+    # )
+
+    for file_name in file_paths:
+        parquet_file = pq.ParquetFile(f"{table_path}/{file_name}")
+        row_groups += parquet_file.num_row_groups
+
+        # Generate rowgroup dataframe
+        new_data = {
+            "ParquetFile": file_name,
+            "RowCount": parquet_file.metadata.num_rows,
+            "RowGroups": parquet_file.num_row_groups,
+        }
+
+        parquet_file_df = pd.concat(
+            [parquet_file_df, pd.DataFrame(new_data, index=[0])], ignore_index=True
+        )
+
+        for i in range(parquet_file.num_row_groups):
+            row_group = parquet_file.metadata.row_group(i)
+            num_rows = row_group.num_rows
+
+            max_rows_per_row_group = max(max_rows_per_row_group, num_rows)
+            min_rows_per_row_group = min(min_rows_per_row_group, num_rows)
+
+            total_compressed_size = 0
+            total_uncompressed_size = 0
+
+            for j in range(row_group.num_columns):
+                column_chunk = row_group.column(j)
+                total_compressed_size += column_chunk.total_compressed_size
+                total_uncompressed_size += column_chunk.total_uncompressed_size
+
+                # Generate Column Chunk Dataframe
+                new_data = {
+                    "ParquetFile": file_name,
+                    "ColumnID": j,
+                    "ColumnName": column_chunk.path_in_schema,
+                    "ColumnType": column_chunk.physical_type,
+                    "CompressedSize": column_chunk.total_compressed_size,
+                    "UncompressedSize": column_chunk.total_uncompressed_size,
+                    "HasDict": column_chunk.has_dictionary_page,
+                    "DictOffset": column_chunk.dictionary_page_offset,
+                    "ValueCount": column_chunk.num_values,
+                    "Encodings": str(column_chunk.encodings),
+                }
+
+                column_chunk_df = pd.concat(
+                    [column_chunk_df, pd.DataFrame(new_data, index=[0])],
+                    ignore_index=True,
+                )
+
+            # Generate rowgroup dataframe
+            new_data = {
+                "ParquetFile": file_name,
+                "RowGroupID": i + 1,
+                "RowCount": num_rows,
+                "CompressedSize": total_compressed_size,
+                "UncompressedSize": total_uncompressed_size,
+                "CompressionRatio": total_compressed_size / total_uncompressed_size,
+            }
+
+            if not row_group_df.empty:
+                row_group_df = pd.concat(
+                    [row_group_df, pd.DataFrame(new_data, index=[0])], ignore_index=True
+                )
+            else:
+                row_group_df = pd.DataFrame(new_data, index=[0])
+
+    avg_rows_per_row_group = row_count / row_groups
+
+    # Generate summary dataframe
+    summary_df = pd.DataFrame(
+        [
+            {
+                "RowCount": row_count,
+                "RowGroups": row_groups,
+                "ParquetFiles": len(file_paths),
+                "MaxRowsPerRowGroup": max_rows_per_row_group,
+                "MinRowsPerRowGroup": min_rows_per_row_group,
+                "AvgRowsPerRowGroup": avg_rows_per_row_group,
+                "VOrderEnabled": is_vorder,
+                # "VOrderLevel": v_order_level,
+            }
+        ]
+    )
+
+    # Clean up data types
+    _update_dataframe_datatypes(
+        dataframe=column_chunk_df, column_map=column_chunk_df_columns
+    )
+    _update_dataframe_datatypes(dataframe=row_group_df, column_map=row_group_df_columns)
+    _update_dataframe_datatypes(
+        dataframe=parquet_file_df, column_map=parquet_file_df_columns
+    )
+
+    # Generate column dataframe
+    column_df = column_chunk_df.groupby(
+        ["ColumnName", "ColumnType"], as_index=False
+    ).agg({"CompressedSize": "sum", "UncompressedSize": "sum"})
+
+    # Add distinct count to column_df
+    for ind, r in column_df.iterrows():
+        col_name = r["ColumnName"]
+        if approx_distinct_count:
+            dc = _get_column_aggregate(
+                table_name=table_name,
+                column_name=col_name,
+                function="approx",
+                lakehouse=lakehouse_name,
+            )
+        else:
+            dc = _get_column_aggregate(
+                table_name=table_name,
+                column_name=col_name,
+                function="distinctcount",
+                lakehouse=lakehouse_name,
+            )
+
+        if "Cardinality" not in column_df.columns:
+            column_df["Cardinality"] = None
+
+        column_df.at[ind, "Cardinality"] = dc
+
+    column_df["Cardinality"] = column_df["Cardinality"].astype(int)
+    summary_df["TotalSize"] = column_df["CompressedSize"].sum()
+
+    dataframes = {
+        "Summary": summary_df,
+        "Parquet Files": parquet_file_df,
+        "Row Groups": row_group_df,
+        "Column Chunks": column_chunk_df,
+        "Columns": column_df,
+    }
+
+    save_table = f"{prefix}Summary"
+
+    if export:
+        dfL = get_lakehouse_tables()
+        dfL_filt = dfL[dfL["Table Name"] == save_table]
+        if dfL_filt.empty:
+            runId = 1
+        else:
+            max_run_id = _get_column_aggregate(
+                lakehouse=lakehouse_name, table_name=save_table
+            )
+            runId = max_run_id + 1
+
+    for name, df in dataframes.items():
+        name = name.replace(" ", "")
+        cols = {
+            "WorkspaceName": workspace_name,
+            "WorkspaceId": workspace_id,
+            "LakehouseName": lakehouse_name,
+            "LakehouseId": lakehouse_id,
+            "TableName": table_name,
+        }
+        for i, (col, param) in enumerate(cols.items()):
+            df[col] = param
+            df.insert(i, col, df.pop(col))
+
+        df["Timestamp"] = now
+        df["Timestamp"] = pd.to_datetime(df["Timestamp"])
+
+        if export:
+            df["RunId"] = runId
+            df["RunId"] = df["RunId"].astype(int)
+            save_as_delta_table(
+                dataframe=df,
+                delta_table_name=f"{prefix}{name}",
+                write_mode="append",
+                merge_schema=True,
+            )
+
+    return dataframes