semantic-link-labs 0.12.8__py3-none-any.whl

sempy_labs/_delta_analyzer_history.py

@@ -0,0 +1,301 @@
+import pandas as pd
+from typing import Optional
+import pyarrow.parquet as pq
+from sempy_labs._helper_functions import (
+    create_abfss_path,
+    resolve_workspace_id,
+    resolve_lakehouse_id,
+    _mount,
+)
+from sempy._utils._log import log
+from tqdm.auto import tqdm
+from uuid import UUID
+from datetime import datetime
+
+
+@log
+def delta_analyzer_history(
+    table_name: str,
+    schema: Optional[str] = None,
+    lakehouse: Optional[str | UUID] = None,
+    workspace: Optional[str | UUID] = None,
+) -> pd.DataFrame:
+    """
+    Analyzes the transaction log for a specified delta table and shows the results in dataframe.  One row per data modification operation.
+
+    Keeps track on the number of Parquet files, rowgroups, file size and #rows impacted by each change.
+
+    Incremental Framing effect: 100% = highly effective, 0% = no benefit at all
+
+    Parameters
+    ----------
+    table_name : str
+        The delta table name.
+    schema : str, default=None
+        The schema name of the delta table.
+    lakehouse : str | uuid.UUID, default=None
+        The Fabric lakehouse name or ID.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID used by the lakehouse.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    pandas.DataFrame
+        Displays a gantt visual showing a timeline for individual parquet files.
+    """
+
+    import notebookutils
+    from IPython.display import display, HTML
+
+    workspace_id = resolve_workspace_id(workspace=workspace)
+    lakehouse_id = resolve_lakehouse_id(lakehouse=lakehouse, workspace=workspace)
+
+    table_path = create_abfss_path(lakehouse_id, workspace_id, table_name, schema)
+    local_path = _mount(lakehouse=lakehouse, workspace=workspace)
+    if schema:  # use schema if specified
+        table_path_local = f"{local_path}/Tables/{schema}/{table_name}"
+    else:
+        table_path_local = f"{local_path}/Tables/{table_name}"
+    delta_table_path = f"{table_path}/_delta_log"
+
+    files = notebookutils.fs.ls(delta_table_path)
+    json_files = [file.name for file in files if file.name.endswith(".json")]
+
+    element_version = total_size = total_rows = total_files = total_rowgroups = 0
+    changes_array = []
+    parquet_files = []
+    my_date_time_format = "%Y-%m-%d %H:%M:%S.%f"
+    now_to_epoch = datetime.now().strftime(my_date_time_format)
+    num_latest_files = len(json_files)
+
+    for idx, file in enumerate(bar := tqdm(json_files), start=1):
+        bar.set_description(
+            f"Analyzing the '{file}' parquet file ({idx}/{num_latest_files})..."
+        )
+
+        change_timestamp = datetime.strptime(
+            "2001-01-01 12:00:00.000", my_date_time_format
+        )
+        df = pd.read_json(f"{delta_table_path}/{file}", lines=True)
+
+        rows_added = size_added = rows_deleted = size_deleted = files_added = (
+            files_removed
+        ) = row_groups_added = row_groups_removed = 0
+        total_files_before_change = total_files
+        total_row_groups_before_change = total_rowgroups
+        operation = predicate = tags = ""
+
+        for _, row in df.iterrows():
+            add_row = row.get("add")
+            remove_row = row.get("remove")
+            commit_row = row.get("commitInfo")
+
+            if isinstance(add_row, dict):
+                file_name = add_row["path"]
+                fs_filename = f"{table_path}/{file_name}"
+                size_added += add_row["size"]
+                files_added += 1
+                filerows_added = 0
+
+                if notebookutils.fs.exists(fs_filename):
+                    parquet_file = pq.ParquetFile(table_path_local + f"/{file_name}")
+                    for i in range(parquet_file.num_row_groups):
+                        row_group = parquet_file.metadata.row_group(i)
+                        num_rows = row_group.num_rows
+                        filerows_added += num_rows
+                        rows_added += num_rows
+
+                    row_groups_added += parquet_file.num_row_groups
+
+                    start = str(
+                        datetime.fromtimestamp(add_row["modificationTime"] / 1000.0)
+                    )
+                    parquet_files.append(
+                        {
+                            "file": file_name,
+                            "start": start,
+                            "end": now_to_epoch,
+                            "rows": filerows_added,
+                            "isCurrent": 1,
+                        }
+                    )
+
+            if isinstance(remove_row, dict):
+                file_name = remove_row["path"]
+                fs_filename = f"{table_path}/{file_name}"
+
+                if notebookutils.fs.exists(fs_filename):
+                    parquet_file = pq.ParquetFile(table_path_local + f"/{file_name}")
+                    for i in range(parquet_file.num_row_groups):
+                        row_group = parquet_file.metadata.row_group(i)
+                        num_rows = row_group.num_rows
+                        rows_deleted += num_rows
+
+                    files_removed += 1
+                    size_deleted += remove_row.get("size", 0)
+                    row_groups_removed += parquet_file.num_row_groups
+
+                    result = next(
+                        (row for row in parquet_files if row["file"] == file_name), None
+                    )
+                    if result:
+                        result.update(
+                            {
+                                "isCurrent": 0,
+                                "end": str(
+                                    datetime.fromtimestamp(
+                                        remove_row["deletionTimestamp"] / 1000.0
+                                    )
+                                ),
+                            }
+                        )
+
+            if isinstance(commit_row, dict):
+                operation = commit_row.get("operation")
+                tags = commit_row.get("tags")
+                predicate = commit_row.get("operationParameters", {}).get("predicate")
+
+                if operation == "VACUUM START":
+                    operation_metrics = commit_row.get("operationMetrics", {})
+                    total_files -= int(operation_metrics.get("numFilesToDelete", 0))
+                    total_size -= int(operation_metrics.get("sizeOfDataToDelete", 0))
+
+                change_timestamp = datetime.fromtimestamp(
+                    commit_row["timestamp"] / 1000.0
+                )
+
+        total_size += size_added - size_deleted
+        total_rows += rows_added - rows_deleted
+        total_files += files_added - files_removed
+        total_rowgroups += row_groups_added - row_groups_removed
+
+        incremental_framing_effect = 1
+        if size_deleted != 0:
+            incremental_framing_effect = (
+                int((total_size - size_added * 1.0) / total_size * 100000) / 1000
+            )
+            # incrementalFramingEffect = round(
+            #     (totalSize - sizeAdded * 1.0) / totalSize, 4
+            # )
+
+        changes_array.append(
+            [
+                element_version,
+                operation,
+                predicate,
+                change_timestamp,
+                incremental_framing_effect,
+                files_added,
+                files_removed,
+                total_files_before_change - files_removed,
+                total_files,
+                size_added,
+                size_deleted,
+                total_size,
+                row_groups_added,
+                row_groups_removed,
+                total_row_groups_before_change - row_groups_removed,
+                total_rowgroups,
+                rows_added,
+                rows_deleted,
+                rows_added - rows_deleted,
+                total_rows,
+                tags,
+            ]
+        )
+
+        element_version += 1
+
+    #  /********************************************************************************************************************
+    #      Display Gantt Chart of files
+    #  ********************************************************************************************************************/
+    spec: str = (
+        """{
+    "$$schema": 'https://vega.github.io/schema/vega-lite/v2.json',
+    "description": "A simple bar chart with ranged data (aka Gantt Chart).",
+    "width" : 1024 ,
+    "data": {
+        "values": %s
+    },
+    "layer":[
+        {"mark": "bar"},
+        {"mark": {
+        "type": "text",
+        "align": "center",
+        "baseline": "middle",
+        "dx": 40
+        },
+        "encoding": {
+        "text": {"field": "rows", "type": "quantitative", "format":","},
+        "color":{
+        "condition": {"test": "datum['isCurrent'] == 1", "value": "black"},
+        "value": "black"
+            }
+        }
+        }],
+    "encoding": {
+        "y": {"field": "file", "type": "ordinal","sort": "isCurrent","title":null,"axis":{"labelPadding":15,"labelLimit":360}},
+        "x": {"field": "start", "type": "temporal","title":null},
+        "x2": {"field": "end", "type": "temporal","title":null},
+            "color": {
+            "field": "isCurrent",
+            "scale": {"range": ["silver", "#ca8861"]}
+            }
+    }
+    }"""
+        % (parquet_files)
+    )
+
+    display(
+        HTML(
+            """
+        <!DOCTYPE html>
+        <html>
+            <head>
+                <script src="https://cdn.jsdelivr.net/npm/vega@5"></script>
+                <script src="https://cdn.jsdelivr.net/npm/vega-lite@5"></script>
+                <script src="https://cdn.jsdelivr.net/npm/vega-embed@6"></script>
+            </head>
+            <body>
+                <div id="vis"></div>
+                <script type="text/javascript">
+                    var spec = """
+            + spec
+            + """;
+                    var opt = {"renderer": "canvas", "actions": false};
+                    vegaEmbed("#vis", spec, opt);
+                </script>
+            </body>
+        </html>"""
+        )
+    )
+
+    return pd.DataFrame(
+        changes_array,
+        columns=[
+            "Change Number",
+            "Change Type",
+            "Predicate",
+            "Modification Time",
+            "Incremental Effect",
+            "Files Added",
+            "Files Removed",
+            "Files Preserved",
+            "Files After Change",
+            "Size Added",
+            "Sized Removed",
+            "Size After Change",
+            "Rowgroups Added",
+            "Rowgroups Removed",
+            "Rowgroups Preserved",
+            "Rowgroups After Change",
+            "Rows Added",
+            "Rows Removed",
+            "Rows Delta",
+            "Rows After Change",
+            "Tags",
+        ],
+    )

sempy_labs/_dictionary_diffs.py

@@ -0,0 +1,221 @@
+import re
+import json
+import difflib
+from collections import defaultdict
+
+
+def color_text(text, color_code):
+    return f"\033[{color_code}m{text}\033[0m"
+
+
+def stringify(payload):
+    try:
+        if isinstance(payload, list):
+            return (
+                "[\n" + ",\n".join(f"  {json.dumps(item)}" for item in payload) + "\n]"
+            )
+        return json.dumps(payload, indent=2, sort_keys=True)
+    except Exception:
+        return str(payload)
+
+
+def extract_top_level_group(path):
+    # For something like: resourcePackages[1].items[1].name → resourcePackages[1].items[1]
+    segments = re.split(r"\.(?![^[]*\])", path)  # split on dots not in brackets
+    return ".".join(segments[:-1]) if len(segments) > 1 else segments[0]
+
+
+def get_by_path(obj, path):
+    """Navigate into nested dict/list based on a dot/bracket path like: a.b[1].c"""
+    tokens = re.findall(r"\w+|\[\d+\]", path)
+    for token in tokens:
+        if token.startswith("["):
+            index = int(token[1:-1])
+            obj = obj[index]
+        else:
+            obj = obj.get(token)
+    return obj
+
+
+def deep_diff(d1, d2, path=""):
+    diffs = []
+    if isinstance(d1, dict) and isinstance(d2, dict):
+        keys = set(d1) | set(d2)
+        for key in sorted(keys):
+            new_path = f"{path}.{key}" if path else key
+            if key not in d1:
+                diffs.append(("+", new_path, None, d2[key]))
+            elif key not in d2:
+                diffs.append(("-", new_path, d1[key], None))
+            else:
+                diffs.extend(deep_diff(d1[key], d2[key], new_path))
+    elif isinstance(d1, list) and isinstance(d2, list):
+        min_len = min(len(d1), len(d2))
+        list_changed = False
+        for i in range(min_len):
+            if d1[i] != d2[i]:
+                list_changed = True
+                break
+        if list_changed or len(d1) != len(d2):
+            diffs.append(("~", path, d1, d2))
+    elif d1 != d2:
+        diffs.append(("~", path, d1, d2))
+    return diffs
+
+
+def diff_parts(d1, d2):
+
+    def build_path_map(parts):
+        return {part["path"]: part["payload"] for part in parts}
+
+    try:
+        paths1 = build_path_map(d1)
+    except Exception:
+        paths1 = d1
+    try:
+        paths2 = build_path_map(d2)
+    except Exception:
+        paths2 = d2
+    all_paths = set(paths1) | set(paths2)
+
+    for part_path in sorted(all_paths):
+        p1 = paths1.get(part_path)
+        p2 = paths2.get(part_path)
+
+        if p1 is None:
+            print(color_text(f"+ {part_path}", "32"))  # Green
+            continue
+        elif p2 is None:
+            print(color_text(f"- {part_path}", "31"))  # Red
+            continue
+        elif p1 == p2:
+            continue
+
+        if p1 is None or p2 is None:
+            print(
+                color_text(f"+ {part_path}", "32")
+                if p2 and not p1
+                else color_text(f"- {part_path}", "31")
+            )
+            continue
+
+        # Header for the changed part
+        print(color_text(f"~ {part_path}", "33"))
+
+        # Collect diffs
+        diffs = deep_diff(p1, p2)
+        # If the diff is only a change of a whole list (like appending to a list), group it under its key
+        merged_list_diffs = []
+        for change_type, full_path, old_val, new_val in diffs:
+            if (
+                change_type == "~"
+                and isinstance(old_val, list)
+                and isinstance(new_val, list)
+            ):
+                merged_list_diffs.append((change_type, full_path, old_val, new_val))
+
+        # Replace individual item diffs with unified list diff
+        if merged_list_diffs:
+            diffs = merged_list_diffs
+
+        # Group diffs by common parent path (e.g. items[1])
+        grouped = defaultdict(list)
+        for change_type, full_path, old_val, new_val in diffs:
+            group_path = extract_top_level_group(full_path)
+            grouped[group_path].append((change_type, full_path, old_val, new_val))
+
+        # Print each group once with unified diff for the full substructure
+        for group_path in sorted(grouped):
+            print("  " + color_text(f"~ {group_path}", "33"))
+
+            try:
+                old_group = get_by_path(p1, group_path)
+                new_group = get_by_path(p2, group_path)
+            except Exception:
+                old_group = new_group = None
+
+            # Skip showing diffs for empty/null groups
+            if isinstance(old_group, dict) and isinstance(new_group, dict):
+                old_keys = set(old_group.keys())
+                new_keys = set(new_group.keys())
+
+                for key in sorted(old_keys - new_keys):
+                    print(
+                        "  "
+                        + color_text(f"- {key}: {json.dumps(old_group[key])}", "31")
+                    )
+                for key in sorted(new_keys - old_keys):
+                    print(
+                        "  "
+                        + color_text(f"+ {key}: {json.dumps(new_group[key])}", "32")
+                    )
+                for key in sorted(old_keys & new_keys):
+                    if old_group[key] != new_group[key]:
+                        print("  " + color_text(f"~ {key}:", "33"))
+                        old_val_str = stringify(old_group[key]).splitlines()
+                        new_val_str = stringify(new_group[key]).splitlines()
+                        for line in difflib.unified_diff(
+                            old_val_str,
+                            new_val_str,
+                            fromfile="old",
+                            tofile="new",
+                            lineterm="",
+                        ):
+                            if line.startswith("@@"):
+                                print("    " + color_text(line, "36"))
+                            elif line.startswith("-") and not line.startswith("---"):
+                                print("    " + color_text(line, "31"))
+                            elif line.startswith("+") and not line.startswith("+++"):
+                                print("    " + color_text(line, "32"))
+            elif old_group is None and new_group is not None:
+                if isinstance(new_group, dict):
+                    # print all added keys
+                    for key, val in new_group.items():
+                        print("  " + color_text(f"+ {key}: {json.dumps(val)}", "32"))
+                elif isinstance(new_group, list):
+                    old_str = []
+                    new_str = stringify(new_group).splitlines()
+                    for line in difflib.unified_diff(
+                        old_str, new_str, fromfile="old", tofile="new", lineterm=""
+                    ):
+                        if line.startswith("@@"):
+                            print("  " + color_text(line, "36"))
+                        elif line.startswith("-") and not line.startswith("---"):
+                            print("  " + color_text(line, "31"))
+                        elif line.startswith("+") and not line.startswith("+++"):
+                            print("  " + color_text(line, "32"))
+                else:
+                    print("  " + color_text(f"+ {json.dumps(new_group)}", "32"))
+
+            elif new_group is None and old_group is not None:
+                if isinstance(old_group, dict):
+                    # print all removed keys
+                    for key, val in old_group.items():
+                        print("  " + color_text(f"- {key}: {json.dumps(val)}", "31"))
+                elif isinstance(old_group, list):
+                    old_str = stringify(old_group).splitlines()
+                    new_str = []
+                    for line in difflib.unified_diff(
+                        old_str, new_str, fromfile="old", tofile="new", lineterm=""
+                    ):
+                        if line.startswith("@@"):
+                            print("  " + color_text(line, "36"))
+                        elif line.startswith("-") and not line.startswith("---"):
+                            print("  " + color_text(line, "31"))
+                        elif line.startswith("+") and not line.startswith("+++"):
+                            print("  " + color_text(line, "32"))
+                else:
+                    print("  " + color_text(f"- {json.dumps(old_group)}", "31"))
+            else:
+                old_str = stringify(old_group).splitlines()
+                new_str = stringify(new_group).splitlines()
+
+                for line in difflib.unified_diff(
+                    old_str, new_str, fromfile="old", tofile="new", lineterm=""
+                ):
+                    if line.startswith("@@"):
+                        print("  " + color_text(line, "36"))
+                    elif line.startswith("-") and not line.startswith("---"):
+                        print("  " + color_text(line, "31"))
+                    elif line.startswith("+") and not line.startswith("+++"):
+                        print("  " + color_text(line, "32"))

sempy_labs/_documentation.py

@@ -0,0 +1,147 @@
+import sempy
+import sempy.fabric as fabric
+import pandas as pd
+from typing import List, Optional
+from sempy._utils._log import log
+
+
+@log
+def list_all_items(workspaces: Optional[str | List[str]] = None):
+
+    df = pd.DataFrame(
+        columns=[
+            "Workspace Name",
+            "Workspace Id",
+            "Item Name",
+            "Item Type",
+            "Description",
+        ]
+    )
+
+    if isinstance(workspaces, str):
+        workspaces = [workspaces]
+
+    dfW = fabric.list_workspaces()
+    if workspaces is not None:
+        dfW = dfW[dfW["Name"].isin(workspaces)]
+
+    for _, r in dfW.iterrows():
+        workspace_name = r["Name"]
+        workspace_id = r["Id"]
+        dfI = fabric.list_items(workspace=workspace_name)
+        for _, r2 in dfI.iterrows():
+
+            new_data = {
+                "Workspace Name": workspace_name,
+                "Workspace Id": workspace_id,
+                "Item Name": r2["Name"],
+                "Item Type": r2["Type"],
+                "Description": r2["Description"],
+            }
+            df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
+
+    return df
+
+
+@log
+def data_dictionary(dataset: str, workspace: Optional[str | None] = None):
+
+    from sempy_labs.tom import connect_semantic_model
+
+    sempy.fabric._client._utils._init_analysis_services()
+    import Microsoft.AnalysisServices.Tabular as TOM
+
+    df = pd.DataFrame(
+        columns=[
+            "Workspace Name",
+            "Model Name",
+            "Table Name",
+            "Object Type",
+            "Object Name",
+            "Hidden Flag",
+            "Description",
+            "Display Folder",
+            "Measure Formula",
+        ]
+    )
+
+    with connect_semantic_model(
+        dataset=dataset, readonly=True, workspace=workspace
+    ) as tom:
+        for t in tom.model.Tables:
+            expr = None
+            if tom.is_calculated_table(table_name=t.Name):
+                pName = next(p.Name for p in t.Partitions)
+                expr = t.Partitions[pName].Source.Expression
+
+            new_data = {
+                "Workspace Name": workspace,
+                "Model Name": dataset,
+                "Table Name": t.Name,
+                "Object Type": t.ObjectType,
+                "Object Name": t.Name,
+                "Hidden Flag": t.IsHidden,
+                "Description": t.Description,
+                "Display Folder": None,
+                "Measure Formula": expr,
+            }
+            df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
+            cols = [c for c in t.Columns if c.Type != TOM.ColumnType.RowNumber]
+            for c in cols:
+
+                def get_calc_column_expression(table_name, column_name):
+                    expr = None
+                    if tom.is_calculated_column(
+                        table_name=table_name, column_name=column_name
+                    ):
+                        expr = c.Expression
+                    return expr
+
+                new_data = {
+                    "Workspace Name": workspace,
+                    "Model Name": dataset,
+                    "Table Name": t.Name,
+                    "Object Type": c.ObjectType,
+                    "Object Name": c.Name,
+                    "Hidden Flag": c.IsHidden,
+                    "Description": c.Description,
+                    "Display Folder": c.DisplayFolder,
+                    "Measure Formula": get_calc_column_expression(t.Name, c.Name),
+                }
+                df = pd.concat(
+                    [df, pd.DataFrame(new_data, index=[0])], ignore_index=True
+                )
+            for m in t.Measures:
+                new_data = {
+                    "Workspace Name": workspace,
+                    "Model Name": dataset,
+                    "Table Name": t.Name,
+                    "Object Type": m.ObjectType,
+                    "Object Name": m.Name,
+                    "Hidden Flag": m.IsHidden,
+                    "Description": m.Description,
+                    "Display Folder": m.DisplayFolder,
+                    "Measure Formula": m.Expression,
+                }
+                df = pd.concat(
+                    [df, pd.DataFrame(new_data, index=[0])], ignore_index=True
+                )
+
+            if t.CalculationGroup is not None:
+                for ci in t.CalculationGroup.CalculationItems:
+                    new_data = {
+                        "Workspace Name": workspace,
+                        "Model Name": dataset,
+                        "Table Name": t.Name,
+                        "Object Type": "Calculation Item",
+                        "Object Name": ci.Name,
+                        "Hidden Flag": t.IsHidden,
+                        "Description": ci.Description,
+                        "Display Folder": None,
+                        "Measure Formula": ci.Expression,
+                    }
+                    df = pd.concat(
+                        [df, pd.DataFrame(new_data, index=[0])], ignore_index=True
+                    )
+
+    return df