semantic-link-labs 0.9.6__tar.gz → 0.9.7__tar.gz

{semantic_link_labs-0.9.6 → semantic_link_labs-0.9.7}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: semantic-link-labs
-Version: 0.9.6
+Version: 0.9.7
 Summary: Semantic Link Labs for Microsoft Fabric
 Author: Microsoft Corporation
 License: MIT License
@@ -15,18 +15,19 @@ Classifier: Framework :: Jupyter
 Requires-Python: <3.12,>=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: semantic-link-sempy>=0.8.5
+Requires-Dist: semantic-link-sempy>=0.9.3
 Requires-Dist: anytree
 Requires-Dist: powerbiclient
 Requires-Dist: polib
 Requires-Dist: jsonpath_ng
 Provides-Extra: test
 Requires-Dist: pytest>=8.2.1; extra == "test"
+Dynamic: license-file
 
 # Semantic Link Labs
 
 [![PyPI version](https://badge.fury.io/py/semantic-link-labs.svg)](https://badge.fury.io/py/semantic-link-labs)
-[![Read The Docs](https://readthedocs.org/projects/semantic-link-labs/badge/?version=0.9.6&style=flat)](https://readthedocs.org/projects/semantic-link-labs/)
+[![Read The Docs](https://readthedocs.org/projects/semantic-link-labs/badge/?version=0.9.7&style=flat)](https://readthedocs.org/projects/semantic-link-labs/)
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 [![Downloads](https://static.pepy.tech/badge/semantic-link-labs)](https://pepy.tech/project/semantic-link-labs)
 
@@ -148,6 +149,7 @@ An even better way to ensure the semantic-link-labs library is available in your
 2. Select your newly created environment within the 'Environment' drop down in the navigation bar at the top of the notebook
 
 ## Version History
+* [0.9.7](https://github.com/microsoft/semantic-link-labs/releases/tag/0.9.7) (April 1, 2025)
 * [0.9.6](https://github.com/microsoft/semantic-link-labs/releases/tag/0.9.6) (March 12, 2025)
 * [0.9.5](https://github.com/microsoft/semantic-link-labs/releases/tag/0.9.5) (March 7, 2025)
 * [0.9.4](https://github.com/microsoft/semantic-link-labs/releases/tag/0.9.4) (February 27, 2025)
@@ -309,7 +311,7 @@ We use [black](github.com/psf/black) formatting as a code formatting standard. M
 
 Run this code to install black
 ```cli
-pip install black
+pip install black==25.1.0
 ```
 
 Run this code to format your code using black

{semantic_link_labs-0.9.6 → semantic_link_labs-0.9.7}/README.md

@@ -1,7 +1,7 @@
 # Semantic Link Labs
 
 [![PyPI version](https://badge.fury.io/py/semantic-link-labs.svg)](https://badge.fury.io/py/semantic-link-labs)
-[![Read The Docs](https://readthedocs.org/projects/semantic-link-labs/badge/?version=0.9.6&style=flat)](https://readthedocs.org/projects/semantic-link-labs/)
+[![Read The Docs](https://readthedocs.org/projects/semantic-link-labs/badge/?version=0.9.7&style=flat)](https://readthedocs.org/projects/semantic-link-labs/)
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 [![Downloads](https://static.pepy.tech/badge/semantic-link-labs)](https://pepy.tech/project/semantic-link-labs)
 
@@ -123,6 +123,7 @@ An even better way to ensure the semantic-link-labs library is available in your
 2. Select your newly created environment within the 'Environment' drop down in the navigation bar at the top of the notebook
 
 ## Version History
+* [0.9.7](https://github.com/microsoft/semantic-link-labs/releases/tag/0.9.7) (April 1, 2025)
 * [0.9.6](https://github.com/microsoft/semantic-link-labs/releases/tag/0.9.6) (March 12, 2025)
 * [0.9.5](https://github.com/microsoft/semantic-link-labs/releases/tag/0.9.5) (March 7, 2025)
 * [0.9.4](https://github.com/microsoft/semantic-link-labs/releases/tag/0.9.4) (February 27, 2025)
@@ -284,7 +285,7 @@ We use [black](github.com/psf/black) formatting as a code formatting standard. M
 
 Run this code to install black
 ```cli
-pip install black
+pip install black==25.1.0
 ```
 
 Run this code to format your code using black

{semantic_link_labs-0.9.6 → semantic_link_labs-0.9.7}/docs/source/conf.py

@@ -13,7 +13,7 @@ sys.path.insert(0, os.path.abspath('../../src/'))
 project = 'semantic-link-labs'
 copyright = '2024, Microsoft and community'
 author = 'Microsoft and community'
-release = '0.9.6'
+release = '0.9.7'
 
 # -- General configuration ---------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration

{semantic_link_labs-0.9.6 → semantic_link_labs-0.9.7}/environment.yml

@@ -6,7 +6,7 @@ dependencies:
 - pytest-cov
 - pytest-mock
 - pip:
-  - semantic-link-sempy>=0.8.5
+  - semantic-link-sempy>=0.9.3
   - azure-identity==1.7.1
   - azure-storage-blob>=12.9.0
   - pandas-stubs

{semantic_link_labs-0.9.6 → semantic_link_labs-0.9.7}/pyproject.toml

@@ -7,7 +7,7 @@ name="semantic-link-labs"
 authors = [
     { name = "Microsoft Corporation" },
 ]
-version="0.9.6"
+version="0.9.7"
 description="Semantic Link Labs for Microsoft Fabric"
 readme="README.md"
 requires-python=">=3.10,<3.12"
@@ -23,7 +23,7 @@ classifiers = [
 license= { text = "MIT License" }
 
 dependencies = [
-    "semantic-link-sempy>=0.8.5",
+    "semantic-link-sempy>=0.9.3",
     "anytree",
     "powerbiclient",
     "polib",

{semantic_link_labs-0.9.6 → semantic_link_labs-0.9.7}/src/semantic_link_labs.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: semantic-link-labs
-Version: 0.9.6
+Version: 0.9.7
 Summary: Semantic Link Labs for Microsoft Fabric
 Author: Microsoft Corporation
 License: MIT License
@@ -15,18 +15,19 @@ Classifier: Framework :: Jupyter
 Requires-Python: <3.12,>=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: semantic-link-sempy>=0.8.5
+Requires-Dist: semantic-link-sempy>=0.9.3
 Requires-Dist: anytree
 Requires-Dist: powerbiclient
 Requires-Dist: polib
 Requires-Dist: jsonpath_ng
 Provides-Extra: test
 Requires-Dist: pytest>=8.2.1; extra == "test"
+Dynamic: license-file
 
 # Semantic Link Labs
 
 [![PyPI version](https://badge.fury.io/py/semantic-link-labs.svg)](https://badge.fury.io/py/semantic-link-labs)
-[![Read The Docs](https://readthedocs.org/projects/semantic-link-labs/badge/?version=0.9.6&style=flat)](https://readthedocs.org/projects/semantic-link-labs/)
+[![Read The Docs](https://readthedocs.org/projects/semantic-link-labs/badge/?version=0.9.7&style=flat)](https://readthedocs.org/projects/semantic-link-labs/)
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 [![Downloads](https://static.pepy.tech/badge/semantic-link-labs)](https://pepy.tech/project/semantic-link-labs)
 
@@ -148,6 +149,7 @@ An even better way to ensure the semantic-link-labs library is available in your
 2. Select your newly created environment within the 'Environment' drop down in the navigation bar at the top of the notebook
 
 ## Version History
+* [0.9.7](https://github.com/microsoft/semantic-link-labs/releases/tag/0.9.7) (April 1, 2025)
 * [0.9.6](https://github.com/microsoft/semantic-link-labs/releases/tag/0.9.6) (March 12, 2025)
 * [0.9.5](https://github.com/microsoft/semantic-link-labs/releases/tag/0.9.5) (March 7, 2025)
 * [0.9.4](https://github.com/microsoft/semantic-link-labs/releases/tag/0.9.4) (February 27, 2025)
@@ -309,7 +311,7 @@ We use [black](github.com/psf/black) formatting as a code formatting standard. M
 
 Run this code to install black
 ```cli
-pip install black
+pip install black==25.1.0
 ```
 
 Run this code to format your code using black

{semantic_link_labs-0.9.6 → semantic_link_labs-0.9.7}/src/semantic_link_labs.egg-info/SOURCES.txt

@@ -49,6 +49,7 @@ src/sempy_labs/_dataflows.py
 src/sempy_labs/_dax.py
 src/sempy_labs/_dax_query_view.py
 src/sempy_labs/_delta_analyzer.py
+src/sempy_labs/_delta_analyzer_history.py
 src/sempy_labs/_deployment_pipelines.py
 src/sempy_labs/_documentation.py
 src/sempy_labs/_environments.py
@@ -164,6 +165,7 @@ src/sempy_labs/graph/_groups.py
 src/sempy_labs/graph/_teams.py
 src/sempy_labs/graph/_users.py
 src/sempy_labs/lakehouse/__init__.py
+src/sempy_labs/lakehouse/_blobs.py
 src/sempy_labs/lakehouse/_get_lakehouse_columns.py
 src/sempy_labs/lakehouse/_get_lakehouse_tables.py
 src/sempy_labs/lakehouse/_lakehouse.py
@@ -190,6 +192,7 @@ src/sempy_labs/report/_report_helper.py
 src/sempy_labs/report/_report_list_functions.py
 src/sempy_labs/report/_report_rebind.py
 src/sempy_labs/report/_reportwrapper.py
+src/sempy_labs/report/_save_report.py
 src/sempy_labs/report/_bpareporttemplate/.platform
 src/sempy_labs/report/_bpareporttemplate/definition.pbir
 src/sempy_labs/report/_bpareporttemplate/.pbi/localSettings.json
@@ -222,5 +225,4 @@ src/sempy_labs/report/_bpareporttemplate/definition/pages/d37dce724a0ccc30044b/v
 src/sempy_labs/tom/__init__.py
 src/sempy_labs/tom/_model.py
 tests/__init__.py
-tests/test_friendly_case.py
-tests/test_tom.py
+tests/test_friendly_case.py

{semantic_link_labs-0.9.6 → semantic_link_labs-0.9.7}/src/semantic_link_labs.egg-info/requires.txt

@@ -1,4 +1,4 @@
-semantic-link-sempy>=0.8.5
+semantic-link-sempy>=0.9.3
 anytree
 powerbiclient
 polib

{semantic_link_labs-0.9.6 → semantic_link_labs-0.9.7}/src/sempy_labs/__init__.py

@@ -1,3 +1,6 @@
+from sempy_labs._delta_analyzer_history import (
+    delta_analyzer_history,
+)
 from sempy_labs._dax_query_view import (
     generate_dax_query_view_url,
 )
@@ -554,4 +557,5 @@ __all__ = [
     "create_sql_database",
     "delete_sql_database",
     "list_sql_databases",
+    "delta_analyzer_history",
 ]

{semantic_link_labs-0.9.6 → semantic_link_labs-0.9.7}/src/sempy_labs/_ai.py

@@ -216,7 +216,9 @@ def generate_aggs(
     f"{icons.green_dot} The '{aggLakeTName}' table has been created/updated in the lakehouse."
 
     # Create/update semantic model agg table
-    tom_server = fabric.create_tom_server(readonly=False, workspace=workspace)
+    tom_server = fabric.create_tom_server(
+        dataset=dataset, readonly=False, workspace=workspace
+    )
     m = tom_server.Databases.GetByName(dataset).Model
     print(f"\n{icons.in_progress} Updating the '{dataset}' semantic model...")
     dfC_agg = dfC[dfC["Table Name"] == aggTableName]

{semantic_link_labs-0.9.6 → semantic_link_labs-0.9.7}/src/sempy_labs/_capacities.py

@@ -1,4 +1,3 @@
-import sempy.fabric as fabric
 from typing import Optional, List, Tuple
 from sempy._utils._log import log
 import sempy_labs._icons as icons

{semantic_link_labs-0.9.6 → semantic_link_labs-0.9.7}/src/sempy_labs/_dax_query_view.py

@@ -5,11 +5,13 @@ from sempy_labs._helper_functions import (
     _get_fabric_context_setting,
     resolve_workspace_id,
 )
+from sempy._utils._log import log
 import gzip
 import base64
 import urllib.parse
 
 
+@log
 def generate_dax_query_view_url(
     dataset: str | UUID, dax_string: str, workspace: Optional[str | UUID] = None
 ):

semantic_link_labs-0.9.7/src/sempy_labs/_delta_analyzer_history.py

@@ -0,0 +1,298 @@
+import pandas as pd
+from typing import Optional
+import pyarrow.parquet as pq
+from sempy_labs._helper_functions import (
+    create_abfss_path,
+    resolve_workspace_id,
+    resolve_lakehouse_id,
+    _mount,
+)
+from sempy._utils._log import log
+from tqdm.auto import tqdm
+from uuid import UUID
+from datetime import datetime
+
+
+@log
+def delta_analyzer_history(
+    table_name: str,
+    schema: Optional[str] = None,
+    lakehouse: Optional[str | UUID] = None,
+    workspace: Optional[str | UUID] = None,
+) -> pd.DataFrame:
+    """
+    Analyzes the transaction log for a specified delta table and shows the results in dataframe.  One row per data modification operation.
+
+    Keeps track on the number of Parquet files, rowgroups, file size and #rows impacted by each change.
+
+    Incremental Framing effect: 100% = highly effective, 0% = no benefit at all
+
+    Parameters
+    ----------
+    table_name : str
+        The delta table name.
+    schema : str, default=None
+        The schema name of the delta table.
+    lakehouse : str | uuid.UUID, default=None
+        The Fabric lakehouse name or ID.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID used by the lakehouse.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    pandas.DataFrame
+        Displays a gantt visual showing a timeline for individual parquet files.
+    """
+
+    import notebookutils
+    from IPython.display import display, HTML
+
+    workspace_id = resolve_workspace_id(workspace=workspace)
+    lakehouse_id = resolve_lakehouse_id(lakehouse=lakehouse, workspace=workspace)
+
+    table_path = create_abfss_path(lakehouse_id, workspace_id, table_name, schema)
+    local_path = _mount(lakehouse=lakehouse, workspace=workspace)
+    table_path_local = f"{local_path}/Tables/{table_name}"
+    delta_table_path = f"{table_path}/_delta_log"
+
+    files = notebookutils.fs.ls(delta_table_path)
+    json_files = [file.name for file in files if file.name.endswith(".json")]
+
+    element_version = total_size = total_rows = total_files = total_rowgroups = 0
+    changes_array = []
+    parquet_files = []
+    my_date_time_format = "%Y-%m-%d %H:%M:%S.%f"
+    now_to_epoch = datetime.now().strftime(my_date_time_format)
+    num_latest_files = len(json_files)
+
+    for idx, file in enumerate(bar := tqdm(json_files), start=1):
+        bar.set_description(
+            f"Analyzing the '{file}' parquet file ({idx}/{num_latest_files})..."
+        )
+
+        change_timestamp = datetime.strptime(
+            "2001-01-01 12:00:00.000", my_date_time_format
+        )
+        df = pd.read_json(f"{delta_table_path}/{file}", lines=True)
+
+        rows_added = size_added = rows_deleted = size_deleted = files_added = (
+            files_removed
+        ) = row_groups_added = row_groups_removed = 0
+        total_files_before_change = total_files
+        total_row_groups_before_change = total_rowgroups
+        operation = predicate = tags = ""
+
+        for _, row in df.iterrows():
+            add_row = row.get("add")
+            remove_row = row.get("remove")
+            commit_row = row.get("commitInfo")
+
+            if isinstance(add_row, dict):
+                file_name = add_row["path"]
+                fs_filename = f"{table_path}/{file_name}"
+                size_added += add_row["size"]
+                files_added += 1
+                filerows_added = 0
+
+                if notebookutils.fs.exists(fs_filename):
+                    parquet_file = pq.ParquetFile(table_path_local + f"/{file_name}")
+                    for i in range(parquet_file.num_row_groups):
+                        row_group = parquet_file.metadata.row_group(i)
+                        num_rows = row_group.num_rows
+                        filerows_added += num_rows
+                        rows_added += num_rows
+
+                    row_groups_added += parquet_file.num_row_groups
+
+                    start = str(
+                        datetime.fromtimestamp(add_row["modificationTime"] / 1000.0)
+                    )
+                    parquet_files.append(
+                        {
+                            "file": file_name,
+                            "start": start,
+                            "end": now_to_epoch,
+                            "rows": filerows_added,
+                            "isCurrent": 1,
+                        }
+                    )
+
+            if isinstance(remove_row, dict):
+                file_name = remove_row["path"]
+                fs_filename = f"{table_path}/{file_name}"
+
+                if notebookutils.fs.exists(fs_filename):
+                    parquet_file = pq.ParquetFile(table_path_local + f"/{file_name}")
+                    for i in range(parquet_file.num_row_groups):
+                        row_group = parquet_file.metadata.row_group(i)
+                        num_rows = row_group.num_rows
+                        rows_deleted += num_rows
+
+                    files_removed += 1
+                    size_deleted += remove_row.get("size", 0)
+                    row_groups_removed += parquet_file.num_row_groups
+
+                    result = next(
+                        (row for row in parquet_files if row["file"] == file_name), None
+                    )
+                    if result:
+                        result.update(
+                            {
+                                "isCurrent": 0,
+                                "end": str(
+                                    datetime.fromtimestamp(
+                                        remove_row["deletionTimestamp"] / 1000.0
+                                    )
+                                ),
+                            }
+                        )
+
+            if isinstance(commit_row, dict):
+                operation = commit_row.get("operation")
+                tags = commit_row.get("tags")
+                predicate = commit_row.get("operationParameters", {}).get("predicate")
+
+                if operation == "VACUUM START":
+                    operation_metrics = commit_row.get("operationMetrics", {})
+                    total_files -= int(operation_metrics.get("numFilesToDelete", 0))
+                    total_size -= int(operation_metrics.get("sizeOfDataToDelete", 0))
+
+                change_timestamp = datetime.fromtimestamp(
+                    commit_row["timestamp"] / 1000.0
+                )
+
+        total_size += size_added - size_deleted
+        total_rows += rows_added - rows_deleted
+        total_files += files_added - files_removed
+        total_rowgroups += row_groups_added - row_groups_removed
+
+        incremental_framing_effect = 1
+        if size_deleted != 0:
+            incremental_framing_effect = (
+                int((total_size - size_added * 1.0) / total_size * 100000) / 1000
+            )
+            # incrementalFramingEffect = round(
+            #     (totalSize - sizeAdded * 1.0) / totalSize, 4
+            # )
+
+        changes_array.append(
+            [
+                element_version,
+                operation,
+                predicate,
+                change_timestamp,
+                incremental_framing_effect,
+                files_added,
+                files_removed,
+                total_files_before_change - files_removed,
+                total_files,
+                size_added,
+                size_deleted,
+                total_size,
+                row_groups_added,
+                row_groups_removed,
+                total_row_groups_before_change - row_groups_removed,
+                total_rowgroups,
+                rows_added,
+                rows_deleted,
+                rows_added - rows_deleted,
+                total_rows,
+                tags,
+            ]
+        )
+
+        element_version += 1
+
+    #  /********************************************************************************************************************
+    #      Display Gantt Chart of files
+    #  ********************************************************************************************************************/
+    spec: str = (
+        """{
+    "$$schema": 'https://vega.github.io/schema/vega-lite/v2.json',
+    "description": "A simple bar chart with ranged data (aka Gantt Chart).",
+    "width" : 1024 ,
+    "data": {
+        "values": %s
+    },
+    "layer":[
+        {"mark": "bar"},
+        {"mark": {
+        "type": "text",
+        "align": "center",
+        "baseline": "middle",
+        "dx": 40
+        },
+        "encoding": {
+        "text": {"field": "rows", "type": "quantitative", "format":","},
+        "color":{
+        "condition": {"test": "datum['isCurrent'] == 1", "value": "black"},
+        "value": "black"
+            }
+        }
+        }],
+    "encoding": {
+        "y": {"field": "file", "type": "ordinal","sort": "isCurrent","title":null,"axis":{"labelPadding":15,"labelLimit":360}},
+        "x": {"field": "start", "type": "temporal","title":null},
+        "x2": {"field": "end", "type": "temporal","title":null},
+            "color": {
+            "field": "isCurrent",
+            "scale": {"range": ["silver", "#ca8861"]}
+            }
+    }
+    }"""
+        % (parquet_files)
+    )
+
+    display(
+        HTML(
+            """
+        <!DOCTYPE html>
+        <html>
+            <head>
+                <script src="https://cdn.jsdelivr.net/npm/vega@5"></script>
+                <script src="https://cdn.jsdelivr.net/npm/vega-lite@5"></script>
+                <script src="https://cdn.jsdelivr.net/npm/vega-embed@6"></script>
+            </head>
+            <body>
+                <div id="vis"></div>
+                <script type="text/javascript">
+                    var spec = """
+            + spec
+            + """;
+                    var opt = {"renderer": "canvas", "actions": false};
+                    vegaEmbed("#vis", spec, opt);
+                </script>
+            </body>
+        </html>"""
+        )
+    )
+
+    return pd.DataFrame(
+        changes_array,
+        columns=[
+            "Change Number",
+            "Change Type",
+            "Predicate",
+            "Modification Time",
+            "Incremental Effect",
+            "Files Added",
+            "Files Removed",
+            "Files Preserved",
+            "Files After Change",
+            "Size Added",
+            "Sized Removed",
+            "Size After Change",
+            "Rowgroups Added",
+            "Rowgroups Removed",
+            "Rowgroups Preserved",
+            "Rowgroups After Change",
+            "Rows Added",
+            "Rows Removed",
+            "Rows Delta",
+            "Rows After Change",
+            "Tags",
+        ],
+    )