semantic-link-labs 0.9.2__tar.gz → 0.9.3__tar.gz

{semantic_link_labs-0.9.2 → semantic_link_labs-0.9.3}/.github/workflows/build.yaml

@@ -28,7 +28,7 @@ jobs:
         shell: bash
 
       - name: Cache conda
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         env:
           # Increase this value to reset cache if environment.yml has not changed
           CACHE_NUMBER: 0
@@ -37,7 +37,7 @@ jobs:
           key:
             ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ steps.get-date.outputs.today }}-${{ hashFiles('environment.yml') }}
 
-      - uses: conda-incubator/setup-miniconda@v2
+      - uses: conda-incubator/setup-miniconda@v3
         with:
           auto-update-conda: true
           activate-environment: fabric

{semantic_link_labs-0.9.2 → semantic_link_labs-0.9.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: semantic-link-labs
-Version: 0.9.2
+Version: 0.9.3
 Summary: Semantic Link Labs for Microsoft Fabric
 Author: Microsoft Corporation
 License: MIT License
@@ -26,7 +26,7 @@ Requires-Dist: pytest>=8.2.1; extra == "test"
 # Semantic Link Labs
 
 [![PyPI version](https://badge.fury.io/py/semantic-link-labs.svg)](https://badge.fury.io/py/semantic-link-labs)
-[![Read The Docs](https://readthedocs.org/projects/semantic-link-labs/badge/?version=0.9.2&style=flat)](https://readthedocs.org/projects/semantic-link-labs/)
+[![Read The Docs](https://readthedocs.org/projects/semantic-link-labs/badge/?version=0.9.3&style=flat)](https://readthedocs.org/projects/semantic-link-labs/)
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 [![Downloads](https://static.pepy.tech/badge/semantic-link-labs)](https://pepy.tech/project/semantic-link-labs)
 
@@ -144,6 +144,7 @@ An even better way to ensure the semantic-link-labs library is available in your
 2. Select your newly created environment within the 'Environment' drop down in the navigation bar at the top of the notebook
 
 ## Version History
+* [0.9.3](https://github.com/microsoft/semantic-link-labs/releases/tag/0.9.3) (February 13, 2025)
 * [0.9.2](https://github.com/microsoft/semantic-link-labs/releases/tag/0.9.2) (February 5, 2025)
 * [0.9.1](https://github.com/microsoft/semantic-link-labs/releases/tag/0.9.1) (January 22, 2025)
 * [0.9.0](https://github.com/microsoft/semantic-link-labs/releases/tag/0.9.0) (January 22, 2025)

{semantic_link_labs-0.9.2 → semantic_link_labs-0.9.3}/README.md

@@ -1,7 +1,7 @@
 # Semantic Link Labs
 
 [![PyPI version](https://badge.fury.io/py/semantic-link-labs.svg)](https://badge.fury.io/py/semantic-link-labs)
-[![Read The Docs](https://readthedocs.org/projects/semantic-link-labs/badge/?version=0.9.2&style=flat)](https://readthedocs.org/projects/semantic-link-labs/)
+[![Read The Docs](https://readthedocs.org/projects/semantic-link-labs/badge/?version=0.9.3&style=flat)](https://readthedocs.org/projects/semantic-link-labs/)
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 [![Downloads](https://static.pepy.tech/badge/semantic-link-labs)](https://pepy.tech/project/semantic-link-labs)
 
@@ -119,6 +119,7 @@ An even better way to ensure the semantic-link-labs library is available in your
 2. Select your newly created environment within the 'Environment' drop down in the navigation bar at the top of the notebook
 
 ## Version History
+* [0.9.3](https://github.com/microsoft/semantic-link-labs/releases/tag/0.9.3) (February 13, 2025)
 * [0.9.2](https://github.com/microsoft/semantic-link-labs/releases/tag/0.9.2) (February 5, 2025)
 * [0.9.1](https://github.com/microsoft/semantic-link-labs/releases/tag/0.9.1) (January 22, 2025)
 * [0.9.0](https://github.com/microsoft/semantic-link-labs/releases/tag/0.9.0) (January 22, 2025)

{semantic_link_labs-0.9.2 → semantic_link_labs-0.9.3}/docs/source/conf.py

@@ -13,7 +13,7 @@ sys.path.insert(0, os.path.abspath('../../src/'))
 project = 'semantic-link-labs'
 copyright = '2024, Microsoft and community'
 author = 'Microsoft and community'
-release = '0.9.2'
+release = '0.9.3'
 
 # -- General configuration ---------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration

{semantic_link_labs-0.9.2 → semantic_link_labs-0.9.3}/pyproject.toml

@@ -7,7 +7,7 @@ name="semantic-link-labs"
 authors = [
     { name = "Microsoft Corporation" },
 ]
-version="0.9.2"
+version="0.9.3"
 description="Semantic Link Labs for Microsoft Fabric"
 readme="README.md"
 requires-python=">=3.10,<3.12"

{semantic_link_labs-0.9.2 → semantic_link_labs-0.9.3}/src/semantic_link_labs.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: semantic-link-labs
-Version: 0.9.2
+Version: 0.9.3
 Summary: Semantic Link Labs for Microsoft Fabric
 Author: Microsoft Corporation
 License: MIT License
@@ -26,7 +26,7 @@ Requires-Dist: pytest>=8.2.1; extra == "test"
 # Semantic Link Labs
 
 [![PyPI version](https://badge.fury.io/py/semantic-link-labs.svg)](https://badge.fury.io/py/semantic-link-labs)
-[![Read The Docs](https://readthedocs.org/projects/semantic-link-labs/badge/?version=0.9.2&style=flat)](https://readthedocs.org/projects/semantic-link-labs/)
+[![Read The Docs](https://readthedocs.org/projects/semantic-link-labs/badge/?version=0.9.3&style=flat)](https://readthedocs.org/projects/semantic-link-labs/)
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 [![Downloads](https://static.pepy.tech/badge/semantic-link-labs)](https://pepy.tech/project/semantic-link-labs)
 
@@ -144,6 +144,7 @@ An even better way to ensure the semantic-link-labs library is available in your
 2. Select your newly created environment within the 'Environment' drop down in the navigation bar at the top of the notebook
 
 ## Version History
+* [0.9.3](https://github.com/microsoft/semantic-link-labs/releases/tag/0.9.3) (February 13, 2025)
 * [0.9.2](https://github.com/microsoft/semantic-link-labs/releases/tag/0.9.2) (February 5, 2025)
 * [0.9.1](https://github.com/microsoft/semantic-link-labs/releases/tag/0.9.1) (January 22, 2025)
 * [0.9.0](https://github.com/microsoft/semantic-link-labs/releases/tag/0.9.0) (January 22, 2025)

{semantic_link_labs-0.9.2 → semantic_link_labs-0.9.3}/src/sempy_labs/_ai.py

@@ -1,10 +1,13 @@
 import sempy
 import sempy.fabric as fabric
 import pandas as pd
-from pyspark.sql import SparkSession
 from typing import List, Optional, Union
 from IPython.display import display
 import sempy_labs._icons as icons
+from sempy_labs._helper_functions import (
+    _read_delta_table,
+    _run_spark_sql_query,
+)
 
 
 def optimize_semantic_model(dataset: str, workspace: Optional[str] = None):
@@ -186,13 +189,13 @@ def generate_aggs(
 
     query = query[:-1]
 
-    spark = SparkSession.builder.getOrCreate()
     fromTablePath = create_abfss_path(
         lakehouse_id=lakehouse_id,
         lakehouse_workspace_id=lakehouse_workspace_id,
         delta_table_name=lakeTName,
     )
-    df = spark.read.format("delta").load(fromTablePath)
+
+    df = _read_delta_table(fromTablePath)
     tempTableName = f"delta_table_{lakeTName}"
     df.createOrReplaceTempView(tempTableName)
     sqlQuery = f"{query} \n FROM {tempTableName} {groupBy}"
@@ -201,7 +204,7 @@ def generate_aggs(
     print(sqlQuery)
 
     # Save query to spark dataframe
-    spark_df = spark.sql(sqlQuery)
+    spark_df = _run_spark_sql_query(sqlQuery)
     f"\nCreating/updating the '{aggLakeTName}' table in the lakehouse..."
     # Write spark dataframe to delta table
     aggFilePath = create_abfss_path(
@@ -419,7 +422,7 @@ def generate_aggs(
 # dfP = fabric.list_partitions(dataset = dataset, workspace = workspace)
 # isDirectLake = any(r['Mode'] == 'DirectLake' for i, r in dfP.iterrows())
 
-# spark = SparkSession.builder.getOrCreate()
+# spark = _create_spark_session()
 # views = spark.sql(f"SHOW VIEWS IN {lakehouse}").collect()
 # for view in views:
 #    viewName = view['viewName']

{semantic_link_labs-0.9.2 → semantic_link_labs-0.9.3}/src/sempy_labs/_connections.py

@@ -19,6 +19,8 @@ def delete_connection(connection: str | UUID):
 
     This is a wrapper function for the following API: `Connections - Delete Connection <https://learn.microsoft.com/rest/api/fabric/core/connections/delete-connection>`_.
 
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
     Parameters
     ----------
     connection : str | uuid.UUID
@@ -26,7 +28,9 @@ def delete_connection(connection: str | UUID):
     """
 
     connection_id = _resolve_connection_id(connection)
-    _base_api(request=f"/v1/connections/{connection_id}", method="delete")
+    _base_api(
+        request=f"/v1/connections/{connection_id}", client="fabric_sp", method="delete"
+    )
     print(f"{icons.green_dot} The '{connection}' connection has been deleted.")
 
 
@@ -36,6 +40,8 @@ def delete_connection_role_assignment(connection: str | UUID, role_assignment_id
 
     This is a wrapper function for the following API: `Connections - Delete Connection Role Assignment <https://learn.microsoft.com/rest/api/fabric/core/connections/delete-connection-role-assignment>`_.
 
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
     Parameters
     ----------
     connection : str | uuid.UUID
@@ -47,6 +53,7 @@ def delete_connection_role_assignment(connection: str | UUID, role_assignment_id
     connection_id = _resolve_connection_id(connection)
     _base_api(
         request=f"/v1/connections/{connection_id}/roleAssignments/{role_assignment_id}",
+        client="fabric_sp",
         method="delete",
     )
 
@@ -77,6 +84,8 @@ def list_connection_role_assignments(connection: str | UUID) -> pd.DataFrame:
 
     This is a wrapper function for the following API: `Connections - List Connection Role Assignments <https://learn.microsoft.com/rest/api/fabric/core/connections/list-connection-role-assignments>`_.
 
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
     Parameters
     ----------
     connection : str | uuid.UUID
@@ -100,7 +109,9 @@ def list_connection_role_assignments(connection: str | UUID) -> pd.DataFrame:
     df = _create_dataframe(columns=columns)
 
     responses = _base_api(
-        request=f"/v1/connections/{connection_id}/roleAssignments", uses_pagination=True
+        request=f"/v1/connections/{connection_id}/roleAssignments",
+        client="fabric_sp",
+        uses_pagination=True,
     )
 
     for r in responses:
@@ -121,6 +132,8 @@ def list_connections() -> pd.DataFrame:
     """
     Lists all available connections.
 
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
     Returns
     -------
     pandas.DataFrame
@@ -136,13 +149,15 @@ def list_connections() -> pd.DataFrame:
         "Connection Type": "string",
         "Privacy Level": "string",
         "Credential Type": "string",
-        "Single Sign on Type": "string",
-        "Connection Encyrption": "string",
+        "Single Sign On Type": "string",
+        "Connection Encryption": "string",
         "Skip Test Connection": "bool",
     }
     df = _create_dataframe(columns=columns)
 
-    responses = _base_api(request="/v1/connections", uses_pagination=True)
+    responses = _base_api(
+        request="/v1/connections", client="fabric_sp", uses_pagination=True
+    )
 
     for r in responses:
         for i in r.get("value", []):
@@ -194,6 +209,8 @@ def list_item_connections(
 
     This is a wrapper function for the following API: `Items - List Item Connections <https://learn.microsoft.com/rest/api/fabric/core/items/list-item-connections>`_.
 
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
     Parameters
     ----------
     item_name : str
@@ -229,6 +246,7 @@ def list_item_connections(
 
     responses = _base_api(
         request=f"/v1/workspaces/{workspace_id}/items/{item_id}/connections",
+        client="fabric_sp",
         uses_pagination=True,
     )
 
@@ -267,7 +285,7 @@ def _list_supported_connection_types(
     df = _create_dataframe(columns=columns)
 
     url = url.rstrip("&")
-    responses = _base_api(request=url, uses_pagination=True)
+    responses = _base_api(request=url, client="fabric_sp", uses_pagination=True)
 
     records = []
     for r in responses:
@@ -309,6 +327,8 @@ def create_cloud_connection(
 
     This is a wrapper function for the following API: `Connections - Create Connection <https://learn.microsoft.com/rest/api/fabric/core/connections/create-connection>`_.
 
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
     Parameters
     ----------
     name : str
@@ -362,7 +382,11 @@ def create_cloud_connection(
     }
 
     _base_api(
-        request="/v1/connections", method="post", payload=payload, status_codes=201
+        request="/v1/connections",
+        client="fabric_sp",
+        method="post",
+        payload=payload,
+        status_codes=201,
     )
 
     print(f"{icons.green_dot} The '{name}' cloud connection has been created.")
@@ -383,6 +407,8 @@ def create_on_prem_connection(
 
     This is a wrapper function for the following API: `Connections - Create Connection <https://learn.microsoft.com/rest/api/fabric/core/connections/create-connection>`_.
 
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
     Parameters
     ----------
     name : str
@@ -440,7 +466,11 @@ def create_on_prem_connection(
     }
 
     _base_api(
-        request="/v1/connections", method="post", payload=payload, status_codes=201
+        request="/v1/connections",
+        client="fabric_sp",
+        method="post",
+        payload=payload,
+        status_codes=201,
     )
 
     print(f"{icons.green_dot} The '{name}' on-prem connection has been created.")
@@ -462,6 +492,8 @@ def create_vnet_connection(
 
     This is a wrapper function for the following API: `Connections - Create Connection <https://learn.microsoft.com/rest/api/fabric/core/connections/create-connection>`_.
 
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
     Parameters
     ----------
     name : str
@@ -520,7 +552,11 @@ def create_vnet_connection(
     }
 
     _base_api(
-        request="/v1/connections", method="post", payload=payload, status_codes=201
+        request="/v1/connections",
+        client="fabric_sp",
+        method="post",
+        payload=payload,
+        status_codes=201,
     )
 
     print(

{semantic_link_labs-0.9.2 → semantic_link_labs-0.9.3}/src/sempy_labs/_delta_analyzer.py

@@ -1,9 +1,8 @@
 import pandas as pd
 import datetime
-from typing import Dict
+from typing import Dict, Optional
 import pyarrow.dataset as ds
 import pyarrow.parquet as pq
-from pyspark.sql import SparkSession
 from sempy_labs._helper_functions import (
     create_abfss_path,
     save_as_delta_table,
@@ -12,19 +11,24 @@ from sempy_labs._helper_functions import (
     _update_dataframe_datatypes,
     resolve_workspace_name_and_id,
     resolve_lakehouse_name_and_id,
+    _read_delta_table,
+    _delta_table_row_count,
 )
 from sempy_labs.lakehouse._get_lakehouse_tables import get_lakehouse_tables
 from sempy_labs.lakehouse._lakehouse import lakehouse_attached
 import sempy_labs._icons as icons
+from uuid import UUID
 
 
 def delta_analyzer(
     table_name: str,
     approx_distinct_count: bool = True,
     export: bool = False,
+    lakehouse: Optional[str | UUID] = None,
+    workspace: Optional[str | UUID] = None,
 ) -> Dict[str, pd.DataFrame]:
     """
-    Analyzes a delta table and shows the results in dictionary containing a set of 5 dataframes. The table being analyzed must be in the lakehouse attached to the notebook.
+    Analyzes a delta table and shows the results in dictionary containing a set of 5 dataframes. If 'export' is set to True, the results will be saved to delta tables in the lakehouse attached to the notebook.
 
     The 5 dataframes returned by this function are:
 
@@ -44,26 +48,52 @@ def delta_analyzer(
         If True, uses approx_count_distinct to calculate the cardinality of each column. If False, uses COUNT(DISTINCT) instead.
     export : bool, default=False
         If True, exports the resulting dataframes to delta tables in the lakehouse attached to the notebook.
+    lakehouse : str | uuid.UUID, default=None
+        The Fabric lakehouse name or ID.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID used by the lakehouse.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
 
     Returns
     -------
     Dict[str, pandas.DataFrame]
         A dictionary of pandas dataframes showing semantic model objects which violated the best practice analyzer rules.
     """
+    import notebookutils
 
-    if not lakehouse_attached():
-        raise ValueError(
-            f"{icons.red_dot} No lakehouse is attached to this notebook. Please attach a lakehouse to the notebook before running the Delta Analyzer."
-        )
+    # display_toggle = notebookutils.common.configs.pandas_display
+
+    # Turn off notebookutils display
+    # if display_toggle is True:
+    #    notebookutils.common.configs.pandas_display = False
 
     prefix = "SLL_DeltaAnalyzer_"
     now = datetime.datetime.now()
-    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace=None)
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace=workspace)
     (lakehouse_name, lakehouse_id) = resolve_lakehouse_name_and_id(
-        lakehouse=None, workspace=None
+        lakehouse=lakehouse, workspace=workspace
     )
     path = create_abfss_path(lakehouse_id, workspace_id, table_name)
-    table_path = f"/lakehouse/default/Tables/{table_name}"
+    lake_path = create_abfss_path(lakehouse_id, workspace_id)
+    mounts = notebookutils.fs.mounts()
+    mount_point = f"/{workspace_name.replace(' ', '')}{lakehouse_name.replace(' ', '')}"
+    if not any(i.get("source") == lake_path for i in mounts):
+        # Mount lakehouse if not mounted
+        notebookutils.fs.mount(lake_path, mount_point)
+        print(
+            f"{icons.green_dot} Mounted the '{lakehouse_name}' lakehouse within the '{workspace_name}' to the notebook."
+        )
+
+    mounts = notebookutils.fs.mounts()
+    local_path = next(
+        i.get("localPath") for i in mounts if i.get("source") == lake_path
+    )
+    table_path = f"{local_path}/Tables/{table_name}"
+
+    # Set back to original value
+    # notebookutils.common.configs.pandas_display = display_toggle
 
     parquet_file_df_columns = {
         "ParquetFile": "string",
@@ -95,39 +125,21 @@ def delta_analyzer(
     row_group_df = _create_dataframe(columns=row_group_df_columns)
     column_chunk_df = _create_dataframe(columns=column_chunk_df_columns)
 
-    spark = SparkSession.builder.getOrCreate()
     # delta_table = DeltaTable.forPath(spark, path)
     # detail_df = spark.sql(f"DESCRIBE DETAIL `{table_name}`").collect()[0]
 
     # num_files = detail_df.numFiles
     # size_in_bytes = detail_df.sizeInBytes
 
-    latest_files = spark.read.format("delta").load(path).inputFiles()
+    latest_files = _read_delta_table(path).inputFiles()
     file_paths = [f.split("/")[-1] for f in latest_files]
-    row_count = spark.table(table_name).count()
+    row_count = _delta_table_row_count(table_name)
     row_groups = 0
     max_rows_per_row_group = 0
     min_rows_per_row_group = float("inf")
-    # dt = DeltaTable.forPath(spark, path)
-    # schema = dt.toDF().schema
-    # is_vorder = False
-    # if (
-    #    dt.detail()
-    #    .collect()[0]
-    #    .asDict()
-    #    .get("properties")
-    #    .get("delta.parquet.vorder.enabled")
-    #    == "true"
-    # ):
-    #    is_vorder = True
 
     schema = ds.dataset(table_path).schema.metadata
     is_vorder = any(b"vorder" in key for key in schema.keys())
-    # v_order_level = (
-    #    int(schema.get(b"com.microsoft.parquet.vorder.level").decode("utf-8"))
-    #    if is_vorder
-    #    else None
-    # )
 
     for file_name in file_paths:
         parquet_file = pq.ParquetFile(f"{table_path}/{file_name}")
@@ -235,14 +247,16 @@ def delta_analyzer(
                 table_name=table_name,
                 column_name=col_name,
                 function="approx",
-                lakehouse=lakehouse_name,
+                lakehouse=lakehouse,
+                workspace=workspace,
             )
         else:
             dc = _get_column_aggregate(
                 table_name=table_name,
                 column_name=col_name,
                 function="distinctcount",
-                lakehouse=lakehouse_name,
+                lakehouse=lakehouse,
+                workspace=workspace,
             )
 
         if "Cardinality" not in column_df.columns:
@@ -264,13 +278,17 @@ def delta_analyzer(
     save_table = f"{prefix}Summary"
 
     if export:
+        if not lakehouse_attached():
+            raise ValueError(
+                f"{icons.red_dot} No lakehouse is attached to this notebook. Please attach a lakehouse to the notebook before running the Delta Analyzer."
+            )
         dfL = get_lakehouse_tables()
         dfL_filt = dfL[dfL["Table Name"] == save_table]
         if dfL_filt.empty:
             runId = 1
         else:
             max_run_id = _get_column_aggregate(
-                lakehouse=lakehouse_name, table_name=save_table
+                table_name=save_table,
             )
             runId = max_run_id + 1