semantic-link-labs 0.9.5__py3-none-any.whl → 0.9.7__py3-none-any.whl

sempy_labs/directlake/_directlake_schema_sync.py

@@ -1,5 +1,5 @@
 import sempy
-import sempy.fabric as fabric
+import pandas as pd
 from sempy_labs.lakehouse import get_lakehouse_columns
 from sempy_labs.directlake._dl_helper import get_direct_lake_source
 from sempy_labs.tom import connect_semantic_model
@@ -19,8 +19,8 @@ def direct_lake_schema_sync(
     dataset: str | UUID,
     workspace: Optional[str | UUID] = None,
     add_to_model: bool = False,
-    **kwargs,
-):
+    remove_from_model: bool = False,
+) -> pd.DataFrame:
     """
     Shows/adds columns which exist in the lakehouse but do not exist in the semantic model (only for tables in the semantic model).
 
@@ -34,22 +34,18 @@ def direct_lake_schema_sync(
         or if no lakehouse attached, resolves to the workspace of the notebook.
     add_to_model : bool, default=False
         If set to True, columns which exist in the lakehouse but do not exist in the semantic model are added to the semantic model. No new tables are added.
+    remove_from_model : bool, default=False
+        If set to True, columns which exist in the semantic model but do not exist in the lakehouse are removed from the semantic model. No new tables are removed.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing the status of columns in the semantic model and lakehouse (prior to adding/removing them from the model using this function).
     """
 
     sempy.fabric._client._utils._init_analysis_services()
     import Microsoft.AnalysisServices.Tabular as TOM
 
-    if "lakehouse" in kwargs:
-        print(
-            "The 'lakehouse' parameter has been deprecated as it is no longer necessary. Please remove this parameter from the function going forward."
-        )
-        del kwargs["lakehouse"]
-    if "lakehouse_workspace" in kwargs:
-        print(
-            "The 'lakehouse_workspace' parameter has been deprecated as it is no longer necessary. Please remove this parameter from the function going forward."
-        )
-        del kwargs["lakehouse_workspace"]
-
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
     (dataset_name, dataset_id) = resolve_dataset_name_and_id(dataset, workspace_id)
 
@@ -67,14 +63,54 @@ def direct_lake_schema_sync(
             f"{icons.red_dot} This function only supports Direct Lake semantic models where the source lakehouse resides in the same workpace as the semantic model."
         )
 
-    lakehouse_workspace = fabric.resolve_workspace_name(lakehouse_workspace_id)
+    lc = get_lakehouse_columns(lakehouse_id, lakehouse_workspace_id)
 
-    lc = get_lakehouse_columns(lakehouse_name, lakehouse_workspace)
+    readonly = True
+    if add_to_model or remove_from_model:
+        readonly = False
+    df = pd.DataFrame(
+        columns=[
+            "TableName",
+            "ColumnName",
+            "SourceTableName",
+            "SourceColumnName",
+            "Status",
+        ]
+    )
 
     with connect_semantic_model(
-        dataset=dataset_id, readonly=False, workspace=workspace_id
+        dataset=dataset_id, readonly=readonly, workspace=workspace_id
     ) as tom:
+        # Check if the columns in the semantic model exist in the lakehouse
+        for c in tom.all_columns():
+            partition_name = next(p.Name for p in c.Table.Partitions)
+            p = c.Table.Partitions[partition_name]
+            if p.SourceType == TOM.PartitionSourceType.Entity:
+                entity_name = p.Source.EntityName
+                source_column = c.SourceColumn
+                lc_filt = lc[
+                    (lc["Table Name"] == entity_name)
+                    & (lc["Column Name"] == source_column)
+                ]
+                # Remove column from model if it doesn't exist in the lakehouse
+                if lc_filt.empty:
+                    new_data = {
+                        "TableName": c.Parent.Name,
+                        "ColumnName": c.Name,
+                        "SourceTableName": entity_name,
+                        "SourceColumnName": source_column,
+                        "Status": "Not in lakehouse",
+                    }
+                    df = pd.concat(
+                        [df, pd.DataFrame(new_data, index=[0])], ignore_index=True
+                    )
+                    if remove_from_model:
+                        tom.remove_object(object=c)
+                        print(
+                            f"{icons.green_dot} The '{c.Parent.Name}'[{c.Name}] column has been removed from the '{dataset_name}' semantic model within the '{workspace_name}' workspace."
+                        )
 
+        # Check if the lakehouse columns exist in the semantic model
         for i, r in lc.iterrows():
             lakeTName = r["Table Name"]
             lakeCName = r["Column Name"]
@@ -97,9 +133,17 @@ def direct_lake_schema_sync(
                     c.SourceColumn == lakeCName and c.Parent.Name == table_name
                     for c in tom.all_columns()
                 ):
-                    print(
-                        f"{icons.yellow_dot} The '{lakeCName}' column exists in the '{lakeTName}' lakehouse table but not in the '{dataset_name}' semantic model within the '{workspace_name}' workspace."
+                    new_data = {
+                        "TableName": table_name,
+                        "ColumnName": None,
+                        "SourceTableName": lakeTName,
+                        "SourceColumnName": lakeCName,
+                        "Status": "Not in semantic model",
+                    }
+                    df = pd.concat(
+                        [df, pd.DataFrame(new_data, index=[0])], ignore_index=True
                     )
+
                     if add_to_model:
                         dt = _convert_data_type(dType)
                         tom.add_data_column(
@@ -111,3 +155,5 @@ def direct_lake_schema_sync(
                         print(
                             f"{icons.green_dot} The '{lakeCName}' column in the '{lakeTName}' lakehouse table was added to the '{dataset_name}' semantic model within the '{workspace_name}' workspace."
                         )
+
+        return df

sempy_labs/directlake/_dl_helper.py

@@ -7,7 +7,6 @@ import sempy_labs._icons as icons
 from sempy._utils._log import log
 from sempy_labs._helper_functions import (
     retry,
-    resolve_lakehouse_name,
     _convert_data_type,
     resolve_dataset_name_and_id,
     resolve_workspace_name_and_id,
@@ -129,11 +128,6 @@ def generate_direct_lake_semantic_model(
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
     if lakehouse_workspace is None:
         lakehouse_workspace = workspace
-    if lakehouse is None:
-        lakehouse_id = fabric.get_lakehouse_id()
-        lakehouse_workspace_id = fabric.get_workspace_id()
-        lakehouse_workspace = fabric.resolve_workspace_name(lakehouse_workspace_id)
-        lakehouse = resolve_lakehouse_name(lakehouse_id, lakehouse_workspace)
 
     dfLT = get_lakehouse_tables(lakehouse=lakehouse, workspace=lakehouse_workspace)
 

sempy_labs/directlake/_generate_shared_expression.py

@@ -1,10 +1,8 @@
-import sempy.fabric as fabric
 from sempy_labs._helper_functions import (
-    resolve_lakehouse_name,
-    resolve_lakehouse_id,
-    resolve_warehouse_id,
     resolve_workspace_name_and_id,
     _base_api,
+    resolve_lakehouse_name_and_id,
+    resolve_item_name_and_id,
 )
 from typing import Optional
 import sempy_labs._icons as icons
@@ -15,6 +13,7 @@ def generate_shared_expression(
     item_name: Optional[str] = None,
     item_type: str = "Lakehouse",
     workspace: Optional[str | UUID] = None,
+    use_sql_endpoint: bool = True,
 ) -> str:
     """
     Dynamically generates the M expression used by a Direct Lake model for a given lakehouse/warehouse.
@@ -30,6 +29,9 @@ def generate_shared_expression(
         The Fabric workspace name or ID used by the item.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
+    use_sql_endpoint : bool, default=True
+        Whether to use the SQL Endpoint for the lakehouse/warehouse.
+        If False, the expression will be generated without using the SQL Endpoint.
 
     Returns
     -------
@@ -45,13 +47,14 @@ def generate_shared_expression(
             f"{icons.red_dot} Invalid item type. Valid options: {item_types}."
         )
 
-    if item_name is None:
-        item_id = fabric.get_lakehouse_id()
-        item_name = resolve_lakehouse_name(item_id, workspace_id)
-    elif item_name is not None and item_type == "Lakehouse":
-        item_id = resolve_lakehouse_id(lakehouse=item_name, workspace=workspace_id)
-    elif item_type == "Warehouse":
-        item_id = resolve_warehouse_id(warehouse=item_name, workspace=workspace_id)
+    if item_type == "Lakehouse":
+        (item_name, item_id) = resolve_lakehouse_name_and_id(
+            lakehouse=item_name, workspace=workspace_id
+        )
+    else:
+        (item_name, item_id) = resolve_item_name_and_id(
+            item=item_name, type=item_type, workspace=workspace_id
+        )
 
     item_type_rest = f"{item_type.lower()}s"
     response = _base_api(
@@ -79,4 +82,8 @@ def generate_shared_expression(
     end_expr = "\nin\n\tdatabase"
     mid_expr = f'Sql.Database("{sqlEPCS}", "{sqlepid}")'
 
-    return f"{start_expr}{mid_expr}{end_expr}"
+    # Build DL/OL expression
+    if not use_sql_endpoint and item_type == "Lakehouse":
+        return f'AzureDataLakeStorage{{"server":"onelake.dfs.fabric.microsoft.com","path":"/{workspace_id}/{item_id}/"}}'
+    else:
+        return f"{start_expr}{mid_expr}{end_expr}"

sempy_labs/directlake/_guardrails.py

@@ -48,6 +48,7 @@ def get_sku_size(workspace: Optional[str | UUID] = None) -> str:
     str
         The SKU size for a workspace.
     """
+    from sempy_labs._capacities import list_capacities
 
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
 
@@ -59,7 +60,7 @@ def get_sku_size(workspace: Optional[str | UUID] = None) -> str:
         )
 
     capacity_id = dfW["Capacity Id"].iloc[0]
-    dfC = fabric.list_capacities()
+    dfC = list_capacities()
     dfC_filt = dfC[dfC["Id"] == capacity_id]
 
     if len(dfC_filt) == 0:

sempy_labs/directlake/_update_directlake_model_lakehouse_connection.py

@@ -1,16 +1,64 @@
-import sempy.fabric as fabric
 from sempy_labs.directlake._generate_shared_expression import generate_shared_expression
 from sempy_labs._helper_functions import (
-    resolve_lakehouse_name,
     resolve_dataset_name_and_id,
     resolve_workspace_name_and_id,
+    resolve_item_name_and_id,
+    resolve_lakehouse_name_and_id,
 )
+from sempy._utils._log import log
 from sempy_labs.tom import connect_semantic_model
 from typing import Optional
 import sempy_labs._icons as icons
 from uuid import UUID
+import re
 
 
+def _extract_expression_list(expression):
+    """
+    Finds the pattern for DL/SQL & DL/OL expressions in the semantic model.
+    """
+
+    pattern_sql = r'Sql\.Database\s*\(\s*"([^"]+)"\s*,\s*"([^"]+)"\s*\)'
+    pattern_no_sql = r'AzureDataLakeStorage\s*\{\s*"server".*?:\s*onelake\.dfs\.fabric\.microsoft\.com"\s*,\s*"path"\s*:\s*"/([\da-fA-F-]+)\s*/\s*([\da-fA-F-]+)\s*/"\s*\}'
+
+    match_sql = re.search(pattern_sql, expression)
+    match_no_sql = re.search(pattern_no_sql, expression)
+
+    result = []
+    if match_sql:
+        value_1, value_2 = match_sql.groups()
+        result = [value_1, value_2, True]
+    elif match_no_sql:
+        value_1, value_2 = match_no_sql.groups()
+        result = [value_1, value_2, False]
+
+    return result
+
+
+def _get_direct_lake_expressions(
+    dataset: str | UUID, workspace: Optional[str | UUID] = None
+) -> dict:
+    """
+    Extracts a dictionary of all Direct Lake expressions from a semantic model.
+    """
+
+    from sempy_labs.tom import connect_semantic_model
+
+    result = {}
+
+    with connect_semantic_model(dataset=dataset, workspace=workspace) as tom:
+        for e in tom.model.Expressions:
+            expr_name = e.Name
+            expr = e.Expression
+
+            list_values = _extract_expression_list(expr)
+            if list_values:
+                result[expr_name] = list_values
+
+    return result
+
+
+@log
 def update_direct_lake_model_lakehouse_connection(
     dataset: str | UUID,
     workspace: Optional[str | UUID] = None,
@@ -37,54 +85,23 @@ def update_direct_lake_model_lakehouse_connection(
         or if no lakehouse attached, resolves to the workspace of the notebook.
     """
 
-    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
-    (dataset_name, dataset_id) = resolve_dataset_name_and_id(dataset, workspace_id)
-
-    if lakehouse_workspace is None:
-        lakehouse_workspace = workspace_name
-
-    if lakehouse is None:
-        lakehouse_id = fabric.get_lakehouse_id()
-        lakehouse = resolve_lakehouse_name(lakehouse_id, lakehouse_workspace)
-
-    # Check if lakehouse is valid
-    dfI = fabric.list_items(workspace=lakehouse_workspace, type="Lakehouse")
-    dfI_filt = dfI[(dfI["Display Name"] == lakehouse)]
-
-    if len(dfI_filt) == 0:
-        raise ValueError(
-            f"{icons.red_dot} The '{lakehouse}' lakehouse does not exist within the '{lakehouse_workspace}' workspace. "
-            f"Therefore it cannot be used to support the '{dataset_name}' semantic model within the '{workspace_name}' workspace."
-        )
-
-    icons.sll_tags.append("UpdateDLConnection")
-
-    shEx = generate_shared_expression(
-        item_name=lakehouse, item_type="Lakehouse", workspace=lakehouse_workspace
-    )
-
-    with connect_semantic_model(
-        dataset=dataset_id, readonly=False, workspace=workspace_id
-    ) as tom:
-
-        if not tom.is_direct_lake():
-            raise ValueError(
-                f"{icons.red_dot} The '{dataset_name}' semantic model is not in Direct Lake. This function is only applicable to Direct Lake semantic models."
-            )
-
-        tom.model.Expressions["DatabaseQuery"].Expression = shEx
-
-    print(
-        f"{icons.green_dot} The expression in the '{dataset_name}' semantic model has been updated to point to the '{lakehouse}' lakehouse in the '{lakehouse_workspace}' workspace."
+    update_direct_lake_model_connection(
+        dataset=dataset,
+        workspace=workspace,
+        source=lakehouse,
+        source_type="Lakehouse",
+        source_workspace=lakehouse_workspace,
     )
 
 
+@log
 def update_direct_lake_model_connection(
     dataset: str | UUID,
     workspace: Optional[str | UUID] = None,
     source: Optional[str] = None,
     source_type: str = "Lakehouse",
     source_workspace: Optional[str | UUID] = None,
+    use_sql_endpoint: bool = True,
 ):
     """
     Remaps a Direct Lake semantic model's SQL Endpoint connection to a new lakehouse/warehouse.
@@ -106,7 +123,14 @@ def update_direct_lake_model_connection(
         The Fabric workspace name or ID used by the lakehouse/warehouse.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
+    use_sql_endpoint : bool, default=True
+        If True, the SQL Endpoint will be used for the connection.
+        If False, Direct Lake over OneLake will be used.
     """
+    if use_sql_endpoint:
+        icons.sll_tags.append("UpdateDLConnection_SQL")
+    else:
+        icons.sll_tags.append("UpdateDLConnection_DLOL")
 
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
     (dataset_name, dataset_id) = resolve_dataset_name_and_id(dataset, workspace_id)
@@ -121,23 +145,25 @@ def update_direct_lake_model_connection(
     if source_workspace is None:
         source_workspace = workspace_name
 
-    if source is None:
-        source_id = fabric.get_lakehouse_id()
-        source = resolve_lakehouse_name(source_id, source_workspace)
-    else:
-        source_id = fabric.resolve_item_id(
-            item_name=source, type=source_type, workspace=source_workspace
+    if source_type == "Lakehouse":
+        (source_name, source_id) = resolve_lakehouse_name_and_id(
+            lakehouse=source, workspace=source_workspace
         )
-        source = fabric.resolve_item_name(
-            item_id=source_id, workspace=source_workspace, type=source_type
+    else:
+        (source_name, source_id) = resolve_item_name_and_id(
+            item=source, type=source_type, workspace=source_workspace
         )
 
-    icons.sll_tags.append("UpdateDLConnection")
-
-    shEx = generate_shared_expression(
-        item_name=source, item_type=source_type, workspace=source_workspace
+    shared_expression = generate_shared_expression(
+        item_name=source_name,
+        item_type=source_type,
+        workspace=source_workspace,
+        use_sql_endpoint=use_sql_endpoint,
     )
 
+    expression_dict = _get_direct_lake_expressions(dataset=dataset, workspace=workspace)
+    expressions = list(expression_dict.keys())
+
     with connect_semantic_model(
         dataset=dataset_id, readonly=False, workspace=workspace_id
     ) as tom:
@@ -147,8 +173,15 @@ def update_direct_lake_model_connection(
                 f"{icons.red_dot} The '{dataset_name}' semantic model within the '{workspace_name}' workspace is not in Direct Lake. This function is only applicable to Direct Lake semantic models."
             )
 
-        tom.model.Expressions["DatabaseQuery"].Expression = shEx
+        # Update the single connection expression
+        if len(expressions) == 1:
+            expr = expressions[0]
+            tom.model.Expressions[expr].Expression = shared_expression
 
-    print(
-        f"{icons.green_dot} The expression in the '{dataset_name}' semantic model within the '{workspace_name}' workspace has been updated to point to the '{source}' {source_type.lower()} in the '{source_workspace}' workspace."
-    )
+            print(
+                f"{icons.green_dot} The expression in the '{dataset_name}' semantic model within the '{workspace_name}' workspace has been updated to point to the '{source}' {source_type.lower()} in the '{source_workspace}' workspace."
+            )
+        else:
+            print(
+                f"{icons.info} Multiple expressions found in the model. Please use the update_direct_lake_partition_entity function to update specific tables."
+            )

sempy_labs/directlake/_update_directlake_partition_entity.py

@@ -1,5 +1,4 @@
 import sempy
-import sempy.fabric as fabric
 from sempy_labs.tom import connect_semantic_model
 from sempy_labs._refresh_semantic_model import refresh_semantic_model
 from sempy_labs.directlake._dl_helper import get_direct_lake_source
@@ -7,12 +6,15 @@ from sempy_labs._helper_functions import (
     _convert_data_type,
     resolve_dataset_name_and_id,
     resolve_workspace_name_and_id,
+    resolve_workspace_name,
 )
+from sempy._utils._log import log
 from typing import List, Optional, Union
 import sempy_labs._icons as icons
 from uuid import UUID
 
 
+@log
 def update_direct_lake_partition_entity(
     dataset: str | UUID,
     table_name: Union[str, List[str]],
@@ -96,6 +98,7 @@ def update_direct_lake_partition_entity(
             )
 
 
+@log
 def add_table_to_direct_lake_semantic_model(
     dataset: str | UUID,
     table_name: str,
@@ -144,7 +147,7 @@ def add_table_to_direct_lake_semantic_model(
             f"{icons.red_dot} This function only supports Direct Lake semantic models where the source lakehouse resides in the same workpace as the semantic model."
         )
 
-    lakehouse_workspace = fabric.resolve_workspace_name(lakehouse_workspace_id)
+    lakehouse_workspace = resolve_workspace_name(workspace_id=lakehouse_workspace_id)
 
     with connect_semantic_model(
         dataset=dataset_id, readonly=False, workspace=workspace_id

sempy_labs/graph/_groups.py

@@ -6,6 +6,7 @@ from sempy_labs._helper_functions import (
     _create_dataframe,
     _update_dataframe_datatypes,
 )
+from sempy._utils._log import log
 import sempy_labs._icons as icons
 from typing import List, Literal
 
@@ -38,6 +39,7 @@ def resolve_group_id(group: str | UUID) -> UUID:
     return group_id
 
 
+@log
 def list_groups() -> pd.DataFrame:
     """
     Shows a list of groups and their properties.
@@ -158,6 +160,7 @@ def _get_group(group_id: UUID) -> pd.DataFrame:
     return df
 
 
+@log
 def list_group_members(group: str | UUID) -> pd.DataFrame:
     """
     Shows a list of the members of a group.
@@ -217,6 +220,7 @@ def list_group_members(group: str | UUID) -> pd.DataFrame:
     return df
 
 
+@log
 def list_group_owners(group: str | UUID) -> pd.DataFrame:
     """
     Shows a list of the owners of a group.
@@ -332,6 +336,7 @@ def _base_add_to_group(
     )
 
 
+@log
 def add_group_members(
     group: str | UUID,
     user: str | UUID | List[str | UUID],
@@ -376,6 +381,7 @@ def add_group_owners(
     _base_add_to_group(group=group, object=user, object_type="owners")
 
 
+@log
 def renew_group(group: str | UUID):
     """
     Renews the group.

sempy_labs/graph/_teams.py

@@ -1,5 +1,6 @@
 import pandas as pd
 from uuid import UUID
+from sempy._utils._log import log
 from sempy_labs._helper_functions import (
     _base_api,
     _create_dataframe,
@@ -7,6 +8,7 @@ from sempy_labs._helper_functions import (
 )
 
 
+@log
 def list_teams() -> pd.DataFrame:
     """
     Shows a list of teams and their properties.

sempy_labs/graph/_users.py

@@ -7,6 +7,7 @@ from sempy_labs._helper_functions import (
     _base_api,
     _create_dataframe,
 )
+from sempy._utils._log import log
 
 
 def resolve_user_id(user: str | UUID) -> UUID:
@@ -33,6 +34,7 @@ def resolve_user_id(user: str | UUID) -> UUID:
         return result.get("id")
 
 
+@log
 def get_user(user: str | UUID) -> pd.DataFrame:
     """
     Shows properties of a given user.
@@ -70,6 +72,7 @@ def get_user(user: str | UUID) -> pd.DataFrame:
     return pd.DataFrame([new_data])
 
 
+@log
 def list_users() -> pd.DataFrame:
     """
     Shows a list of users and their properties.
@@ -120,6 +123,7 @@ def list_users() -> pd.DataFrame:
     return df
 
 
+@log
 def send_mail(
     user: UUID | str,
     subject: str,

sempy_labs/lakehouse/__init__.py

@@ -1,12 +1,15 @@
-from sempy_labs.lakehouse._get_lakehouse_columns import get_lakehouse_columns
-from sempy_labs.lakehouse._get_lakehouse_tables import get_lakehouse_tables
+from sempy_labs.lakehouse._get_lakehouse_columns import (
+    get_lakehouse_columns,
+)
+from sempy_labs.lakehouse._get_lakehouse_tables import (
+    get_lakehouse_tables,
+)
 from sempy_labs.lakehouse._lakehouse import (
     lakehouse_attached,
     optimize_lakehouse_tables,
     vacuum_lakehouse_tables,
     run_table_maintenance,
 )
-
 from sempy_labs.lakehouse._shortcuts import (
     # create_shortcut,
     create_shortcut_onelake,
@@ -14,6 +17,10 @@ from sempy_labs.lakehouse._shortcuts import (
     reset_shortcut_cache,
     list_shortcuts,
 )
+from sempy_labs.lakehouse._blobs import (
+    recover_lakehouse_object,
+    list_blobs,
+)
 
 __all__ = [
     "get_lakehouse_columns",
@@ -27,4 +34,6 @@ __all__ = [
     "reset_shortcut_cache",
     "run_table_maintenance",
     "list_shortcuts",
+    "recover_lakehouse_object",
+    "list_blobs",
 ]