semantic-link-labs 0.5.0__py3-none-any.whl → 0.7.0__py3-none-any.whl

sempy_labs/directlake/_update_directlake_partition_entity.py

@@ -1,17 +1,18 @@
 import sempy
 import sempy.fabric as fabric
 from sempy_labs.tom import connect_semantic_model
-from sempy_labs._helper_functions import resolve_lakehouse_name
+from sempy_labs._refresh_semantic_model import refresh_semantic_model
+from sempy_labs.directlake._dl_helper import get_direct_lake_source
 from typing import List, Optional, Union
 import sempy_labs._icons as icons
 
+
 def update_direct_lake_partition_entity(
     dataset: str,
     table_name: Union[str, List[str]],
     entity_name: Union[str, List[str]],
     workspace: Optional[str] = None,
-    lakehouse: Optional[str] = None,
-    lakehouse_workspace: Optional[str] = None
+    **kwargs,
 ):
     """
     Remaps a table (or tables) in a Direct Lake semantic model to a table in a lakehouse.
@@ -28,23 +29,30 @@ def update_direct_lake_partition_entity(
         The Fabric workspace name in which the semantic model exists.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
-    lakehouse : str, default=None
-        The Fabric lakehouse used by the Direct Lake semantic model.
-        Defaults to None which resolves to the lakehouse attached to the notebook.
-    lakehouse_workspace : str, default=None
-        The Fabric workspace used by the lakehouse.
-        Defaults to None which resolves to the workspace of the attached lakehouse
-        or if no lakehouse attached, resolves to the workspace of the notebook.
     """
 
+    if "lakehouse" in kwargs:
+        print(
+            "The 'lakehouse' parameter has been deprecated as it is no longer necessary. Please remove this parameter from the function going forward."
+        )
+        del kwargs["lakehouse"]
+    if "lakehouse_workspace" in kwargs:
+        print(
+            "The 'lakehouse_workspace' parameter has been deprecated as it is no longer necessary. Please remove this parameter from the function going forward."
+        )
+        del kwargs["lakehouse_workspace"]
+
     workspace = fabric.resolve_workspace_name(workspace)
 
-    if lakehouse_workspace is None:
-        lakehouse_workspace = workspace
+    artifact_type, lakehouse_name, lakehouse_id, lakehouse_workspace_id = (
+        get_direct_lake_source(dataset=dataset, workspace=workspace)
+    )
 
-    if lakehouse is None:
-        lakehouse_id = fabric.get_lakehouse_id()
-        lakehouse = resolve_lakehouse_name(lakehouse_id, lakehouse_workspace)
+    if artifact_type == "Warehouse":
+        raise ValueError(
+            f"{icons.red_dot} This function is only valid for Direct Lake semantic models which source from lakehouses, not warehouses."
+        )
+    lakehouse_workspace = fabric.resolve_workspace_name(lakehouse_workspace_id)
 
     # Support both str & list types
     if isinstance(table_name, str):
@@ -53,22 +61,161 @@ def update_direct_lake_partition_entity(
         entity_name = [entity_name]
 
     if len(table_name) != len(entity_name):
-        raise ValueError(f"{icons.red_dot} The 'table_name' and 'entity_name' arrays must be of equal length.")
+        raise ValueError(
+            f"{icons.red_dot} The 'table_name' and 'entity_name' arrays must be of equal length."
+        )
 
     with connect_semantic_model(
         dataset=dataset, readonly=False, workspace=workspace
     ) as tom:
 
         if not tom.is_direct_lake():
-            raise ValueError(f"{icons.red_dot} The '{dataset}' semantic model within the '{workspace}' workspace is not in Direct Lake mode.")
+            raise ValueError(
+                f"{icons.red_dot} The '{dataset}' semantic model within the '{workspace}' workspace is not in Direct Lake mode."
+            )
 
         for tName in table_name:
             i = table_name.index(tName)
             eName = entity_name[i]
-            try:
-                tom.model.Tables[tName].Partitions[0].EntityName = eName
+            part_name = next(
+                p.Name
+                for t in tom.model.Tables
+                for p in t.Partitions
+                if t.Name == tName
+            )
+
+            if part_name is None:
+                raise ValueError(
+                    f"{icons.red_dot} The '{tName}' table in the '{dataset}' semantic model has not been updated."
+                )
+            else:
+                tom.model.Tables[tName].Partitions[part_name].EntityName = eName
                 print(
-                    f"{icons.green_dot} The '{tName}' table in the '{dataset}' semantic model has been updated to point to the '{eName}' table in the '{lakehouse}' lakehouse within the '{lakehouse_workspace}' workspace."
+                    f"{icons.green_dot} The '{tName}' table in the '{dataset}' semantic model has been updated to point to the '{eName}' table "
+                    f"in the '{lakehouse_name}' lakehouse within the '{lakehouse_workspace}' workspace."
                 )
-            except Exception as e:
-                raise ValueError(f"{icons.red_dot} The '{tName}' table in the '{dataset}' semantic model has not been updated.") from e
+
+
+def add_table_to_direct_lake_semantic_model(
+    dataset: str,
+    table_name: str,
+    lakehouse_table_name: str,
+    refresh: Optional[bool] = True,
+    workspace: Optional[str] = None,
+):
+    """
+    Adds a table and all of its columns to a Direct Lake semantic model, based on a Fabric lakehouse table.
+
+    Parameters
+    ----------
+    dataset : str
+        Name of the semantic model.
+    table_name : str, List[str]
+        Name of the table in the semantic model.
+    lakehouse_table_name : str
+        The name of the Fabric lakehouse table.
+    refresh : bool, default=True
+        Refreshes the table after it is added to the semantic model.
+    workspace : str, default=None
+        The name of the Fabric workspace in which the semantic model resides.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    """
+
+    sempy.fabric._client._utils._init_analysis_services()
+    import Microsoft.AnalysisServices.Tabular as TOM
+    from sempy_labs.lakehouse._get_lakehouse_columns import get_lakehouse_columns
+    from sempy_labs.lakehouse._get_lakehouse_tables import get_lakehouse_tables
+
+    workspace = fabric.resolve_workspace_name(workspace)
+
+    artifact_type, lakehouse_name, lakehouse_id, lakehouse_workspace_id = (
+        get_direct_lake_source(dataset=dataset, workspace=workspace)
+    )
+
+    if artifact_type == "Warehouse":
+        raise ValueError(
+            f"{icons.red_dot} This function is only valid for Direct Lake semantic models which source from Fabric lakehouses (not warehouses)."
+        )
+
+    lakehouse_workspace = fabric.resolve_workspace_name(lakehouse_workspace_id)
+
+    with connect_semantic_model(
+        dataset=dataset, readonly=False, workspace=workspace
+    ) as tom:
+
+        table_count = tom.model.Tables.Count
+
+        if tom.is_direct_lake() is False and table_count > 0:
+            raise ValueError(
+                "This function is only valid for Direct Lake semantic models or semantic models with no tables."
+            )
+
+        if any(
+            p.Name == lakehouse_table_name
+            for p in tom.all_partitions()
+            if p.SourceType == TOM.PartitionSourceType.Entity
+        ):
+            t_name = next(
+                p.Parent.Name
+                for p in tom.all_partitions()
+                if p.Name
+                == lakehouse_table_name & p.SourceType
+                == TOM.PartitionSourceType.Entity
+            )
+            raise ValueError(
+                f"The '{lakehouse_table_name}' table already exists in the '{dataset}' semantic model within the '{workspace}' workspace as the '{t_name}' table."
+            )
+
+        if any(t.Name == table_name for t in tom.model.Tables):
+            raise ValueError(
+                f"The '{table_name}' table already exists in the '{dataset}' semantic model within the '{workspace}' workspace."
+            )
+
+        dfL = get_lakehouse_tables(
+            lakehouse=lakehouse_name, workspace=lakehouse_workspace
+        )
+        dfL_filt = dfL[dfL["Table Name"] == lakehouse_table_name]
+
+        if len(dfL_filt) == 0:
+            raise ValueError(
+                f"The '{lakehouse_table_name}' table does not exist in the '{lakehouse_name}' lakehouse within the '{lakehouse_workspace}' workspace."
+            )
+
+        dfLC = get_lakehouse_columns(
+            lakehouse=lakehouse_name, workspace=lakehouse_workspace
+        )
+        dfLC_filt = dfLC[dfLC["Table Name"] == lakehouse_table_name]
+
+        tom.add_table(name=table_name)
+        print(
+            f"{icons.green_dot} The '{table_name}' table has been added to the '{dataset}' semantic model within the '{workspace}' workspace."
+        )
+        tom.add_entity_partition(
+            table_name=table_name, entity_name=lakehouse_table_name
+        )
+        print(
+            f"{icons.green_dot} The '{lakehouse_table_name}' partition has been added to the '{table_name}' table in the '{dataset}' semantic model within the '{workspace}' workspace."
+        )
+
+        for i, r in dfLC_filt.iterrows():
+            lakeCName = r["Column Name"]
+            dType = r["Data Type"]
+            dt = icons.data_type_mapping.get(dType)
+            tom.add_data_column(
+                table_name=table_name,
+                column_name=lakeCName,
+                source_column=lakeCName,
+                data_type=dt,
+            )
+            print(
+                f"{icons.green_dot} The '{lakeCName}' column has been added to the '{table_name}' table as a '{dt}' data type in the '{dataset}' semantic model within the '{workspace}' workspace."
+            )
+
+        if refresh:
+            refresh_semantic_model(
+                dataset=dataset, tables=table_name, workspace=workspace
+            )

sempy_labs/directlake/_warm_cache.py

@@ -1,4 +1,3 @@
-import sempy
 import sempy.fabric as fabric
 import pandas as pd
 from tqdm.auto import tqdm
@@ -18,7 +17,7 @@ def warm_direct_lake_cache_perspective(
     perspective: str,
     add_dependencies: Optional[bool] = False,
     workspace: Optional[str] = None,
-):
+) -> pd.DataFrame:
     """
     Warms the cache of a Direct Lake semantic model by running a simple DAX query against the columns in a perspective.
 
@@ -45,7 +44,9 @@ def warm_direct_lake_cache_perspective(
 
     dfP = fabric.list_partitions(dataset=dataset, workspace=workspace)
     if not any(r["Mode"] == "DirectLake" for i, r in dfP.iterrows()):
-        raise ValueError(f"{icons.red_dot} The '{dataset}' semantic model in the '{workspace}' workspace is not in Direct Lake mode. This function is specifically for semantic models in Direct Lake mode.")
+        raise ValueError(
+            f"{icons.red_dot} The '{dataset}' semantic model in the '{workspace}' workspace is not in Direct Lake mode. This function is specifically for semantic models in Direct Lake mode."
+        )
 
     dfPersp = fabric.list_perspectives(dataset=dataset, workspace=workspace)
     dfPersp["DAX Object Name"] = format_dax_object_name(
@@ -54,7 +55,9 @@ def warm_direct_lake_cache_perspective(
     dfPersp_filt = dfPersp[dfPersp["Perspective Name"] == perspective]
 
     if len(dfPersp_filt) == 0:
-        raise ValueError(f"{icons.red_dot} The '{perspective} perspective does not exist or contains no objects within the '{dataset}' semantic model in the '{workspace}' workspace.")
+        raise ValueError(
+            f"{icons.red_dot} The '{perspective} perspective does not exist or contains no objects within the '{dataset}' semantic model in the '{workspace}' workspace."
+        )
 
     dfPersp_c = dfPersp_filt[dfPersp_filt["Object Type"] == "Column"]
 

sempy_labs/lakehouse/_get_lakehouse_columns.py

@@ -1,4 +1,3 @@
-import sempy
 import sempy.fabric as fabric
 import pandas as pd
 from pyspark.sql import SparkSession
@@ -10,6 +9,7 @@ from sempy_labs._helper_functions import (
 from typing import Optional
 from sempy._utils._log import log
 
+
 @log
 def get_lakehouse_columns(
     lakehouse: Optional[str] = None, workspace: Optional[str] = None

sempy_labs/lakehouse/_get_lakehouse_tables.py

@@ -1,4 +1,3 @@
-import sempy
 import sempy.fabric as fabric
 import pandas as pd
 from pyspark.sql import SparkSession
@@ -17,6 +16,8 @@ from sempy_labs.lakehouse._lakehouse import lakehouse_attached
 from typing import Optional
 import sempy_labs._icons as icons
 from sempy._utils._log import log
+from sempy.fabric.exceptions import FabricHTTPException
+
 
 @log
 def get_lakehouse_tables(
@@ -51,6 +52,8 @@ def get_lakehouse_tables(
         Shows the tables/columns within a lakehouse and their properties.
     """
 
+    from sempy_labs._helper_functions import pagination
+
     df = pd.DataFrame(
         columns=[
             "Workspace Name",
@@ -73,34 +76,52 @@ def get_lakehouse_tables(
     if count_rows:  # Setting countrows defaults to extended=True
         extended = True
 
+    if (
+        workspace_id != fabric.get_workspace_id()
+        and lakehouse_id != fabric.get_lakehouse_id()
+        and count_rows
+    ):
+        raise ValueError(
+            f"{icons.red_dot} If 'count_rows' is set to True, you must run this function against the default lakehouse attached to the notebook. "
+            "Count rows runs a spark query and cross-workspace spark queries are currently not supported."
+        )
+
     client = fabric.FabricRestClient()
     response = client.get(
         f"/v1/workspaces/{workspace_id}/lakehouses/{lakehouse_id}/tables"
     )
 
-    for i in response.json()["data"]:
-        tName = i["name"]
-        tType = i["type"]
-        tFormat = i["format"]
-        tLocation = i["location"]
-        if not extended:
+    if response.status_code != 200:
+        raise FabricHTTPException(response)
+
+    responses = pagination(client, response)
+
+    dfs = []
+    for r in responses:
+        for i in r.get("data", []):
             new_data = {
                 "Workspace Name": workspace,
                 "Lakehouse Name": lakehouse,
-                "Table Name": tName,
-                "Format": tFormat,
-                "Type": tType,
-                "Location": tLocation,
+                "Table Name": i.get("name"),
+                "Format": i.get("format"),
+                "Type": i.get("type"),
+                "Location": i.get("location"),
             }
-            df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
-        else:
-            sku_value = get_sku_size(workspace)
-            guardrail = get_directlake_guardrails_for_sku(sku_value)
-
-            spark = SparkSession.builder.getOrCreate()
+            dfs.append(pd.DataFrame(new_data, index=[0]))
+    df = pd.concat(dfs, ignore_index=True)
 
-            intColumns = ["Files", "Row Groups", "Table Size"]
-            if tType == "Managed" and tFormat == "delta":
+    if extended:
+        sku_value = get_sku_size(workspace)
+        guardrail = get_directlake_guardrails_for_sku(sku_value)
+        spark = SparkSession.builder.getOrCreate()
+        df["Files"] = None
+        df["Row Groups"] = None
+        df["Table Size"] = None
+        if count_rows:
+            df["Row Count"] = None
+        for i, r in df.iterrows():
+            tName = r["Table Name"]
+            if r["Type"] == "Managed" and r["Format"] == "delta":
                 detail_df = spark.sql(f"DESCRIBE DETAIL `{tName}`").collect()[0]
                 num_files = detail_df.numFiles
                 size_in_bytes = detail_df.sizeInBytes
@@ -120,62 +141,35 @@ def get_lakehouse_tables(
                         ).num_row_groups
                     except FileNotFoundError:
                         continue
+                df.at[i, "Files"] = num_files
+                df.at[i, "Row Groups"] = num_rowgroups
+                df.at[i, "Table Size"] = size_in_bytes
+            if count_rows:
+                num_rows = spark.table(tName).count()
+                df.at[i, "Row Count"] = num_rows
+
+    if extended:
+        intColumns = ["Files", "Row Groups", "Table Size"]
+        df[intColumns] = df[intColumns].astype(int)
+        df["SKU"] = guardrail["Fabric SKUs"].iloc[0]
+        df["Parquet File Guardrail"] = guardrail["Parquet files per table"].iloc[0]
+        df["Row Group Guardrail"] = guardrail["Row groups per table"].iloc[0]
+        df["Row Count Guardrail"] = (
+            guardrail["Rows per table (millions)"].iloc[0] * 1000000
+        )
 
-                if count_rows:
-                    num_rows = spark.table(tName).count()
-                    intColumns.append("Row Count")
-                    new_data = {
-                        "Workspace Name": workspace,
-                        "Lakehouse Name": lakehouse,
-                        "Table Name": tName,
-                        "Format": tFormat,
-                        "Type": tType,
-                        "Location": tLocation,
-                        "Files": num_files,
-                        "Row Groups": num_rowgroups,
-                        "Row Count": num_rows,
-                        "Table Size": size_in_bytes,
-                    }
-                else:
-                    new_data = {
-                        "Workspace Name": workspace,
-                        "Lakehouse Name": lakehouse,
-                        "Table Name": tName,
-                        "Format": tFormat,
-                        "Type": tType,
-                        "Location": tLocation,
-                        "Files": num_files,
-                        "Row Groups": num_rowgroups,
-                        "Table Size": size_in_bytes,
-                    }
-
-                df = pd.concat(
-                    [df, pd.DataFrame(new_data, index=[0])], ignore_index=True
-                )
-                df[intColumns] = df[intColumns].astype(int)
-
-            df["SKU"] = guardrail["Fabric SKUs"].iloc[0]
-            df["Parquet File Guardrail"] = guardrail["Parquet files per table"].iloc[0]
-            df["Row Group Guardrail"] = guardrail["Row groups per table"].iloc[0]
-            df["Row Count Guardrail"] = (
-                guardrail["Rows per table (millions)"].iloc[0] * 1000000
-            )
+        df["Parquet File Guardrail Hit"] = df["Files"] > df["Parquet File Guardrail"]
+        df["Row Group Guardrail Hit"] = df["Row Groups"] > df["Row Group Guardrail"]
+    if count_rows:
+        df["Row Count"] = df["Row Count"].astype(int)
+        df["Row Count Guardrail Hit"] = df["Row Count"] > df["Row Count Guardrail"]
 
-            df["Parquet File Guardrail Hit"] = (
-                df["Files"] > df["Parquet File Guardrail"]
+    if export:
+        if not lakehouse_attached():
+            raise ValueError(
+                f"{icons.red_dot} In order to save the report.json file, a lakehouse must be attached to the notebook. Please attach a lakehouse to this notebook."
             )
-            df["Row Group Guardrail Hit"] = df["Row Groups"] > df["Row Group Guardrail"]
 
-            if count_rows:
-                df["Row Count Guardrail Hit"] = (
-                    df["Row Count"] > df["Row Count Guardrail"]
-                )
-
-    if export:
-        lakeAttach = lakehouse_attached()
-        if lakeAttach is False:
-            raise ValueError(f"{icons.red_dot} In order to save the report.json file, a lakehouse must be attached to the notebook. Please attach a lakehouse to this notebook.")
-            
         spark = SparkSession.builder.getOrCreate()
 
         lakehouse_id = fabric.get_lakehouse_id()

sempy_labs/lakehouse/_lakehouse.py

@@ -1,4 +1,3 @@
-import sempy
 import sempy.fabric as fabric
 from tqdm.auto import tqdm
 from pyspark.sql import SparkSession
@@ -7,6 +6,7 @@ from typing import List, Optional, Union
 import sempy_labs._icons as icons
 from sempy._utils._log import log
 
+
 def lakehouse_attached() -> bool:
     """
     Identifies if a lakehouse is attached to the notebook.
@@ -25,6 +25,7 @@ def lakehouse_attached() -> bool:
     else:
         return False
 
+
 @log
 def optimize_lakehouse_tables(
     tables: Optional[Union[str, List[str]]] = None,
@@ -36,8 +37,9 @@ def optimize_lakehouse_tables(
 
     Parameters
     ----------
-    tables : str | List[str] | None
-        The table(s) to optimize. If no tables are specified, all tables in the lakehouse will be optimized.
+    tables : str | List[str], default=None
+        The table(s) to optimize.
+        Defaults to None which resovles to optimizing all tables within the lakehouse.
     lakehouse : str, default=None
         The Fabric lakehouse.
         Defaults to None which resolves to the lakehouse attached to the notebook.

sempy_labs/lakehouse/_shortcuts.py

@@ -1,4 +1,3 @@
-import sempy
 import sempy.fabric as fabric
 from sempy_labs._helper_functions import (
     resolve_lakehouse_name,
@@ -7,6 +6,7 @@ from sempy_labs._helper_functions import (
 )
 from typing import Optional
 import sempy_labs._icons as icons
+from sempy.fabric.exceptions import FabricHTTPException
 
 
 def create_shortcut_onelake(
@@ -53,7 +53,7 @@ def create_shortcut_onelake(
         shortcut_name = table_name
 
     client = fabric.FabricRestClient()
-    tablePath = "Tables/" + table_name
+    tablePath = f"Tables/{table_name}"
 
     request_body = {
         "path": "Tables",
@@ -74,12 +74,15 @@ def create_shortcut_onelake(
         )
         if response.status_code == 201:
             print(
-                f"{icons.green_dot} The shortcut '{shortcut_name}' was created in the '{destination_lakehouse}' lakehouse within the '{destination_workspace} workspace. It is based on the '{table_name}' table in the '{source_lakehouse}' lakehouse within the '{source_workspace}' workspace."
+                f"{icons.green_dot} The shortcut '{shortcut_name}' was created in the '{destination_lakehouse}' lakehouse within"
+                f" the '{destination_workspace} workspace. It is based on the '{table_name}' table in the '{source_lakehouse}' lakehouse within the '{source_workspace}' workspace."
             )
         else:
             print(response.status_code)
     except Exception as e:
-        raise ValueError(f"{icons.red_dot} Failed to create a shortcut for the '{table_name}' table.") from e
+        raise ValueError(
+            f"{icons.red_dot} Failed to create a shortcut for the '{table_name}' table."
+        ) from e
 
 
 def create_shortcut(
@@ -114,7 +117,9 @@ def create_shortcut(
     sourceValues = list(source_titles.keys())
 
     if source not in sourceValues:
-        raise ValueError(f"{icons.red_dot} The 'source' parameter must be one of these values: {sourceValues}.")
+        raise ValueError(
+            f"{icons.red_dot} The 'source' parameter must be one of these values: {sourceValues}."
+        )
 
     sourceTitle = source_titles[source]
 
@@ -147,12 +152,15 @@ def create_shortcut(
         )
         if response.status_code == 201:
             print(
-                f"{icons.green_dot} The shortcut '{shortcutActualName}' was created in the '{lakehouse}' lakehouse within the '{workspace} workspace. It is based on the '{subpath}' table in '{sourceTitle}'."
+                f"{icons.green_dot} The shortcut '{shortcutActualName}' was created in the '{lakehouse}' lakehouse within"
+                f" the '{workspace} workspace. It is based on the '{subpath}' table in '{sourceTitle}'."
             )
         else:
             print(response.status_code)
     except Exception as e:
-        raise ValueError(f"{icons.red_dot} Failed to create a shortcut for the '{shortcut_name}' table.") from e
+        raise ValueError(
+            f"{icons.red_dot} Failed to create a shortcut for the '{shortcut_name}' table."
+        ) from e
 
 
 def delete_shortcut(
@@ -187,9 +195,8 @@ def delete_shortcut(
         f"/v1/workspaces/{workspace_id}/items/{lakehouse_id}/shortcuts/Tables/{shortcut_name}"
     )
 
-    if response.status_code == 200:
-        print(
-            f"{icons.green_dot} The '{shortcut_name}' shortcut in the '{lakehouse}' within the '{workspace}' workspace has been deleted."
-        )
-    else:
-        raise ValueError(f"{icons.red_dot} The '{shortcut_name}' has not been deleted.")
+    if response.status_code != 200:
+        raise FabricHTTPException(response)
+    print(
+        f"{icons.green_dot} The '{shortcut_name}' shortcut in the '{lakehouse}' within the '{workspace}' workspace has been deleted."
+    )

sempy_labs/migration/__init__.py

@@ -27,5 +27,5 @@ __all__ = [
     "migrate_model_objects_to_semantic_model",
     "migrate_tables_columns_to_semantic_model",
     "migration_validation",
-    "refresh_calc_tables"
+    "refresh_calc_tables",
 ]