semantic-link-labs 0.4.1__py3-none-any.whl

sempy_labs/directlake/__init__.py

@@ -0,0 +1,45 @@
+from sempy_labs.directlake._directlake_schema_compare import direct_lake_schema_compare
+from sempy_labs.directlake._directlake_schema_sync import direct_lake_schema_sync
+from sempy_labs.directlake._fallback import (
+    check_fallback_reason,
+)
+from sempy_labs.directlake._get_directlake_lakehouse import get_direct_lake_lakehouse
+from sempy_labs.directlake._get_shared_expression import get_shared_expression
+from sempy_labs.directlake._guardrails import (
+    get_direct_lake_guardrails,
+    get_sku_size,
+    get_directlake_guardrails_for_sku,
+)
+from sempy_labs.directlake._list_directlake_model_calc_tables import (
+    list_direct_lake_model_calc_tables,
+)
+from sempy_labs.directlake._show_unsupported_directlake_objects import (
+    show_unsupported_direct_lake_objects,
+)
+from sempy_labs.directlake._update_directlake_model_lakehouse_connection import (
+    update_direct_lake_model_lakehouse_connection,
+)
+from sempy_labs.directlake._update_directlake_partition_entity import (
+    update_direct_lake_partition_entity,
+)
+from sempy_labs.directlake._warm_cache import (
+    warm_direct_lake_cache_isresident,
+    warm_direct_lake_cache_perspective,
+)
+
+__all__ = [
+    "direct_lake_schema_compare",
+    "direct_lake_schema_sync",
+    "check_fallback_reason",
+    "get_direct_lake_lakehouse",
+    "get_shared_expression",
+    "get_direct_lake_guardrails",
+    "get_sku_size",
+    "get_directlake_guardrails_for_sku",
+    "list_direct_lake_model_calc_tables",
+    "show_unsupported_direct_lake_objects",
+    "update_direct_lake_model_lakehouse_connection",
+    "update_direct_lake_partition_entity",
+    "warm_direct_lake_cache_isresident",
+    "warm_direct_lake_cache_perspective",
+]

sempy_labs/directlake/_directlake_schema_compare.py

@@ -0,0 +1,110 @@
+import sempy
+import sempy.fabric as fabric
+import pandas as pd
+from sempy_labs._helper_functions import (
+    format_dax_object_name,
+    resolve_lakehouse_name,
+    get_direct_lake_sql_endpoint,
+)
+from IPython.display import display
+from sempy_labs.lakehouse._get_lakehouse_columns import get_lakehouse_columns
+from sempy_labs._list_functions import list_tables
+from typing import Optional
+
+
+def direct_lake_schema_compare(
+    dataset: str,
+    workspace: Optional[str] = None,
+    lakehouse: Optional[str] = None,
+    lakehouse_workspace: Optional[str] = None,
+):
+    """
+    Checks that all the tables in a Direct Lake semantic model map to tables in their corresponding lakehouse and that the columns in each table exist.
+
+    Parameters
+    ----------
+    dataset : str
+        Name of the semantic model.
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    lakehouse : str, default=None
+        The Fabric lakehouse used by the Direct Lake semantic model.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    lakehouse_workspace : str, default=None
+        The Fabric workspace used by the lakehouse.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    if workspace == None:
+        workspace_id = fabric.get_workspace_id()
+        workspace = fabric.resolve_workspace_name(workspace_id)
+
+    if lakehouse_workspace is None:
+        lakehouse_workspace = workspace
+
+    if lakehouse == None:
+        lakehouse_id = fabric.get_lakehouse_id()
+        lakehouse = resolve_lakehouse_name(lakehouse_id, lakehouse_workspace)
+
+    dfP = fabric.list_partitions(dataset=dataset, workspace=workspace)
+    sqlEndpointId = get_direct_lake_sql_endpoint(dataset, workspace)
+    dfI = fabric.list_items(workspace=lakehouse_workspace, type="SQLEndpoint")
+    dfI_filt = dfI[(dfI["Id"] == sqlEndpointId)]
+
+    if len(dfI_filt) == 0:
+        print(
+            f"The SQL Endpoint in the '{dataset}' semantic model in the '{workspace} workspace does not point to the '{lakehouse}' lakehouse in the '{lakehouse_workspace}' workspace as specified."
+        )
+        return
+
+    if not any(r["Mode"] == "DirectLake" for i, r in dfP.iterrows()):
+        print(f"The '{dataset}' semantic model is not in Direct Lake mode.")
+        return
+
+    dfT = list_tables(dataset, workspace)
+    dfC = fabric.list_columns(dataset=dataset, workspace=workspace)
+    lc = get_lakehouse_columns(lakehouse, lakehouse_workspace)
+
+    dfT.rename(columns={"Type": "Table Type"}, inplace=True)
+    dfP_filt = dfP[dfP["Mode"] == "DirectLake"]
+    dfC = pd.merge(dfC, dfP[["Table Name", "Query"]], on="Table Name", how="inner")
+    dfC = pd.merge(
+        dfC,
+        dfT[["Name", "Table Type"]],
+        left_on="Table Name",
+        right_on="Name",
+        how="inner",
+    )
+    dfC["Full Column Name"] = format_dax_object_name(dfC["Query"], dfC["Source"])
+    dfC_filt = dfC[dfC["Table Type"] == "Table"]
+    # Schema compare
+    missingtbls = dfP_filt[~dfP_filt["Query"].isin(lc["Table Name"])]
+    missingtbls = missingtbls[["Table Name", "Query"]]
+    missingtbls.rename(columns={"Query": "Source Table"}, inplace=True)
+    missingcols = dfC_filt[~dfC_filt["Full Column Name"].isin(lc["Full Column Name"])]
+    missingcols = missingcols[
+        ["Table Name", "Column Name", "Type", "Data Type", "Source"]
+    ]
+    missingcols.rename(columns={"Source": "Source Column"}, inplace=True)
+
+    if len(missingtbls) == 0:
+        print(
+            f"All tables exist in the '{lakehouse}' lakehouse within the '{lakehouse_workspace}' workspace."
+        )
+    else:
+        print(
+            f"The following tables exist in the '{dataset}' semantic model within the '{workspace}' workspace but do not exist in the '{lakehouse}' lakehouse within the '{lakehouse_workspace}' workspace."
+        )
+        display(missingtbls)
+    if len(missingcols) == 0:
+        print(
+            f"All columns exist in the '{lakehouse}' lakehouse within the '{lakehouse_workspace}' workspace."
+        )
+    else:
+        print(
+            f"The following columns exist in the '{dataset}' semantic model within the '{workspace}' workspace but do not exist in the '{lakehouse}' lakehouse within the '{lakehouse_workspace}' workspace."
+        )
+        display(missingcols)

sempy_labs/directlake/_directlake_schema_sync.py

@@ -0,0 +1,128 @@
+import sempy
+import sempy.fabric as fabric
+import pandas as pd
+from sempy_labs.lakehouse._get_lakehouse_columns import get_lakehouse_columns
+from sempy_labs._helper_functions import (
+    format_dax_object_name,
+    resolve_lakehouse_name,
+    get_direct_lake_sql_endpoint,
+)
+from typing import Optional
+from sempy._utils._log import log
+
+
+@log
+def direct_lake_schema_sync(
+    dataset: str,
+    workspace: Optional[str] = None,
+    add_to_model: Optional[bool] = False,
+    lakehouse: Optional[str] = None,
+    lakehouse_workspace: Optional[str] = None,
+):
+    """
+    Shows/adds columns which exist in the lakehouse but do not exist in the semantic model (only for tables in the semantic model).
+
+    Parameters
+    ----------
+    dataset : str
+        Name of the semantic model.
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    add_to_model : bool, default=False
+        If set to True, columns which exist in the lakehouse but do not exist in the semantic model are added to the semantic model. No new tables are added.
+    lakehouse : str, default=None
+        The Fabric lakehouse used by the Direct Lake semantic model.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    lakehouse_workspace : str, default=None
+        The Fabric workspace used by the lakehouse.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    sempy.fabric._client._utils._init_analysis_services()
+    import Microsoft.AnalysisServices.Tabular as TOM
+    import System
+
+    if workspace == None:
+        workspace_id = fabric.get_workspace_id()
+        workspace = fabric.resolve_workspace_name(workspace_id)
+
+    if lakehouse_workspace is None:
+        lakehouse_workspace = workspace
+
+    if lakehouse == None:
+        lakehouse_id = fabric.get_lakehouse_id()
+        lakehouse = resolve_lakehouse_name(lakehouse_id, lakehouse_workspace)
+
+    sqlEndpointId = get_direct_lake_sql_endpoint(dataset, workspace)
+
+    dfI = fabric.list_items(workspace=lakehouse_workspace, type="SQLEndpoint")
+    dfI_filt = dfI[(dfI["Id"] == sqlEndpointId)]
+
+    if len(dfI_filt) == 0:
+        print(
+            f"The SQL Endpoint in the '{dataset}' semantic model in the '{workspace} workspace does not point to the '{lakehouse}' lakehouse in the '{lakehouse_workspace}' workspace as specified."
+        )
+        return
+
+    dfP = fabric.list_partitions(dataset=dataset, workspace=workspace)
+    dfP_filt = dfP[dfP["Source Type"] == "Entity"]
+    dfC = fabric.list_columns(dataset=dataset, workspace=workspace)
+    dfC_filt = dfC[dfC["Table Name"].isin(dfP_filt["Table Name"].values)]
+    dfC_filt = pd.merge(
+        dfC_filt, dfP_filt[["Table Name", "Query"]], on="Table Name", how="left"
+    )
+    dfC_filt["Column Object"] = format_dax_object_name(
+        dfC_filt["Query"], dfC_filt["Source"]
+    )
+
+    lc = get_lakehouse_columns(lakehouse, lakehouse_workspace)
+    lc_filt = lc[lc["Table Name"].isin(dfP_filt["Query"].values)]
+
+    mapping = {
+        "string": "String",
+        "bigint": "Int64",
+        "int": "Int64",
+        "smallint": "Int64",
+        "boolean": "Boolean",
+        "timestamp": "DateTime",
+        "date": "DateTime",
+        "decimal(38,18)": "Decimal",
+        "double": "Double",
+    }
+
+    tom_server = fabric.create_tom_server(readonly=False, workspace=workspace)
+    m = tom_server.Databases.GetByName(dataset).Model
+    for i, r in lc_filt.iterrows():
+        lakeTName = r["Table Name"]
+        lakeCName = r["Column Name"]
+        fullColName = r["Full Column Name"]
+        dType = r["Data Type"]
+
+        if fullColName not in dfC_filt["Column Object"].values:
+            dfL = dfP_filt[dfP_filt["Query"] == lakeTName]
+            tName = dfL["Table Name"].iloc[0]
+            if add_to_model:
+                col = TOM.DataColumn()
+                col.Name = lakeCName
+                col.SourceColumn = lakeCName
+                dt = mapping.get(dType)
+                try:
+                    col.DataType = System.Enum.Parse(TOM.DataType, dt)
+                except:
+                    print(
+                        f"ERROR: '{dType}' data type is not mapped properly to the semantic model data types."
+                    )
+                    return
+
+                m.Tables[tName].Columns.Add(col)
+                print(
+                    f"The '{lakeCName}' column has been added to the '{tName}' table as a '{dt}' data type within the '{dataset}' semantic model within the '{workspace}' workspace."
+                )
+            else:
+                print(
+                    f"The {fullColName} column exists in the lakehouse but not in the '{tName}' table in the '{dataset}' semantic model within the '{workspace}' workspace."
+                )
+        m.SaveChanges()

sempy_labs/directlake/_fallback.py

@@ -0,0 +1,62 @@
+import sempy
+import sempy.fabric as fabric
+import numpy as np
+from typing import List, Optional, Union
+
+
+def check_fallback_reason(dataset: str, workspace: Optional[str] = None):
+    """
+    Shows the reason a table in a Direct Lake semantic model would fallback to DirectQuery.
+
+    Parameters
+    ----------
+    dataset : str
+        Name of the semantic model.
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    pandas.DataFrame
+        The tables in the semantic model and their fallback reason.
+    """
+
+    if workspace == None:
+        workspace_id = fabric.get_workspace_id()
+        workspace = fabric.resolve_workspace_name(workspace_id)
+
+    dfP = fabric.list_partitions(dataset=dataset, workspace=workspace)
+    dfP_filt = dfP[dfP["Mode"] == "DirectLake"]
+
+    if len(dfP_filt) == 0:
+        print(
+            f"The '{dataset}' semantic model is not in Direct Lake. This function is only applicable to Direct Lake semantic models."
+        )
+    else:
+        df = fabric.evaluate_dax(
+            dataset=dataset,
+            workspace=workspace,
+            dax_string="""
+        SELECT [TableName] AS [Table Name],[FallbackReason] AS [FallbackReasonID]
+        FROM $SYSTEM.TMSCHEMA_DELTA_TABLE_METADATA_STORAGES
+        """,
+        )
+
+        value_mapping = {
+            0: "No reason for fallback",
+            1: "This table is not framed",
+            2: "This object is a view in the lakehouse",
+            3: "The table does not exist in the lakehouse",
+            4: "Transient error",
+            5: "Using OLS will result in fallback to DQ",
+            6: "Using RLS will result in fallback to DQ",
+        }
+
+        # Create a new column based on the mapping
+        df["Fallback Reason Detail"] = np.vectorize(value_mapping.get)(
+            df["FallbackReasonID"]
+        )
+
+        return df

sempy_labs/directlake/_get_directlake_lakehouse.py

@@ -0,0 +1,69 @@
+import sempy
+import sempy.fabric as fabric
+from sempy_labs._helper_functions import (
+    resolve_lakehouse_id,
+    resolve_lakehouse_name,
+    get_direct_lake_sql_endpoint,
+)
+from typing import Optional, Tuple
+from uuid import UUID
+from sempy_labs._helper_functions import resolve_workspace_name_and_id
+
+
+def get_direct_lake_lakehouse(
+    dataset: str,
+    workspace: Optional[str] = None,
+    lakehouse: Optional[str] = None,
+    lakehouse_workspace: Optional[str] = None,
+) -> Tuple[str, UUID]:
+    """
+    Identifies the lakehouse used by a Direct Lake semantic model.
+
+    Parameters
+    ----------
+    dataset : str
+        Name of the semantic model.
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    lakehouse : str, default=None
+        The Fabric lakehouse used by the Direct Lake semantic model.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    lakehouse_workspace : str, default=None
+        The Fabric workspace used by the lakehouse.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    str, uuid.UUID
+        The lakehouse name and lakehouse ID.
+    """
+
+    workspace = fabric.resolve_workspace_name(workspace)
+
+    if lakehouse_workspace is None:
+        lakehouse_workspace = workspace
+
+    if lakehouse == None:
+        lakehouse_id = fabric.get_lakehouse_id()
+        lakehouse = resolve_lakehouse_name(lakehouse_id, lakehouse_workspace)
+
+    dfP = fabric.list_partitions(dataset=dataset, workspace=workspace)
+    dfP_filt = dfP[dfP["Mode"] == "DirectLake"]
+
+    if len(dfP_filt) == 0:
+        raise ValueError(
+            f"ERROR: The '{dataset}' semantic model within the '{workspace}' workspace is not in Direct Lake mode."
+        )
+
+    sqlEndpointId = get_direct_lake_sql_endpoint(dataset, workspace)
+
+    dfI = fabric.list_items(workspace=lakehouse_workspace, type="SQLEndpoint")
+    dfI_filt = dfI[dfI["Id"] == sqlEndpointId]
+    lakehouseName = dfI_filt["Display Name"].iloc[0]
+
+    lakehouseId = resolve_lakehouse_id(lakehouseName, lakehouse_workspace)
+
+    return lakehouseName, lakehouseId

sempy_labs/directlake/_get_shared_expression.py

@@ -0,0 +1,59 @@
+import sempy
+import sempy.fabric as fabric
+from sempy_labs._helper_functions import (
+    resolve_lakehouse_name,
+    resolve_workspace_name_and_id,
+)
+from sempy_labs._list_functions import list_lakehouses
+from typing import Optional
+
+
+def get_shared_expression(
+    lakehouse: Optional[str] = None, workspace: Optional[str] = None
+):
+    """
+    Dynamically generates the M expression used by a Direct Lake model for a given lakehouse.
+
+    Parameters
+    ----------
+    lakehouse : str, default=None
+        The Fabric lakehouse used by the Direct Lake semantic model.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    workspace : str, default=None
+        The Fabric workspace used by the lakehouse.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    str
+        Shows the expression which can be used to connect a Direct Lake semantic model to its SQL Endpoint.
+    """
+
+    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
+    if lakehouse == None:
+        lakehouse_id = fabric.get_lakehouse_id()
+        lakehouse = resolve_lakehouse_name(lakehouse_id)
+
+    dfL = list_lakehouses(workspace=workspace)
+    lakeDetail = dfL[dfL["Lakehouse Name"] == lakehouse]
+
+    sqlEPCS = lakeDetail["SQL Endpoint Connection String"].iloc[0]
+    sqlepid = lakeDetail["SQL Endpoint ID"].iloc[0]
+    provStatus = lakeDetail["SQL Endpoint Provisioning Status"].iloc[0]
+
+    if provStatus == "InProgress":
+        print(
+            f"The SQL Endpoint for the '{lakehouse}' lakehouse within the '{workspace}' workspace has not yet been provisioned. Please wait until it has been provisioned."
+        )
+        return
+
+    sh = (
+        'let\n\tdatabase = Sql.Database("'
+        + sqlEPCS
+        + '", "'
+        + sqlepid
+        + '")\nin\n\tdatabase'
+    )
+
+    return sh

sempy_labs/directlake/_guardrails.py

@@ -0,0 +1,84 @@
+import sempy
+import sempy.fabric as fabric
+import pandas as pd
+from typing import List, Optional, Union
+
+
+def get_direct_lake_guardrails():
+    """
+    Shows the guardrails for when Direct Lake semantic models will fallback to Direct Query based on Microsoft's online documentation.
+
+    Parameters
+    ----------
+
+    Returns
+    -------
+    pandas.DataFrame
+        A table showing the Direct Lake guardrails by SKU.
+    """
+
+    url = "https://learn.microsoft.com/power-bi/enterprise/directlake-overview"
+
+    tables = pd.read_html(url)
+    df = tables[0]
+    df["Fabric SKUs"] = df["Fabric SKUs"].str.split("/")
+    df = df.explode("Fabric SKUs", ignore_index=True)
+
+    return df
+
+
+def get_sku_size(workspace: Optional[str] = None):
+    """
+    Shows the SKU size for a workspace.
+
+    Parameters
+    ----------
+    workspace : str, default=None
+        The Fabric workspace.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    str
+        The SKU size for a workspace.
+    """
+
+    if workspace == None:
+        workspace_id = fabric.get_workspace_id()
+        workspace = fabric.resolve_workspace_name(workspace_id)
+
+    dfC = fabric.list_capacities()
+    dfW = fabric.list_workspaces().sort_values(by="Name", ascending=True)
+    dfC.rename(columns={"Id": "Capacity Id"}, inplace=True)
+    dfCW = pd.merge(
+        dfW,
+        dfC[["Capacity Id", "Sku", "Region", "State"]],
+        on="Capacity Id",
+        how="inner",
+    )
+    sku_value = dfCW.loc[dfCW["Name"] == workspace, "Sku"].iloc[0]
+
+    return sku_value
+
+
+def get_directlake_guardrails_for_sku(sku_size: str):
+    """
+    Shows the guardrails for Direct Lake based on the SKU used by your workspace's capacity.
+    *Use the result of the 'get_sku_size' function as an input for this function's skuSize parameter.*
+
+    Parameters
+    ----------
+    sku_size : str
+        Sku size of a workspace/capacity
+
+    Returns
+    -------
+    pandas.DataFrame
+        A table showing the Direct Lake guardrails for the given SKU.
+    """
+
+    df = get_direct_lake_guardrails()
+    filtered_df = df[df["Fabric SKUs"] == sku_size]
+
+    return filtered_df

sempy_labs/directlake/_list_directlake_model_calc_tables.py

@@ -0,0 +1,54 @@
+import sempy
+import sempy.fabric as fabric
+import pandas as pd
+from sempy_labs._list_functions import list_tables, list_annotations
+from typing import Optional
+from sempy._utils._log import log
+
+
+@log
+def list_direct_lake_model_calc_tables(dataset: str, workspace: Optional[str] = None):
+    """
+    Shows the calculated tables and their respective DAX expression for a Direct Lake model (which has been migrated from import/DirectQuery.
+
+    Parameters
+    ----------
+    dataset : str
+        Name of the semantic model.
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing the calculated tables which were migrated to Direct Lake and whose DAX expressions are stored as model annotations.
+    """
+
+    if workspace == None:
+        workspace_id = fabric.get_workspace_id()
+        workspace = fabric.resolve_workspace_name(workspace_id)
+
+    df = pd.DataFrame(columns=["Table Name", "Source Expression"])
+
+    dfP = fabric.list_partitions(dataset=dataset, workspace=workspace)
+    dfP_filt = dfP[dfP["Mode"] == "DirectLake"]
+
+    if len(dfP_filt) == 0:
+        print(f"The '{dataset}' semantic model is not in Direct Lake mode.")
+    else:
+        dfA = list_annotations(dataset, workspace)
+        dfT = list_tables(dataset, workspace)
+        dfA_filt = dfA[
+            (dfA["Object Type"] == "Model") & (dfA["Annotation Name"].isin(dfT["Name"]))
+        ]
+
+        for i, r in dfA_filt.iterrows():
+            tName = r["Annotation Name"]
+            se = r["Annotation Value"]
+
+            new_data = {"Table Name": tName, "Source Expression": se}
+            df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
+
+        return df