semantic-link-labs 0.12.8__py3-none-any.whl

sempy_labs/directlake/_get_directlake_lakehouse.py

@@ -0,0 +1,62 @@
+import sempy.fabric as fabric
+from .._helper_functions import (
+    resolve_lakehouse_id,
+)
+from typing import Optional, Tuple
+from uuid import UUID
+import sempy_labs._icons as icons
+from sempy._utils._log import log
+
+
+@log
+def get_direct_lake_lakehouse(
+    dataset: str | UUID,
+    workspace: Optional[str | UUID] = None,
+    lakehouse: Optional[str] = None,
+    lakehouse_workspace: Optional[str | UUID] = None,
+) -> Tuple[str, UUID]:
+    """
+    Identifies the lakehouse used by a Direct Lake semantic model.
+
+    Parameters
+    ----------
+    dataset : str | uuid.UUID
+        Name or ID of the semantic model.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    lakehouse : str, default=None
+        The Fabric lakehouse used by the Direct Lake semantic model.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    lakehouse_workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID used by the lakehouse.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    Tuple[str, uuid.UUID]
+        The lakehouse name and lakehouse ID.
+    """
+
+    from sempy_labs.directlake._dl_helper import get_direct_lake_source
+
+    artifact_type, artifact_name, artifact_id, workspace_id = get_direct_lake_source(
+        dataset=dataset, workspace=workspace
+    )
+
+    if artifact_type in ["Lakehouse", "Warehouse"]:
+        return artifact_name, artifact_id
+    else:
+        fabric.refresh_tom_cache(workspace=workspace)
+        dfP = fabric.list_partitions(dataset=dataset, workspace=workspace)
+        dfP_filt = dfP[dfP["Mode"] == "DirectLake"]
+        if dfP_filt.empty:
+            raise ValueError(
+                f"{icons.red_dot} The '{dataset}' semantic model within the '{workspace}' workspace is not in Direct Lake mode."
+            )
+        lakehouse_id = resolve_lakehouse_id(
+            lakehouse=lakehouse, workspace=lakehouse_workspace
+        )
+        return lakehouse, lakehouse_id

sempy_labs/directlake/_get_shared_expression.py

@@ -0,0 +1,34 @@
+from typing import Optional
+from uuid import UUID
+from sempy._utils._log import log
+
+
+@log
+def get_shared_expression(
+    lakehouse: Optional[str] = None, workspace: Optional[str | UUID] = None
+) -> str:
+    """
+    Dynamically generates the M expression used by a Direct Lake model for a given lakehouse.
+
+    Parameters
+    ----------
+    lakehouse : str, default=None
+        The Fabric lakehouse used by the Direct Lake semantic model.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID used by the lakehouse.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    str
+        Shows the expression which can be used to connect a Direct Lake semantic model to its SQL Endpoint.
+    """
+    from sempy_labs.directlake._generate_shared_expression import (
+        generate_shared_expression,
+    )
+
+    return generate_shared_expression(
+        item_name=lakehouse, item_type="Lakehouse", workspace=workspace
+    )

sempy_labs/directlake/_guardrails.py

@@ -0,0 +1,96 @@
+import sempy.fabric as fabric
+import pandas as pd
+from typing import Optional
+import sempy_labs._icons as icons
+from uuid import UUID
+from .._helper_functions import (
+    resolve_workspace_name_and_id,
+)
+from sempy._utils._log import log
+
+
+@log
+def get_direct_lake_guardrails() -> pd.DataFrame:
+    """
+    Shows the guardrails for when Direct Lake semantic models will fallback to Direct Query
+    based on Microsoft's `online documentation <https://learn.microsoft.com/power-bi/enterprise/directlake-overview>`_.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A table showing the Direct Lake guardrails by SKU.
+    """
+
+    url = "https://learn.microsoft.com/power-bi/enterprise/directlake-overview"
+
+    tables = pd.read_html(url)
+    for df in tables:
+        first_column_name = df.columns[0]
+        if first_column_name.startswith("Fabric"):
+            df[first_column_name] = df[first_column_name].str.split("/")
+            df = df.explode(first_column_name, ignore_index=True)
+            break
+
+    return df
+
+
+def get_sku_size(workspace: Optional[str | UUID] = None) -> str:
+    """
+    Shows the SKU size for a workspace.
+
+    Parameters
+    ----------
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    str
+        The SKU size for a workspace.
+    """
+    from sempy_labs._capacities import list_capacities
+
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+
+    dfW = fabric.list_workspaces(filter=f"id eq '{workspace_id}'")
+
+    if len(dfW) == 0:
+        raise ValueError(
+            f"{icons.red_dot} The '{workspace_name}' is not a valid workspace."
+        )
+
+    capacity_id = dfW["Capacity Id"].iloc[0]
+    dfC = list_capacities()
+    dfC_filt = dfC[dfC["Id"] == capacity_id]
+
+    if len(dfC_filt) == 0:
+        raise ValueError(
+            f"{icons.red_dot} The '{capacity_id}' Id is not a valid capacity Id."
+        )
+
+    return dfC_filt["Sku"].iloc[0]
+
+
+def get_directlake_guardrails_for_sku(sku_size: str) -> pd.DataFrame:
+    """
+    Shows the guardrails for Direct Lake based on the SKU used by your workspace's capacity.
+    * Use the result of the 'get_sku_size' function as an input for this function's sku_size parameter.*
+
+    Parameters
+    ----------
+    sku_size : str
+        Sku size of a workspace/capacity.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A table showing the Direct Lake guardrails for the given SKU.
+    """
+
+    df = get_direct_lake_guardrails()
+    col_name = df.columns[0]
+    filtered_df = df[df[col_name] == sku_size]
+
+    return filtered_df

sempy_labs/directlake/_list_directlake_model_calc_tables.py

@@ -0,0 +1,70 @@
+import sempy.fabric as fabric
+import pandas as pd
+from .._list_functions import list_tables
+from ..tom import connect_semantic_model
+from typing import Optional
+from sempy._utils._log import log
+import sempy_labs._icons as icons
+from uuid import UUID
+from .._helper_functions import (
+    resolve_dataset_name_and_id,
+    resolve_workspace_name_and_id,
+)
+
+
+@log
+def list_direct_lake_model_calc_tables(
+    dataset: str | UUID, workspace: Optional[str | UUID] = None
+) -> pd.DataFrame:
+    """
+    Shows the calculated tables and their respective DAX expression for a Direct Lake model (which has been migrated from import/DirectQuery).
+
+    Parameters
+    ----------
+    dataset : str | uuid.UUID
+        Name or ID of the semantic model.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing the calculated tables which were migrated to Direct Lake and whose DAX expressions are stored as model annotations.
+    """
+
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    (dataset_name, dataset_id) = resolve_dataset_name_and_id(dataset, workspace_id)
+
+    df = pd.DataFrame(columns=["Table Name", "Source Expression"])
+
+    with connect_semantic_model(
+        dataset=dataset_id, readonly=True, workspace=workspace_id
+    ) as tom:
+
+        is_direct_lake = tom.is_direct_lake()
+
+        if not is_direct_lake:
+            raise ValueError(
+                f"{icons.red_dot} The '{dataset}' semantic model is not in Direct Lake mode."
+            )
+        else:
+            fabric.refresh_tom_cache(workspace=workspace)
+            dfA = fabric.list_annotations(dataset=dataset_id, workspace=workspace_id)
+            dfT = list_tables(dataset_id, workspace_id)
+            dfA_filt = dfA[
+                (dfA["Object Type"] == "Model")
+                & (dfA["Annotation Name"].isin(dfT["Name"]))
+            ]
+
+            for i, r in dfA_filt.iterrows():
+                tName = r["Annotation Name"]
+                se = r["Annotation Value"]
+
+                new_data = {"Table Name": tName, "Source Expression": se}
+                df = pd.concat(
+                    [df, pd.DataFrame(new_data, index=[0])], ignore_index=True
+                )
+
+            return df

sempy_labs/directlake/_show_unsupported_directlake_objects.py

@@ -0,0 +1,90 @@
+import sempy.fabric as fabric
+import pandas as pd
+from .._helper_functions import (
+    format_dax_object_name,
+    resolve_dataset_name_and_id,
+    resolve_workspace_name_and_id,
+)
+from typing import Optional, Tuple
+from sempy._utils._log import log
+from uuid import UUID
+
+
+@log
+def show_unsupported_direct_lake_objects(
+    dataset: str | UUID, workspace: Optional[str | UUID] = None
+) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
+    """
+    Returns a list of a semantic model's objects which are not supported by Direct Lake based on
+    `official documentation <https://learn.microsoft.com/power-bi/enterprise/directlake-overview#known-issues-and-limitations>`_.
+
+    Parameters
+    ----------
+    dataset : str | uuid.UUID
+        Name or ID of the semantic model.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    pandas.DataFrame, pandas.DataFrame, pandas.DataFrame
+        3 pandas dataframes showing objects in a semantic model which are not supported by Direct Lake.
+    """
+
+    pd.options.mode.chained_assignment = None
+
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    (dataset_name, dataset_id) = resolve_dataset_name_and_id(dataset, workspace_id)
+
+    fabric.refresh_tom_cache(workspace=workspace)
+
+    dfT = fabric.list_tables(dataset=dataset_id, workspace=workspace_id)
+    dfC = fabric.list_columns(dataset=dataset_id, workspace=workspace_id)
+    dfR = fabric.list_relationships(dataset=dataset_id, workspace=workspace_id)
+
+    # Calc tables
+    dfT_filt = dfT[dfT["Type"] == "Calculated Table"]
+    dfT_filt.rename(columns={"Name": "Table Name"}, inplace=True)
+    t = dfT_filt[["Table Name", "Type"]]
+
+    # Calc columns
+    dfC_filt = dfC[(dfC["Type"] == "Calculated") | (dfC["Data Type"] == "Binary")]
+    c = dfC_filt[["Table Name", "Column Name", "Type", "Data Type", "Source"]]
+
+    # Relationships
+    dfC["Column Object"] = format_dax_object_name(dfC["Table Name"], dfC["Column Name"])
+    dfR["From Object"] = format_dax_object_name(dfR["From Table"], dfR["From Column"])
+    dfR["To Object"] = format_dax_object_name(dfR["To Table"], dfR["To Column"])
+    merged_from = pd.merge(
+        dfR, dfC, left_on="From Object", right_on="Column Object", how="left"
+    )
+    merged_to = pd.merge(
+        dfR, dfC, left_on="To Object", right_on="Column Object", how="left"
+    )
+
+    dfR["From Column Data Type"] = merged_from["Data Type"]
+    dfR["To Column Data Type"] = merged_to["Data Type"]
+
+    dfR_filt = dfR[(dfR["From Column Data Type"] != dfR["To Column Data Type"])]
+    r = dfR_filt[
+        [
+            "From Table",
+            "From Column",
+            "To Table",
+            "To Column",
+            "From Column Data Type",
+            "To Column Data Type",
+        ]
+    ]
+
+    # print('Calculated Tables are not supported...')
+    # display(t)
+    # print("Learn more about Direct Lake limitations here: https://learn.microsoft.com/power-bi/enterprise/directlake-overview#known-issues-and-limitations")
+    # print('Calculated columns are not supported. Columns of binary data type are not supported.')
+    # display(c)
+    # print('Columns used for relationship cannot be of data type datetime and they also must be of the same data type.')
+    # display(r)
+
+    return t, c, r

sempy_labs/directlake/_update_directlake_model_lakehouse_connection.py

@@ -0,0 +1,239 @@
+from ._generate_shared_expression import generate_shared_expression
+from .._helper_functions import (
+    resolve_dataset_name_and_id,
+    resolve_workspace_name_and_id,
+    resolve_item_name_and_id,
+    resolve_lakehouse_name_and_id,
+)
+from sempy._utils._log import log
+from ..tom import connect_semantic_model
+from typing import Optional, List
+import sempy_labs._icons as icons
+from uuid import UUID
+import re
+
+
+@log
+def _extract_expression_list(expression):
+    """
+    Finds the pattern for DL/SQL & DL/OL expressions in the semantic model.
+    """
+
+    pattern_sql = r'Sql\.Database\s*\(\s*"([^"]+)"\s*,\s*"([^"]+)"\s*\)'
+    pattern_no_sql = (
+        r'AzureStorage\.DataLake\(".*?/([0-9a-fA-F\-]{36})/([0-9a-fA-F\-]{36})"'
+    )
+
+    match_sql = re.search(pattern_sql, expression)
+    match_no_sql = re.search(pattern_no_sql, expression)
+
+    result = []
+    if match_sql:
+        value_1, value_2 = match_sql.groups()
+        result = [value_1, value_2, True]
+    elif match_no_sql:
+        value_1, value_2 = match_no_sql.groups()
+        result = [value_1, value_2, False]
+
+    return result
+
+
+@log
+def _get_direct_lake_expressions(
+    dataset: str | UUID, workspace: Optional[str | UUID] = None
+) -> dict:
+    """
+    Extracts a dictionary of all Direct Lake expressions from a semantic model.
+    """
+
+    from sempy_labs.tom import connect_semantic_model
+
+    result = {}
+
+    with connect_semantic_model(dataset=dataset, workspace=workspace) as tom:
+        for e in tom.model.Expressions:
+            expr_name = e.Name
+            expr = e.Expression
+
+            list_values = _extract_expression_list(expr)
+            if list_values:
+                result[expr_name] = list_values
+
+    return result
+
+
+@log
+def update_direct_lake_model_lakehouse_connection(
+    dataset: str | UUID,
+    workspace: Optional[str | UUID] = None,
+    lakehouse: Optional[str] = None,
+    lakehouse_workspace: Optional[str | UUID] = None,
+):
+    """
+    Remaps a Direct Lake semantic model's SQL Endpoint connection to a new lakehouse.
+
+    Parameters
+    ----------
+    dataset : str | UUID
+        Name or ID of the semantic model.
+    workspace : str | UUID, default=None
+        The Fabric workspace name or ID in which the semantic model exists.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    lakehouse : str, default=None
+        The Fabric lakehouse used by the Direct Lake semantic model.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    lakehouse_workspace : str | UUID, default=None
+        The Fabric workspace name or ID used by the lakehouse.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    update_direct_lake_model_connection(
+        dataset=dataset,
+        workspace=workspace,
+        source=lakehouse,
+        source_type="Lakehouse",
+        source_workspace=lakehouse_workspace,
+    )
+
+
+@log
+def update_direct_lake_model_connection(
+    dataset: str | UUID,
+    workspace: Optional[str | UUID] = None,
+    source: Optional[str] = None,
+    source_type: str = "Lakehouse",
+    source_workspace: Optional[str | UUID] = None,
+    use_sql_endpoint: bool = True,
+    tables: Optional[str | List[str]] = None,
+):
+    """
+    Remaps a Direct Lake semantic model's SQL Endpoint connection to a new lakehouse/warehouse.
+
+    Parameters
+    ----------
+    dataset : str | uuid.UUID
+        Name or ID of the semantic model.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID in which the semantic model exists.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    source : str, default=None
+        The name of the Fabric lakehouse/warehouse used by the Direct Lake semantic model.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    source_type : str, default="Lakehouse"
+        The type of source for the Direct Lake semantic model. Valid options: "Lakehouse", "Warehouse".
+    source_workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID used by the lakehouse/warehouse.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    use_sql_endpoint : bool, default=True
+        If True, the SQL Endpoint will be used for the connection.
+        If False, Direct Lake over OneLake will be used.
+    tables : str | List[str], default=None
+        The name(s) of the table(s) to update in the Direct Lake semantic model.
+        If None, all tables will be updated (if there is only one expression).
+        If multiple tables are specified, they must be provided as a list.
+    """
+    if use_sql_endpoint:
+        icons.sll_tags.append("UpdateDLConnection_SQL")
+    else:
+        icons.sll_tags.append("UpdateDLConnection_DLOL")
+
+    if isinstance(tables, str):
+        tables = [tables]
+
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    (dataset_name, dataset_id) = resolve_dataset_name_and_id(dataset, workspace_id)
+
+    source_type = source_type.capitalize()
+
+    if source_type not in ["Lakehouse", "Warehouse"]:
+        raise ValueError(
+            f"{icons.red_dot} The 'source_type' must be either 'Lakehouse' or 'Warehouse'."
+        )
+
+    if source_workspace is None:
+        source_workspace = workspace_name
+
+    if source_type == "Lakehouse":
+        (source_name, source_id) = resolve_lakehouse_name_and_id(
+            lakehouse=source, workspace=source_workspace
+        )
+    else:
+        (source_name, source_id) = resolve_item_name_and_id(
+            item=source, type=source_type, workspace=source_workspace
+        )
+
+    shared_expression = generate_shared_expression(
+        item_name=source_name,
+        item_type=source_type,
+        workspace=source_workspace,
+        use_sql_endpoint=use_sql_endpoint,
+    )
+
+    expression_dict = _get_direct_lake_expressions(dataset=dataset, workspace=workspace)
+    expressions = list(expression_dict.keys())
+
+    with connect_semantic_model(
+        dataset=dataset_id, readonly=False, workspace=workspace_id
+    ) as tom:
+
+        if not tom.is_direct_lake():
+            raise ValueError(
+                f"{icons.red_dot} The '{dataset_name}' semantic model within the '{workspace_name}' workspace is not in Direct Lake. This function is only applicable to Direct Lake semantic models."
+            )
+
+        # Update the single connection expression
+        if len(expressions) > 1 and not tables:
+            print(
+                f"{icons.info} Multiple expressions found in the model. Please specify the tables to update using the 'tables parameter."
+            )
+            return
+        elif len(expressions) == 1 and not tables:
+            expr = expressions[0]
+            tom.model.Expressions[expr].Expression = shared_expression
+
+            print(
+                f"{icons.green_dot} The expression in the '{dataset_name}' semantic model within the '{workspace_name}' workspace has been updated to point to the '{source}' {source_type.lower()} in the '{source_workspace}' workspace."
+            )
+        else:
+            import sempy
+
+            sempy.fabric._client._utils._init_analysis_services()
+            import Microsoft.AnalysisServices.Tabular as TOM
+
+            expr_list = _extract_expression_list(shared_expression)
+
+            expr_name = next(
+                (name for name, exp in expression_dict.items() if exp == expr_list),
+                None,
+            )
+
+            # If the expression does not already exist, create it
+            def generate_unique_name(existing_names):
+                i = 1
+                while True:
+                    candidate = f"DatabaseQuery{i}"
+                    if candidate not in existing_names:
+                        return candidate
+                    i += 1
+
+            if not expr_name:
+                expr_name = generate_unique_name(expressions)
+                tom.add_expression(name=expr_name, expression=shared_expression)
+
+            all_tables = [t.Name for t in tom.model.Tables]
+            for t_name in tables:
+                if t_name not in all_tables:
+                    raise ValueError(
+                        f"{icons.red_dot} The table '{t_name}' does not exist in the '{dataset_name}' semantic model within the '{workspace_name}' workspace."
+                    )
+                p = next(p for p in tom.model.Tables[t_name].Partitions)
+                if p.Mode != TOM.ModeType.DirectLake:
+                    raise ValueError(
+                        f"{icons.red_dot} The table '{t_name}' in the '{dataset_name}' semantic model within the '{workspace_name}' workspace is not in Direct Lake mode. This function is only applicable to Direct Lake tables."
+                    )
+
+                p.Source.ExpressionSource = tom.model.Expressions[expr_name]