semantic-link-labs 0.12.8__py3-none-any.whl

sempy_labs/directlake/_update_directlake_partition_entity.py

@@ -0,0 +1,259 @@
+from sempy_labs.tom import connect_semantic_model
+from sempy_labs._refresh_semantic_model import refresh_semantic_model
+from sempy_labs.directlake._dl_helper import get_direct_lake_source
+from sempy_labs._helper_functions import (
+    _convert_data_type,
+    resolve_dataset_name_and_id,
+    resolve_workspace_name_and_id,
+    resolve_workspace_name,
+)
+from sempy._utils._log import log
+from typing import List, Optional, Union
+import sempy_labs._icons as icons
+from uuid import UUID
+import json
+
+
+@log
+def update_direct_lake_partition_entity(
+    dataset: str | UUID,
+    table_name: Union[str, List[str]],
+    entity_name: Union[str, List[str]],
+    schema: Optional[str] = None,
+    workspace: Optional[str | UUID] = None,
+):
+    """
+    Remaps a table (or tables) in a Direct Lake semantic model to a table in a lakehouse.
+
+    Parameters
+    ----------
+    dataset : str | uuid.UUID
+        Name or ID of the semantic model.
+    table_name : str, List[str]
+        Name of the table(s) in the semantic model.
+    entity_name : str, List[str]
+        Name of the lakehouse table to be mapped to the semantic model table.
+    schema : str, default=None
+        The schema of the lakehouse table to be mapped to the semantic model table.
+        Defaults to None which resolves to the existing schema of the lakehouse table.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID in which the semantic model exists.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    (dataset_name, dataset_id) = resolve_dataset_name_and_id(dataset, workspace_id)
+
+    # Support both str & list types
+    if isinstance(table_name, str):
+        table_name = [table_name]
+    if isinstance(entity_name, str):
+        entity_name = [entity_name]
+
+    if len(table_name) != len(entity_name):
+        raise ValueError(
+            f"{icons.red_dot} The 'table_name' and 'entity_name' arrays must be of equal length."
+        )
+
+    icons.sll_tags.append("UpdateDLPartition")
+
+    with connect_semantic_model(
+        dataset=dataset_id, readonly=False, workspace=workspace_id
+    ) as tom:
+
+        if not tom.is_direct_lake():
+            raise ValueError(
+                f"{icons.red_dot} The '{dataset_name}' semantic model within the '{workspace_name}' workspace is not in Direct Lake mode."
+            )
+
+        for tName in table_name:
+            i = table_name.index(tName)
+            eName = entity_name[i]
+            part_name = next(
+                p.Name
+                for t in tom.model.Tables
+                for p in t.Partitions
+                if t.Name == tName
+            )
+            current_slt = tom.model.Tables[tName].SourceLineageTag
+
+            if part_name is None:
+                raise ValueError(
+                    f"{icons.red_dot} The '{tName}' table in the '{dataset_name}' semantic model has not been updated."
+                )
+
+            tom.model.Tables[tName].Partitions[part_name].Source.EntityName = eName
+
+            # Update source lineage tag
+            if schema:
+                # Only set schema for DL over SQL (not DL over OneLake)
+                expression_source_name = (
+                    tom.model.Tables[tName]
+                    .Partitions[part_name]
+                    .Source.ExpressionSource.Name
+                )
+                expr = tom.model.Expressions[expression_source_name].Expression
+                if "Sql.Database" in expr:
+                    tom.model.Tables[tName].Partitions[
+                        part_name
+                    ].Source.SchemaName = schema
+                tom.model.Tables[tName].SourceLineageTag = f"[{schema}].[{eName}]"
+            else:
+                tom.model.Tables[tName].SourceLineageTag = f"[dbo].[{eName}]"
+
+            new_slt = tom.model.Tables[tName].SourceLineageTag
+
+            # PBI_RemovedChildren logic
+            try:
+                e_name = (
+                    tom.model.Tables[tName]
+                    .Partitions[part_name]
+                    .Source.ExpressionSource.Name
+                )
+                ann = tom.get_annotation_value(
+                    object=tom.model.Expressions[e_name], name="PBI_RemovedChildren"
+                )
+                if ann:
+                    e = json.loads(ann)
+                    for i in e:
+                        sltag = (
+                            i.get("remoteItemId", {})
+                            .get("analysisServicesObject", {})
+                            .get("sourceLineageTag", {})
+                        )
+                        if sltag == current_slt:
+                            i["remoteItemId"]["analysisServicesObject"][
+                                "sourceLineageTag"
+                            ] = new_slt
+                    tom.set_annotation(
+                        object=tom.model.Expressions[e_name],
+                        name="PBI_RemovedChildren",
+                        value=json.dumps(e),
+                    )
+            except Exception as e:
+                print(
+                    f"{icons.yellow_dot} Warning: Could not update PBI_RemovedChildren annotation for table '{tName}'. {str(e)}"
+                )
+
+            print(
+                f"{icons.green_dot} The '{tName}' table in the '{dataset_name}' semantic model within the '{workspace_name}' workspace has been updated to point to the '{eName}' table."
+            )
+
+
+@log
+def add_table_to_direct_lake_semantic_model(
+    dataset: str | UUID,
+    table_name: str,
+    lakehouse_table_name: str,
+    refresh: bool = True,
+    workspace: Optional[str | UUID] = None,
+    columns: Optional[List[str] | str] = None,
+):
+    """
+    Adds a table and all of its columns to a Direct Lake semantic model, based on a Fabric lakehouse table.
+
+    Parameters
+    ----------
+    dataset : str | uuid.UUID
+        Name or ID of the semantic model.
+    table_name : str, List[str]
+        Name of the table in the semantic model.
+    lakehouse_table_name : str
+        The name of the Fabric lakehouse table.
+    refresh : bool, default=True
+        Refreshes the table after it is added to the semantic model.
+    workspace : str | uuid.UUID, default=None
+        The name or ID of the Fabric workspace in which the semantic model resides.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    columns : List[str] | str, default=None
+        A list of column names to add to the table. If None, all columns from the
+        lakehouse table will be added.
+    """
+
+    from sempy_labs.lakehouse._get_lakehouse_columns import get_lakehouse_columns
+
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    (dataset_name, dataset_id) = resolve_dataset_name_and_id(dataset, workspace_id)
+
+    artifact_type, lakehouse_name, lakehouse_id, lakehouse_workspace_id = (
+        get_direct_lake_source(dataset=dataset_id, workspace=workspace_id)
+    )
+
+    if artifact_type == "Warehouse":
+        raise ValueError(
+            f"{icons.red_dot} This function is only valid for Direct Lake semantic models which source from Fabric lakehouses (not warehouses)."
+        )
+
+    if artifact_type is None:
+        raise ValueError(
+            f"{icons.red_dot} This function only supports Direct Lake semantic models where the source lakehouse resides in the same workpace as the semantic model."
+        )
+
+    if isinstance(columns, str):
+        columns = [columns]
+
+    lakehouse_workspace = resolve_workspace_name(workspace_id=lakehouse_workspace_id)
+
+    with connect_semantic_model(
+        dataset=dataset_id, readonly=False, workspace=workspace_id
+    ) as tom:
+
+        table_count = tom.model.Tables.Count
+
+        if not tom.is_direct_lake() and table_count > 0:
+            raise ValueError(
+                f"{icons.red_dot} This function is only valid for Direct Lake semantic models or semantic models with no tables."
+            )
+
+        if any(t.Name == table_name for t in tom.model.Tables):
+            raise ValueError(
+                f"The '{table_name}' table already exists in the '{dataset_name}' semantic model within the '{workspace_name}' workspace."
+            )
+
+        dfLC = get_lakehouse_columns(
+            lakehouse=lakehouse_name, workspace=lakehouse_workspace
+        )
+        dfLC_filt = dfLC[dfLC["Table Name"] == lakehouse_table_name]
+        if dfLC_filt.empty:
+            raise ValueError(
+                f"{icons.red_dot} The '{lakehouse_table_name}' table was not found in the '{lakehouse_name}' lakehouse within the '{lakehouse_workspace}' workspace."
+            )
+        if columns:
+            dfLC_filt = dfLC_filt[dfLC_filt["Column Name"].isin(columns)]
+
+        if dfLC_filt.empty:
+            raise ValueError(
+                f"{icons.red_dot} No matching columns were found in the '{lakehouse_table_name}' table in the '{lakehouse_name}' lakehouse within the '{lakehouse_workspace}' workspace."
+            )
+
+        tom.add_table(name=table_name)
+        print(
+            f"{icons.green_dot} The '{table_name}' table has been added to the '{dataset_name}' semantic model within the '{workspace_name}' workspace."
+        )
+        tom.add_entity_partition(
+            table_name=table_name, entity_name=lakehouse_table_name
+        )
+        print(
+            f"{icons.green_dot} The '{lakehouse_table_name}' partition has been added to the '{table_name}' table in the '{dataset_name}' semantic model within the '{workspace_name}' workspace."
+        )
+
+        for _, r in dfLC_filt.iterrows():
+            lakeCName = r["Column Name"]
+            dType = r["Data Type"]
+            dt = _convert_data_type(dType)
+            tom.add_data_column(
+                table_name=table_name,
+                column_name=lakeCName,
+                source_column=lakeCName,
+                data_type=dt,
+            )
+            print(
+                f"{icons.green_dot} The '{lakeCName}' column has been added to the '{table_name}' table as a '{dt}' data type in the '{dataset_name}' semantic model within the '{workspace_name}' workspace."
+            )
+
+        if refresh:
+            refresh_semantic_model(
+                dataset=dataset_id, tables=table_name, workspace=workspace_id
+            )

sempy_labs/directlake/_warm_cache.py

@@ -0,0 +1,236 @@
+import sempy.fabric as fabric
+import pandas as pd
+from tqdm.auto import tqdm
+import numpy as np
+import time
+from .._helper_functions import (
+    format_dax_object_name,
+    resolve_dataset_name_and_id,
+    resolve_workspace_name_and_id,
+)
+from .._refresh_semantic_model import refresh_semantic_model
+from .._model_dependencies import get_measure_dependencies
+from typing import Optional
+from sempy._utils._log import log
+import sempy_labs._icons as icons
+from uuid import UUID
+
+
+@log
+def warm_direct_lake_cache_perspective(
+    dataset: str | UUID,
+    perspective: str,
+    add_dependencies: bool = False,
+    workspace: Optional[str | UUID] = None,
+) -> pd.DataFrame:
+    """
+    Warms the cache of a Direct Lake semantic model by running a simple DAX query against the columns in a perspective.
+
+    Parameters
+    ----------
+    dataset : str | uuid.UUID
+        Name or ID of the semantic model.
+    perspective : str
+        Name of the perspective which contains objects to be used for warming the cache.
+    add_dependencies : bool, default=False
+        Includes object dependencies in the cache warming process.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    pandas.DataFrame
+        Returns a pandas dataframe showing the columns that have been put into memory.
+    """
+
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    (dataset_name, dataset_id) = resolve_dataset_name_and_id(dataset, workspace_id)
+
+    fabric.refresh_tom_cache(workspace=workspace)
+
+    dfP = fabric.list_partitions(dataset=dataset_id, workspace=workspace_id)
+    if not any(r["Mode"] == "DirectLake" for _, r in dfP.iterrows()):
+        raise ValueError(
+            f"{icons.red_dot} The '{dataset_name}' semantic model in the '{workspace_name}' workspace is not in Direct Lake mode. This function is specifically for semantic models in Direct Lake mode."
+        )
+
+    dfPersp = fabric.list_perspectives(dataset=dataset_id, workspace=workspace_id)
+    dfPersp["DAX Object Name"] = format_dax_object_name(
+        dfPersp["Table Name"], dfPersp["Object Name"]
+    )
+    dfPersp_filt = dfPersp[dfPersp["Perspective Name"] == perspective]
+
+    if len(dfPersp_filt) == 0:
+        raise ValueError(
+            f"{icons.red_dot} The '{perspective} perspective does not exist or contains no objects within the '{dataset}' semantic model in the '{workspace}' workspace."
+        )
+
+    dfPersp_c = dfPersp_filt[dfPersp_filt["Object Type"] == "Column"]
+
+    column_values = dfPersp_c["DAX Object Name"].tolist()
+
+    if add_dependencies:
+        # Measure dependencies
+        md = get_measure_dependencies(dataset_id, workspace_id)
+        md["Referenced Full Object"] = format_dax_object_name(
+            md["Referenced Table"], md["Referenced Object"]
+        )
+        dfPersp_m = dfPersp_filt[(dfPersp_filt["Object Type"] == "Measure")]
+        md_filt = md[
+            (md["Object Name"].isin(dfPersp_m["Object Name"].values))
+            & (md["Referenced Object Type"] == "Column")
+        ]
+        measureDep = md_filt["Referenced Full Object"].unique()
+
+        # Hierarchy dependencies
+        dfPersp_h = dfPersp_filt[(dfPersp_filt["Object Type"] == "Hierarchy")]
+        dfH = fabric.list_hierarchies(dataset=dataset_id, workspace=workspace_id)
+        dfH["Hierarchy Object"] = format_dax_object_name(
+            dfH["Table Name"], dfH["Hierarchy Name"]
+        )
+        dfH["Column Object"] = format_dax_object_name(
+            dfH["Table Name"], dfH["Column Name"]
+        )
+        dfH_filt = dfH[
+            dfH["Hierarchy Object"].isin(dfPersp_h["DAX Object Name"].values)
+        ]
+        hierarchyDep = dfH_filt["Column Object"].unique()
+
+        # Relationship dependencies
+        unique_table_names = dfPersp_filt["Table Name"].unique()
+        dfR = fabric.list_relationships(dataset=dataset_id, workspace=workspace_id)
+        dfR["From Object"] = format_dax_object_name(
+            dfR["From Table"], dfR["From Column"]
+        )
+        dfR["To Object"] = format_dax_object_name(dfR["To Table"], dfR["To Column"])
+        filtered_dfR = dfR[
+            dfR["From Table"].isin(unique_table_names)
+            & dfR["To Table"].isin(unique_table_names)
+        ]
+
+        fromObjects = filtered_dfR["From Object"].unique()
+        toObjects = filtered_dfR["To Object"].unique()
+
+        merged_list = np.concatenate(
+            [column_values, measureDep, hierarchyDep, fromObjects, toObjects]
+        )
+        merged_list_unique = list(set(merged_list))
+
+    else:
+        merged_list_unique = column_values
+
+    df = pd.DataFrame(merged_list_unique, columns=["DAX Object Name"])
+    df[["Table Name", "Column Name"]] = df["DAX Object Name"].str.split(
+        "[", expand=True
+    )
+    df["Table Name"] = df["Table Name"].str[1:-1]
+    df["Column Name"] = df["Column Name"].str[0:-1]
+
+    return _put_columns_into_memory(dataset=dataset, workspace=workspace, col_df=df)
+
+
+@log
+def warm_direct_lake_cache_isresident(
+    dataset: str | UUID, workspace: Optional[str | UUID] = None
+) -> pd.DataFrame:
+    """
+    Performs a refresh on the semantic model and puts the columns which were in memory prior to the refresh back into memory.
+
+    Parameters
+    ----------
+    dataset : str | uuid.UUID
+        Name or ID of the semantic model.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    pandas.DataFrame
+        Returns a pandas dataframe showing the columns that have been put into memory.
+    """
+
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    (dataset_name, dataset_id) = resolve_dataset_name_and_id(dataset, workspace_id)
+
+    dfP = fabric.list_partitions(dataset=dataset_id, workspace=workspace_id)
+    if not any(r["Mode"] == "DirectLake" for _, r in dfP.iterrows()):
+        raise ValueError(
+            f"{icons.red_dot} The '{dataset_name}' semantic model in the '{workspace_name}' workspace is not in Direct Lake mode. This function is specifically for semantic models in Direct Lake mode."
+        )
+
+    # Identify columns which are currently in memory (Is Resident = True)
+    dfC = fabric.list_columns(dataset=dataset_id, workspace=workspace_id, extended=True)
+    dfC_filtered = dfC[dfC["Is Resident"] == True]
+
+    if len(dfC_filtered) == 0:
+        raise ValueError(
+            f"{icons.yellow_dot} At present, no columns are in memory in the '{dataset_name}' semantic model in the '{workspace_name}' workspace."
+        )
+
+    # Refresh/frame dataset
+    refresh_semantic_model(
+        dataset=dataset_id, refresh_type="full", workspace=workspace_id
+    )
+    time.sleep(2)
+
+    return _put_columns_into_memory(
+        dataset=dataset, workspace=workspace, col_df=dfC_filtered
+    )
+
+
+@log
+def _put_columns_into_memory(dataset, workspace, col_df, return_dataframe: bool = True):
+
+    row_limit = 1000000
+
+    dfT = fabric.list_tables(dataset=dataset, workspace=workspace, extended=True)
+    col_df = col_df.copy()
+
+    col_df["DAX Object"] = format_dax_object_name(
+        col_df["Table Name"], col_df["Column Name"]
+    )
+    tbls = col_df["Table Name"].unique()
+
+    for table_name in (bar := tqdm(tbls)):
+        dfT_filt = dfT[dfT["Name"] == table_name]
+        col_df_filt = col_df[col_df["Table Name"] == table_name]
+        if not dfT_filt.empty:
+            row_count = dfT_filt["Row Count"].iloc[0]
+            bar.set_description(f"Warming the '{table_name}' table...")
+            if pd.isna(row_count):
+                pass
+            elif row_count < row_limit:
+                columns = col_df_filt["DAX Object"].tolist()
+                css = ", ".join(columns)
+                dax = f"EVALUATE TOPN(1, SELECTCOLUMNS('{table_name}', {css}))"
+                fabric.evaluate_dax(
+                    dataset=dataset, dax_string=dax, workspace=workspace
+                )
+            else:
+                for _, r in col_df_filt.iterrows():
+                    dax_object = r["DAX Object"]
+                    dax = f"""EVALUATE TOPN(1, SELECTCOLUMNS('{table_name}', {dax_object}))"""
+                    fabric.evaluate_dax(
+                        dataset=dataset, dax_string=dax, workspace=workspace
+                    )
+
+    if return_dataframe:
+        print(
+            f"{icons.green_dot} The following columns have been put into memory. Temperature indicates the current column temperature."
+        )
+
+        dfC = fabric.list_columns(dataset=dataset, workspace=workspace, extended=True)
+        dfC["DAX Object"] = format_dax_object_name(
+            dfC["Table Name"], dfC["Column Name"]
+        )
+        dfC_filt = dfC[dfC["DAX Object"].isin(col_df["DAX Object"].values)]
+
+        return (
+            dfC_filt[["Table Name", "Column Name", "Is Resident", "Temperature"]]
+            .sort_values(by=["Table Name", "Column Name"], ascending=True)
+            .reset_index(drop=True)
+        )

sempy_labs/dotnet_lib/dotnet.runtime.config.json

@@ -0,0 +1,10 @@
+{
+  "runtimeOptions": {
+    "tfm": "net6.0",
+    "framework": {
+      "name": "Microsoft.NETCore.App",
+      "version": "6.0.0"
+    },
+    "rollForward": "Major"
+  }
+}

sempy_labs/environment/__init__.py

@@ -0,0 +1,23 @@
+from ._items import (
+    list_environments,
+    create_environment,
+    delete_environment,
+    publish_environment,
+)
+from ._pubstage import (
+    get_published_spark_compute,
+    get_staging_spark_compute,
+    list_published_libraries,
+    list_staging_libraries,
+)
+
+__all__ = [
+    "list_environments",
+    "create_environment",
+    "delete_environment",
+    "publish_environment",
+    "get_published_spark_compute",
+    "get_staging_spark_compute",
+    "list_published_libraries",
+    "list_staging_libraries",
+]