semantic-link-labs 0.8.9__py3-none-any.whl → 0.8.11__py3-none-any.whl

sempy_labs/_connections.py

@@ -5,6 +5,7 @@ from typing import Optional
 from sempy_labs._helper_functions import (
     pagination,
     _is_valid_uuid,
+    resolve_workspace_name_and_id,
 )
 from uuid import UUID
 import sempy_labs._icons as icons
@@ -19,7 +20,7 @@ def delete_connection(connection: str | UUID):
 
     Parameters
     ----------
-    connection : str | UUID
+    connection : str | uuid.UUID
         The connection name or ID.
     """
 
@@ -42,9 +43,9 @@ def delete_connection_role_assignment(connection: str | UUID, role_assignment_id
 
     Parameters
     ----------
-    connection : str | UUID
+    connection : str | uuid.UUID
         The connection name or ID.
-    role_assignment_id : UUID
+    role_assignment_id : uuid.UUID
         The role assignment ID.
     """
 
@@ -87,7 +88,7 @@ def list_connection_role_assignments(connection: str | UUID) -> pd.DataFrame:
 
     Parameters
     ----------
-    connection : str | UUID
+    connection : str | uuid.UUID
         The connection name or ID.
 
     Returns
@@ -205,7 +206,7 @@ def list_connections() -> pd.DataFrame:
 
 
 def list_item_connections(
-    item_name: str, item_type: str, workspace: Optional[str] = None
+    item_name: str, item_type: str, workspace: Optional[str | UUID] = None
 ) -> pd.DataFrame:
     """
     Shows the list of connections that the specified item is connected to.
@@ -218,8 +219,8 @@ def list_item_connections(
         The item name.
     item_type : str
         The `item type <https://learn.microsoft.com/rest/api/fabric/core/items/update-item?tabs=HTTP#itemtype>`_.
-    workspace : str, default=None
-        The Fabric workspace name.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
 
@@ -229,11 +230,10 @@ def list_item_connections(
         A pandas dataframe showing the list of connections that the specified item is connected to.
     """
 
-    workspace = fabric.resolve_workspace_name(workspace)
-    workspace_id = fabric.resolve_workspace_id(workspace)
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
     item_type = item_type[0].upper() + item_type[1:]
     item_id = fabric.resolve_item_id(
-        item_name=item_name, type=item_type, workspace=workspace
+        item_name=item_name, type=item_type, workspace=workspace_id
     )
 
     client = fabric.FabricRestClient()
@@ -416,7 +416,7 @@ def create_on_prem_connection(
     ----------
     name : str
         The name of the connection.
-    gateway : str | UUID
+    gateway : str | uuid.UUID
         The name or Id of the gateway.
     server_name : str
         The name of the server.
@@ -485,7 +485,7 @@ def create_vnet_connection(
     user_name: str,
     password: str,
     privacy_level: str,
-    connection_encryption: Optional[str] = "NotEncrypted",
+    connection_encryption: str = "NotEncrypted",
     skip_test_connection: bool = False,
 ):
     """
@@ -497,7 +497,7 @@ def create_vnet_connection(
     ----------
     name : str
         The name of the connection.
-    gateway : str | UUID
+    gateway : str | uuid.UUID
         The name or Id of the gateway.
     server_name : str
         The name of the server.

sempy_labs/_data_pipelines.py

@@ -9,9 +9,10 @@ from sempy_labs._helper_functions import (
     _decode_b64,
 )
 from sempy.fabric.exceptions import FabricHTTPException
+from uuid import UUID
 
 
-def list_data_pipelines(workspace: Optional[str] = None) -> pd.DataFrame:
+def list_data_pipelines(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
     """
     Shows the data pipelines within a workspace.
 
@@ -19,8 +20,8 @@ def list_data_pipelines(workspace: Optional[str] = None) -> pd.DataFrame:
 
     Parameters
     ----------
-    workspace : str, default=None
-        The Fabric workspace name.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
 
@@ -32,7 +33,7 @@ def list_data_pipelines(workspace: Optional[str] = None) -> pd.DataFrame:
 
     df = pd.DataFrame(columns=["Data Pipeline Name", "Data Pipeline ID", "Description"])
 
-    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
 
     client = fabric.FabricRestClient()
     response = client.get(f"/v1/workspaces/{workspace_id}/dataPipelines")
@@ -54,7 +55,7 @@ def list_data_pipelines(workspace: Optional[str] = None) -> pd.DataFrame:
 
 
 def create_data_pipeline(
-    name: str, description: Optional[str] = None, workspace: Optional[str] = None
+    name: str, description: Optional[str] = None, workspace: Optional[str | UUID] = None
 ):
     """
     Creates a Fabric data pipeline.
@@ -67,13 +68,13 @@ def create_data_pipeline(
         Name of the data pipeline.
     description : str, default=None
         A description of the environment.
-    workspace : str, default=None
-        The Fabric workspace name.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
     """
 
-    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
 
     request_body = {"displayName": name}
 
@@ -88,11 +89,11 @@ def create_data_pipeline(
     lro(client, response, status_codes=[201, 202])
 
     print(
-        f"{icons.green_dot} The '{name}' data pipeline has been created within the '{workspace}' workspace."
+        f"{icons.green_dot} The '{name}' data pipeline has been created within the '{workspace_name}' workspace."
     )
 
 
-def delete_data_pipeline(name: str, workspace: Optional[str] = None):
+def delete_data_pipeline(name: str, workspace: Optional[str | UUID] = None):
     """
     Deletes a Fabric data pipeline.
 
@@ -102,16 +103,16 @@ def delete_data_pipeline(name: str, workspace: Optional[str] = None):
     ----------
     name: str
         Name of the data pipeline.
-    workspace : str, default=None
+    workspace : str | uuid.UUID, default=None
         The Fabric workspace name.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
     """
 
-    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
 
     item_id = fabric.resolve_item_id(
-        item_name=name, type="DataPipeline", workspace=workspace
+        item_name=name, type="DataPipeline", workspace=workspace_id
     )
 
     client = fabric.FabricRestClient()
@@ -121,12 +122,12 @@ def delete_data_pipeline(name: str, workspace: Optional[str] = None):
         raise FabricHTTPException(response)
 
     print(
-        f"{icons.green_dot} The '{name}' data pipeline within the '{workspace}' workspace has been deleted."
+        f"{icons.green_dot} The '{name}' data pipeline within the '{workspace_name}' workspace has been deleted."
     )
 
 
 def get_data_pipeline_definition(
-    name: str, workspace: Optional[str] = None, decode: bool = True
+    name: str, workspace: Optional[str | UUID] = None, decode: bool = True
 ) -> dict | pd.DataFrame:
     """
     Obtains the definition of a data pipeline.
@@ -135,8 +136,8 @@ def get_data_pipeline_definition(
     ----------
     name : str
         The name of the data pipeline.
-    workspace : str, default=None
-        The Fabric workspace name.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
     decode : bool, default=True
@@ -150,10 +151,9 @@ def get_data_pipeline_definition(
         A pandas dataframe showing the data pipelines within a workspace.
     """
 
-    workspace = fabric.resolve_workspace_name(workspace)
-    workspace_id = fabric.resolve_workspace_id(workspace)
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
     item_id = fabric.resolve_item_id(
-        item_name=name, type="DataPipeline", workspace=workspace
+        item_name=name, type="DataPipeline", workspace=workspace_id
     )
 
     client = fabric.FabricRestClient()

sempy_labs/_dataflows.py

@@ -10,14 +10,14 @@ from sempy.fabric.exceptions import FabricHTTPException
 from uuid import UUID
 
 
-def list_dataflows(workspace: Optional[str] = None):
+def list_dataflows(workspace: Optional[str | UUID] = None):
     """
     Shows a list of all dataflows which exist within a workspace.
 
     Parameters
     ----------
-    workspace : str, default=None
-        The Fabric workspace name.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
 
@@ -27,7 +27,7 @@ def list_dataflows(workspace: Optional[str] = None):
         A pandas dataframe showing the dataflows which exist within a workspace.
     """
 
-    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
     client = fabric.PowerBIRestClient()
     response = client.get(f"/v1.0/myorg/groups/{workspace_id}/dataflows")
     if response.status_code != 200:
@@ -37,26 +37,27 @@ def list_dataflows(workspace: Optional[str] = None):
         columns=["Dataflow Id", "Dataflow Name", "Configured By", "Users", "Generation"]
     )
 
+    data = []  # Collect rows here
+
     for v in response.json().get("value", []):
         new_data = {
             "Dataflow Id": v.get("objectId"),
             "Dataflow Name": v.get("name"),
             "Configured By": v.get("configuredBy"),
-            "Users": [v.get("users")],
+            "Users": v.get("users", []),
             "Generation": v.get("generation"),
         }
-        df = pd.concat(
-            [df, pd.DataFrame(new_data, index=[0])],
-            ignore_index=True,
-        )
+        data.append(new_data)
 
-    df["Generation"] = df["Generation"].astype(int)
+    if data:
+        df = pd.DataFrame(data)
+        df["Generation"] = df["Generation"].astype(int)
 
     return df
 
 
 def assign_workspace_to_dataflow_storage(
-    dataflow_storage_account: str, workspace: Optional[str] = None
+    dataflow_storage_account: str, workspace: Optional[str | UUID] = None
 ):
     """
     Assigns a dataflow storage account to a workspace.
@@ -67,13 +68,13 @@ def assign_workspace_to_dataflow_storage(
     ----------
     dataflow_storage_account : str
         The name of the dataflow storage account.
-    workspace : str, default=None
-        The name of the workspace.
+    workspace : str | uuid.UUID, default=None
+        The name or ID of the workspace.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
     """
 
-    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
 
     df = list_dataflow_storage_accounts()
     df_filt = df[df["Dataflow Storage Account Name"] == dataflow_storage_account]
@@ -95,7 +96,7 @@ def assign_workspace_to_dataflow_storage(
     if response.status_code != 200:
         raise FabricHTTPException(response)
     print(
-        f"{icons.green_dot} The '{dataflow_storage_account}' dataflow storage account has been assigned to the '{workspace}' workspacce."
+        f"{icons.green_dot} The '{dataflow_storage_account}' dataflow storage account has been assigned to the '{workspace_name}' workspacce."
     )
 
 
@@ -137,7 +138,7 @@ def list_dataflow_storage_accounts() -> pd.DataFrame:
 
 
 def list_upstream_dataflows(
-    dataflow: str | UUID, workspace: Optional[str] = None
+    dataflow: str | UUID, workspace: Optional[str | UUID] = None
 ) -> pd.DataFrame:
     """
     Shows a list of upstream dataflows for the specified dataflow.
@@ -146,10 +147,10 @@ def list_upstream_dataflows(
 
     Parameters
     ----------
-    dataflow : str | UUID
+    dataflow : str | uuid.UUID
         Name or UUID of the dataflow.
-    workspace : str, default=None
-        The Fabric workspace name.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
 
@@ -159,10 +160,9 @@ def list_upstream_dataflows(
         A pandas dataframe showing a list of upstream dataflows for the specified dataflow.
     """
 
-    workspace_name = fabric.resolve_workspace_name(workspace)
-    workspace_id = fabric.resolve_workspace_id(workspace)
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
     (dataflow_name, dataflow_id) = _resolve_dataflow_name_and_id(
-        dataflow=dataflow, workspace=workspace
+        dataflow=dataflow, workspace=workspace_id
     )
     client = fabric.PowerBIRestClient()
 
@@ -194,7 +194,7 @@ def list_upstream_dataflows(
             tgt_workspace_id = v.get("groupId")
             tgt_workspace_name = fabric.resolve_workspace_name(tgt_workspace_id)
             (tgt_dataflow_name, _) = _resolve_dataflow_name_and_id(
-                dataflow=tgt_dataflow_id, workspace=tgt_workspace_name
+                dataflow=tgt_dataflow_id, workspace=tgt_workspace_id
             )
 
             df.loc[len(df)] = {
@@ -222,13 +222,12 @@ def list_upstream_dataflows(
 
 
 def _resolve_dataflow_name_and_id(
-    dataflow: str | UUID, workspace: Optional[str] = None
+    dataflow: str | UUID, workspace: Optional[str | UUID] = None
 ) -> Tuple[str, UUID]:
 
-    if workspace is None:
-        workspace = fabric.resolve_workspace_name(workspace)
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
 
-    dfD = list_dataflows(workspace=workspace)
+    dfD = list_dataflows(workspace=workspace_id)
 
     if _is_valid_uuid(dataflow):
         dfD_filt = dfD[dfD["Dataflow Id"] == dataflow]
@@ -237,7 +236,7 @@ def _resolve_dataflow_name_and_id(
 
     if len(dfD_filt) == 0:
         raise ValueError(
-            f"{icons.red_dot} The '{dataflow}' dataflow does not exist within the '{workspace}' workspace."
+            f"{icons.red_dot} The '{dataflow}' dataflow does not exist within the '{workspace_name}' workspace."
         )
 
     dataflow_id = dfD_filt["Dataflow Id"].iloc[0]

sempy_labs/_dax.py

@@ -1,22 +1,23 @@
 import sempy.fabric as fabric
 import pandas as pd
 from sempy_labs._helper_functions import (
-    resolve_dataset_id,
     resolve_workspace_name_and_id,
     format_dax_object_name,
+    resolve_dataset_name_and_id,
 )
 from sempy_labs._model_dependencies import get_model_calc_dependencies
 from typing import Optional, List
 from sempy._utils._log import log
-from tqdm.auto import tqdm
+from uuid import UUID
+from sempy_labs.directlake._warm_cache import _put_columns_into_memory
 
 
 @log
 def evaluate_dax_impersonation(
-    dataset: str,
+    dataset: str | UUID,
     dax_query: str,
     user_name: Optional[str] = None,
-    workspace: Optional[str] = None,
+    workspace: Optional[str | UUID] = None,
 ):
     """
     Runs a DAX query against a semantic model using the `REST API <https://learn.microsoft.com/en-us/rest/api/power-bi/datasets/execute-queries-in-group>`_.
@@ -26,14 +27,14 @@ def evaluate_dax_impersonation(
 
     Parameters
     ----------
-    dataset : str
-        Name of the semantic model.
+    dataset : str | uuid.UUID
+        Name or ID of the semantic model.
     dax_query : str
         The DAX query.
     user_name : str
         The user name (i.e. hello@goodbye.com).
-    workspace : str, default=None
-        The Fabric workspace name.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
 
@@ -43,8 +44,8 @@ def evaluate_dax_impersonation(
         A pandas dataframe holding the result of the DAX query.
     """
 
-    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
-    dataset_id = resolve_dataset_id(dataset=dataset, workspace=workspace)
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    (dataset_name, dataset_id) = resolve_dataset_name_and_id(dataset, workspace_id)
 
     request_body = {
         "queries": [{"query": dax_query}],
@@ -66,27 +67,27 @@ def evaluate_dax_impersonation(
 
 @log
 def get_dax_query_dependencies(
-    dataset: str,
+    dataset: str | UUID,
     dax_string: str | List[str],
     put_in_memory: bool = False,
     show_vertipaq_stats: bool = True,
-    workspace: Optional[str] = None,
+    workspace: Optional[str | UUID] = None,
 ) -> pd.DataFrame:
     """
     Obtains the columns on which a DAX query depends, including model dependencies. Shows Vertipaq statistics (i.e. Total Size, Data Size, Dictionary Size, Hierarchy Size) for easy prioritizing.
 
     Parameters
     ----------
-    dataset : str
-        Name of the semantic model.
+    dataset : str | uuid.UUID
+        Name or ID of the semantic model.
     dax_string : str | List[str]
         The DAX query or list of DAX queries.
     put_in_memory : bool, default=False
         If True, ensures that the dependent columns are put into memory in order to give realistic Vertipaq stats (i.e. Total Size etc.).
     show_vertipaq_stats : bool, default=True
         If True, shows vertipaq stats (i.e. Total Size, Data Size, Dictionary Size, Hierarchy Size)
-    workspace : str, default=None
-        The Fabric workspace name.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
 
@@ -96,15 +97,15 @@ def get_dax_query_dependencies(
         A pandas dataframe showing the dependent columns of a given DAX query including model dependencies.
     """
 
-    if workspace is None:
-        workspace = fabric.resolve_workspace_name(workspace)
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    (dataset_name, dataset_id) = resolve_dataset_name_and_id(dataset, workspace_id)
 
     if isinstance(dax_string, str):
         dax_string = [dax_string]
 
     final_df = pd.DataFrame(columns=["Object Type", "Table", "Object"])
 
-    cd = get_model_calc_dependencies(dataset=dataset, workspace=workspace)
+    cd = get_model_calc_dependencies(dataset=dataset_id, workspace=workspace_id)
 
     for dax in dax_string:
         # Escape quotes in dax
@@ -121,7 +122,7 @@ def get_dax_query_dependencies(
             RETURN all_dependencies
             """
         dep = fabric.evaluate_dax(
-            dataset=dataset, workspace=workspace, dax_string=final_query
+            dataset=dataset_id, workspace=workspace_id, dax_string=final_query
         )
 
         # Clean up column names and values (remove outside square brackets, underscorees in object type)
@@ -168,7 +169,7 @@ def get_dax_query_dependencies(
     final_df["Full Object"] = format_dax_object_name(
         final_df["Table Name"], final_df["Column Name"]
     )
-    dfC = fabric.list_columns(dataset=dataset, workspace=workspace, extended=True)
+    dfC = fabric.list_columns(dataset=dataset_id, workspace=workspace_id, extended=True)
     dfC["Full Object"] = format_dax_object_name(dfC["Table Name"], dfC["Column Name"])
 
     dfC_filtered = dfC[dfC["Full Object"].isin(final_df["Full Object"].values)][
@@ -188,32 +189,22 @@ def get_dax_query_dependencies(
         not_in_memory = dfC_filtered[dfC_filtered["Is Resident"] == False]
 
         if len(not_in_memory) > 0:
-            tbls = not_in_memory["Table Name"].unique()
-
-            # Run basic query to get columns into memory; completed one table at a time (so as not to overload the capacity)
-            for table_name in (bar := tqdm(tbls)):
-                bar.set_description(f"Warming the '{table_name}' table...")
-                css = ", ".join(
-                    not_in_memory[not_in_memory["Table Name"] == table_name][
-                        "Full Object"
-                    ]
-                    .astype(str)
-                    .tolist()
-                )
-                dax = f"""EVALUATE TOPN(1,SUMMARIZECOLUMNS({css}))"""
-                fabric.evaluate_dax(
-                    dataset=dataset, dax_string=dax, workspace=workspace
-                )
+            _put_columns_into_memory(
+                dataset=dataset,
+                workspace=workspace,
+                col_df=dfC_filtered,
+                return_dataframe=False,
+            )
 
             # Get column stats again
             dfC = fabric.list_columns(
-                dataset=dataset, workspace=workspace, extended=True
+                dataset=dataset_id, workspace=workspace_id, extended=True
             )
             dfC["Full Object"] = format_dax_object_name(
                 dfC["Table Name"], dfC["Column Name"]
             )
 
-            dfC_filtered = dfC[dfC["Full Object"].isin(df["Full Object"].values)][
+            dfC_filtered = dfC[dfC["Full Object"].isin(final_df["Full Object"].values)][
                 [
                     "Table Name",
                     "Column Name",
@@ -233,19 +224,19 @@ def get_dax_query_dependencies(
 
 @log
 def get_dax_query_memory_size(
-    dataset: str, dax_string: str, workspace: Optional[str] = None
+    dataset: str | UUID, dax_string: str, workspace: Optional[str | UUID] = None
 ) -> int:
     """
     Obtains the total size, in bytes, used by all columns that a DAX query depends on.
 
     Parameters
     ----------
-    dataset : str
-        Name of the semantic model.
+    dataset : str | uuid.UUID
+        Name or ID of the semantic model.
     dax_string : str
         The DAX query.
-    workspace : str, default=None
-        The Fabric workspace name.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
 
@@ -255,11 +246,14 @@ def get_dax_query_memory_size(
         The total size, in bytes, used by all columns that the DAX query depends on.
     """
 
-    if workspace is None:
-        workspace = fabric.resolve_workspace_name(workspace)
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    (dataset_name, dataset_id) = resolve_dataset_name_and_id(dataset, workspace_id)
 
     df = get_dax_query_dependencies(
-        dataset=dataset, workspace=workspace, dax_string=dax_string, put_in_memory=True
+        dataset=dataset_id,
+        workspace=workspace_id,
+        dax_string=dax_string,
+        put_in_memory=True,
     )
 
     return df["Total Size"].sum()