semantic-link-labs 0.11.2__py3-none-any.whl → 0.12.0__py3-none-any.whl

sempy_labs/_workloads.py

@@ -1,7 +1,7 @@
 import pandas as pd
 from typing import Optional
 import sempy_labs._icons as icons
-from ._helper_functions import (
+from sempy_labs._helper_functions import (
     _update_dataframe_datatypes,
     _base_api,
     _create_dataframe,

sempy_labs/_workspace_identity.py

@@ -1,4 +1,4 @@
-from ._helper_functions import (
+from sempy_labs._helper_functions import (
     resolve_workspace_name_and_id,
     _base_api,
 )

sempy_labs/_workspaces.py

@@ -1,7 +1,8 @@
 import pandas as pd
 import sempy_labs._icons as icons
-from typing import Optional
-from ._helper_functions import (
+from typing import Optional, Literal
+from sempy_labs._helper_functions import (
+    resolve_workspace_id,
     resolve_workspace_name_and_id,
     resolve_capacity_id,
     _base_api,
@@ -127,7 +128,7 @@ def list_workspace_users(workspace: Optional[str | UUID] = None) -> pd.DataFrame
         A pandas dataframe the users of a workspace and their properties.
     """
 
-    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    workspace_id = resolve_workspace_id(workspace)
 
     columns = {
         "User Name": "string",
@@ -176,6 +177,8 @@ def add_user_to_workspace(
 
     This is a wrapper function for the following API: `Groups - Add Group User <https://learn.microsoft.com/rest/api/power-bi/groups/add-group-user>`_.
 
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
     Parameters
     ----------
     email_address : str
@@ -217,6 +220,7 @@ def add_user_to_workspace(
         request=f"/v1.0/myorg/groups/{workspace_id}/users",
         method="post",
         payload=payload,
+        client="fabric_sp",
     )
     print(
         f"{icons.green_dot} The '{email_address}' user has been added as a{plural} '{role_name}' within the '{workspace_name}' workspace."
@@ -234,6 +238,8 @@ def assign_workspace_to_capacity(
 
     This is a wrapper function for the following API: `Workspaces - Assign To Capacity <https://learn.microsoft.com/rest/api/fabric/core/workspaces/assign-to-capacity>`_.
 
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
     Parameters
     ----------
     capacity : str | uuid.UUID
@@ -260,6 +266,7 @@ def assign_workspace_to_capacity(
         method="post",
         payload=payload,
         status_codes=[200, 202],
+        client="fabric_sp",
     )
     print(
         f"{icons.green_dot} The '{workspace_name}' workspace has been assigned to the '{capacity}' capacity."
@@ -320,7 +327,7 @@ def list_workspace_role_assignments(
         A pandas dataframe showing the members of a given workspace and their roles.
     """
 
-    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    workspace_id = resolve_workspace_id(workspace)
 
     columns = {
         "User Name": "string",
@@ -336,16 +343,23 @@ def list_workspace_role_assignments(
         client="fabric_sp",
     )
 
+    rows = []
     for r in responses:
         for i in r.get("value", []):
             principal = i.get("principal", {})
-            new_data = {
-                "User Name": principal.get("displayName"),
-                "Role Name": i.get("role"),
-                "Type": principal.get("type"),
-                "User Email": principal.get("userDetails", {}).get("userPrincipalName"),
-            }
-            df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
+            rows.append(
+                {
+                    "User Name": principal.get("displayName"),
+                    "Role Name": i.get("role"),
+                    "Type": principal.get("type"),
+                    "User Email": principal.get("userDetails", {}).get(
+                        "userPrincipalName"
+                    ),
+                }
+            )
+
+    if rows:
+        df = pd.DataFrame(rows, columns=list(columns.keys()))
 
     return df
 
@@ -357,6 +371,8 @@ def delete_workspace(workspace: Optional[str | UUID] = None):
 
     This is a wrapper function for the following API: `Workspaces - Delete Workspace <https://learn.microsoft.com/rest/api/fabric/core/workspaces/delete-workspace>`_.
 
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
     Parameters
     ----------
     workspace : str | uuid.UUID, default=None
@@ -372,3 +388,121 @@ def delete_workspace(workspace: Optional[str | UUID] = None):
     )
 
     print(f"{icons.green_dot} The '{workspace_name}' workspace has been deleted.")
+
+
+@log
+def get_workspace_network_communication_policy(
+    workspace: Optional[str | UUID] = None,
+) -> pd.DataFrame:
+    """
+    Returns networking communication policy for the specified workspace. This feature is currently in preview.
+
+    This is a wrapper function for the following API: `Workspaces - Get Network Communication Policy <https://learn.microsoft.com/rest/api/fabric/core/workspaces/get-network-communication-policy>`_.
+
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
+    Parameters
+    ----------
+    workspace : str | uuid.UUID, default=None
+        The workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing the networking communication policy for the specified workspace.
+    """
+
+    workspace_id = resolve_workspace_id(workspace)
+
+    columns = {
+        "Inbound Public Access Rules": "string",
+        "Outbound Public Access Rules": "string",
+    }
+
+    df = _create_dataframe(columns=columns)
+
+    data = _base_api(
+        request=f"/v1/workspaces/{workspace_id}/networking/communicationPolicy",
+        client="fabric_sp",
+    ).json()
+
+    if data:
+        df = pd.DataFrame(
+            [
+                {
+                    "Inbound Public Access Rules": data.get("inbound", {})
+                    .get("publicAccessRules", {})
+                    .get("defaultAction"),
+                    "Outbound Public Access Rules": data.get("outbound", {})
+                    .get("publicAccessRules", {})
+                    .get("defaultAction"),
+                }
+            ]
+        )
+
+    return df
+
+
+@log
+def set_workspace_network_communication_policy(
+    inbound_policy: Literal["Allow", "Deny"],
+    outbound_policy: Literal["Allow", "Deny"],
+    workspace: Optional[str | UUID] = None,
+):
+    """
+    Sets networking communication policy for the specified workspace. This API uses the PUT method and will overwrite all settings. Remaining policy will be set to default value if only partial policy is provided in the request body. Always run Get Network Communication Policy first and provide full policy in the request body. This feature is currently in preview.
+
+    This is a wrapper function for the following API: `Workspaces - Set Network Communication Policy <https://learn.microsoft.com/rest/api/fabric/core/workspaces/set-network-communication-policy>`_.
+
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
+    Parameters
+    ----------
+    inbound_policy : Literal['Allow', 'Deny']
+        The policy for all inbound communications to a workspace.
+    outbound_policy : Literal['Allow', 'Deny']
+        The policy for all outbound communications to a workspace.
+    workspace : str | uuid.UUID, default=None
+        The workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+
+    inbound_policy = inbound_policy.capitalize()
+    outbound_policy = outbound_policy.capitalize()
+
+    if inbound_policy not in ["Allow", "Deny"]:
+        raise ValueError(
+            f"{icons.red_dot} The 'inbound_policy' must be either 'Allow' or 'Deny'."
+        )
+    if outbound_policy not in ["Allow", "Deny"]:
+        raise ValueError(
+            f"{icons.red_dot} The 'outbound_policy' must be either 'Allow' or 'Deny'."
+        )
+
+    payload = {
+        "inbound": {
+            "publicAccessRules": {
+                "defaultAction": inbound_policy,
+            }
+        },
+        "outbound": {
+            "publicAccessRules": {
+                "defaultAction": outbound_policy,
+            }
+        },
+    }
+    _base_api(
+        request=f"/v1/workspaces/{workspace_id}/networking/communicationPolicy",
+        client="fabric_sp",
+        payload=payload,
+        method="put",
+    )
+
+    print(
+        f"{icons.green_dot} The networking communication policy has been updated for the '{workspace_name}' workspace."
+    )

sempy_labs/admin/__init__.py

@@ -90,6 +90,10 @@ from ._tags import (
     create_tags,
     delete_tag,
 )
+from ._tenant_keys import (
+    list_tenant_keys,
+    rotate_tenant_key,
+)
 
 __all__ = [
     "list_items",
@@ -149,4 +153,6 @@ __all__ = [
     "list_tags",
     "create_tags",
     "delete_tag",
+    "list_tenant_keys",
+    "rotate_tenant_key",
 ]

sempy_labs/admin/_capacities.py

@@ -3,7 +3,7 @@ from uuid import UUID
 import sempy_labs._icons as icons
 from typing import Optional, Tuple
 from sempy._utils._log import log
-from .._helper_functions import (
+from sempy_labs._helper_functions import (
     _base_api,
     _build_url,
     _create_dataframe,
@@ -226,6 +226,7 @@ def get_capacity_state(capacity: Optional[str | UUID] = None):
 @log
 def list_capacities(
     capacity: Optional[str | UUID] = None,
+    include_tenant_key: bool = False,
 ) -> pd.DataFrame:
     """
     Shows the a list of capacities and their properties.
@@ -238,6 +239,8 @@ def list_capacities(
     ----------
     capacity : str | uuid.UUID, default=None
         The capacity name or ID to filter. If None, all capacities are returned.
+    include_tenant_key : bool, default=False
+        If True, obtains the `tenant key <https://learn.microsoft.com/rest/api/power-bi/admin/get-capacities-as-admin#example-with-expand-on-tenant-key>`_ properties.
 
     Returns
     -------
@@ -252,27 +255,47 @@ def list_capacities(
         "Region": "string",
         "State": "string",
         "Admins": "list",
+        "Users": "list",
     }
+    if include_tenant_key:
+        columns.update(
+            {
+                "Tenant Key Id": "string",
+                "Tenant Key Name": "string",
+            }
+        )
     df = _create_dataframe(columns=columns)
 
-    responses = _base_api(
-        request="/v1.0/myorg/admin/capacities", client="fabric_sp", uses_pagination=True
-    )
+    url = "/v1.0/myorg/admin/capacities"
+    if include_tenant_key:
+        url += "?$expand=tenantKey"
+
+    responses = _base_api(request=url, client="fabric_sp", uses_pagination=True)
 
     rows = []
     for r in responses:
         for i in r.get("value", []):
-            rows.append(
+            row = {
+                "Capacity Id": i.get("id", "").lower(),
+                "Capacity Name": i.get("displayName"),
+                "Sku": i.get("sku"),
+                "Region": i.get("region"),
+                "State": i.get("state"),
+                "Admins": i.get("admins", []),
+                "Users": i.get("users", []),
+            }
+
+        if include_tenant_key:
+            tenant_key = i.get("tenantKey") or {}
+            row.update(
                 {
-                    "Capacity Id": i.get("id").lower(),
-                    "Capacity Name": i.get("displayName"),
-                    "Sku": i.get("sku"),
-                    "Region": i.get("region"),
-                    "State": i.get("state"),
-                    "Admins": [i.get("admins", [])],
+                    "Tenant Key Id": tenant_key.get("id"),
+                    "Tenant Key Name": tenant_key.get("name"),
                 }
             )
 
+        rows.append(row)
+
     if rows:
         df = pd.DataFrame(rows, columns=list(columns.keys()))
 

sempy_labs/admin/_items.py

@@ -228,8 +228,8 @@ def list_item_access_details(
             f"{icons.red_dot} The parameter 'item' and 'type' are mandatory."
         )
 
-    workspace_name, workspace_id = _resolve_workspace_name_and_id(workspace)
-    item_name, item_id = _resolve_item_name_and_id(
+    (workspace_name, workspace_id) = _resolve_workspace_name_and_id(workspace)
+    (item_name, item_id) = _resolve_item_name_and_id(
         item=item, type=type, workspace=workspace_name
     )
 

sempy_labs/admin/_tenant_keys.py

@@ -0,0 +1,89 @@
+import pandas as pd
+from sempy_labs._helper_functions import (
+    _create_dataframe,
+    _update_dataframe_datatypes,
+    _base_api,
+)
+from sempy._utils._log import log
+from uuid import UUID
+import sempy_labs._icons as icons
+
+
+@log
+def list_tenant_keys() -> pd.DataFrame:
+    """
+    Returns the encryption keys for the tenant.
+
+    This is a wrapper function for the following API: `Admin - Get Power BI Encryption Keys <https://learn.microsoft.com/rest/api/power-bi/admin/get-power-bi-encryption-keys>`_.
+
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing the encryption keys for the tenant.
+    """
+
+    columns = {
+        "Tenant Key Id": "str",
+        "Tenant Key Name": "str",
+        "Key Vault Key Identifier": "str",
+        "Is Default": "bool",
+        "Created At": "datetime",
+        "Updated At": "datetime",
+    }
+
+    df = _create_dataframe(columns=columns)
+
+    result = _base_api(
+        request="/v1.0/myorg/admin/tenantKeys", client="fabric_sp"
+    ).json()
+
+    rows = []
+    for i in result.get("value", []):
+        rows.append(
+            {
+                "Tenant Key Id": i.get("id"),
+                "Tenant Key Name": i.get("name"),
+                "Key Vault Key Identifier": i.get("keyVaultKeyIdentifier"),
+                "Is Default": i.get("isDefault"),
+                "Created At": i.get("createdAt"),
+                "Updated At": i.get("updatedAt"),
+            }
+        )
+
+    if rows:
+        df = pd.DataFrame(rows, columns=list(columns.keys()))
+        _update_dataframe_datatypes(dataframe=df, column_map=columns)
+
+    return df
+
+
+@log
+def rotate_tenant_key(tenant_key_id: UUID, key_vault_key_identifier: str):
+    """
+    Rotate the encryption key for Power BI workspaces assigned to a capacity.
+
+    This is a wrapper function for the following API: `Admin - Rotate Power BI Encryption Key <https://learn.microsoft.com/rest/api/power-bi/admin/rotate-power-bi-encryption-key>`_.
+
+    Parameters
+    ----------
+    tenant_key_id : uuid.UUID
+        The tenant key ID
+    key_vault_key_identifier : str
+        The URI that uniquely specifies the encryption key in Azure Key Vault
+    """
+
+    payload = {
+        "keyVaultKeyIdentifier": key_vault_key_identifier,
+    }
+
+    _base_api(
+        request=f"/v1.0/myorg/admin/tenantKeys/{tenant_key_id}/Default.Rotate",
+        method="post",
+        payload=payload,
+    )
+
+    print(
+        f"{icons.green_dot} The encryption key for tenant {tenant_key_id} has been rotated successfully using the '{key_vault_key_identifier}' Key Vault key identifier."
+    )

sempy_labs/directlake/_dl_helper.py

@@ -145,7 +145,7 @@ def generate_direct_lake_semantic_model(
     expr = generate_shared_expression(
         item_name=lakehouse, item_type="Lakehouse", workspace=lakehouse_workspace
     )
-    dfD = fabric.list_datasets(workspace=workspace_id)
+    dfD = fabric.list_datasets(workspace=workspace_id, mode="rest")
     dfD_filt = dfD[dfD["Dataset Name"] == dataset]
 
     if len(dfD_filt) > 0 and not overwrite:
@@ -225,7 +225,10 @@ def get_direct_lake_source(
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
     sql_endpoint_id = get_direct_lake_sql_endpoint(dataset=dataset, workspace=workspace)
     dfI = fabric.list_items(workspace=workspace)
-    dfI_filt = dfI[(dfI["Id"] == sql_endpoint_id) & (dfI["Type"] == "SQLEndpoint")]
+    dfI_filt = dfI[
+        (dfI["Id"] == sql_endpoint_id)
+        & (dfI["Type"].isin(["SQLEndpoint", "Warehouse"]))
+    ]
 
     artifact_type, artifact_name, artifact_id = None, None, None
 

sempy_labs/graph/_users.py

@@ -137,7 +137,6 @@ def send_mail(
     cc_recipients: Optional[str | List[str]] = None,
     bcc_recipients: Optional[str | List[str]] = None,
     priority: Literal["Normal", "High", "Low"] = "Normal",
-    follow_up_flag: bool = False,
     attachments: Optional[str | List[str]] = None,
 ):
     """
@@ -165,8 +164,6 @@ def send_mail(
         The email address of the BCC recipients.
     priority : Literal["Normal", "High", "Low"], default="Normal"
         The email priority.
-    follow_up_flag : bool, default=False
-        Whether to set a follow-up flag for the email.
     attachments : str | List[str], default=None
         The abfss path or a list of the abfss paths of the attachments to include in the email.
     """
@@ -220,8 +217,8 @@ def send_mail(
     if bcc_email_addresses:
         payload["message"]["bccRecipients"] = bcc_email_addresses
 
-    if follow_up_flag:
-        payload["message"]["flag"] = {"flagStatus": "flagged"}
+    # if follow_up_flag:
+    #    payload["message"]["flag"] = {"flagStatus": "flagged"}
 
     content_types = {
         ".txt": "text/plain",
@@ -244,6 +241,7 @@ def send_mail(
         ".pbip": "application/vnd.ms-powerbi.report",
         ".pbit": "application/vnd.ms-powerbi.report",
         ".vpax": "application/zip",
+        ".geojson": "application/geo+json",
     }
 
     def file_path_to_content_bytes(file_path):

sempy_labs/lakehouse/__init__.py

@@ -30,6 +30,9 @@ from ._helper import (
     update_lakehouse,
     load_table,
 )
+from ._materialized_lake_views import (
+    refresh_materialized_lake_views,
+)
 
 __all__ = [
     "get_lakehouse_columns",
@@ -50,4 +53,5 @@ __all__ = [
     "delete_lakehouse",
     "update_lakehouse",
     "load_table",
+    "refresh_materialized_lake_views",
 ]

sempy_labs/lakehouse/_helper.py

@@ -1,7 +1,7 @@
 from uuid import UUID
 from typing import Optional, Literal
 import pyarrow.dataset as ds
-from .._helper_functions import (
+from sempy_labs._helper_functions import (
     _mount,
     delete_item,
     _base_api,
@@ -68,14 +68,23 @@ def is_v_ordered(
         latest_file = os.path.join(delta_log_path, json_files[0])
 
         with open(latest_file, "r") as f:
-            for line in f:
-                try:
-                    data = json.loads(line)
-                    if "commitInfo" in data:
-                        tags = data["commitInfo"].get("tags", {})
-                        return tags.get("VORDER", "false").lower() == "true"
-                except json.JSONDecodeError:
-                    continue  # Skip malformed lines
+            all_data = [
+                json.loads(line) for line in f if line.strip()
+            ]  # one dict per line
+            for data in all_data:
+                if "metaData" in data:
+                    return (
+                        data.get("metaData", {})
+                        .get("configuration", {})
+                        .get("delta.parquet.vorder.enabled", "false")
+                        == "true"
+                    )
+
+            # If no metaData, fall back to commitInfo
+            for data in all_data:
+                if "commitInfo" in data:
+                    tags = data["commitInfo"].get("tags", {})
+                    return tags.get("VORDER", "false").lower() == "true"
 
         return False  # Default if not found
 

sempy_labs/lakehouse/_lakehouse.py

@@ -2,7 +2,7 @@ from tqdm.auto import tqdm
 from typing import List, Optional, Union
 from sempy._utils._log import log
 from uuid import UUID
-from .._helper_functions import (
+from sempy_labs._helper_functions import (
     _base_api,
     resolve_lakehouse_name_and_id,
     resolve_workspace_name_and_id,
@@ -13,7 +13,7 @@ import sempy_labs._icons as icons
 import re
 import time
 import pandas as pd
-from .._job_scheduler import (
+from sempy_labs._job_scheduler import (
     _get_item_job_instance,
 )
 
@@ -100,11 +100,15 @@ def optimize_lakehouse_tables(
         tables = [tables]
 
     df_tables = df_delta[df_delta["Table Name"].isin(tables)] if tables else df_delta
+    df_tables.reset_index(drop=True, inplace=True)
 
-    for _, r in (bar := tqdm(df_tables.iterrows())):
+    total = len(df_tables)
+    for idx, r in (bar := tqdm(df_tables.iterrows(), total=total, bar_format="{desc}")):
         table_name = r["Table Name"]
         path = r["Location"]
-        bar.set_description(f"Optimizing the '{table_name}' table...")
+        bar.set_description(
+            f"Optimizing the '{table_name}' table ({idx + 1}/{total})..."
+        )
         _optimize_table(path=path)
 
 
@@ -145,11 +149,13 @@ def vacuum_lakehouse_tables(
         tables = [tables]
 
     df_tables = df_delta[df_delta["Table Name"].isin(tables)] if tables else df_delta
+    df_tables.reset_index(drop=True, inplace=True)
 
-    for _, r in (bar := tqdm(df_tables.iterrows())):
+    total = len(df_tables)
+    for idx, r in (bar := tqdm(df_tables.iterrows(), total=total, bar_format="{desc}")):
         table_name = r["Table Name"]
         path = r["Location"]
-        bar.set_description(f"Vacuuming the '{table_name}' table...")
+        bar.set_description(f"Vacuuming the '{table_name}' table ({idx}/{total})...")
         _vacuum_table(path=path, retain_n_hours=retain_n_hours)
 
 
@@ -231,7 +237,7 @@ def run_table_maintenance(
     if optimize:
         payload["executionData"]["optimizeSettings"] = {}
     if v_order:
-        payload["executionData"]["optimizeSettings"] = {"vorder": True}
+        payload["executionData"]["optimizeSettings"] = {"vOrder": True}
     if vacuum:
         payload["executionData"]["vacuumSettings"] = {}
     if vacuum and retention_period is not None:
@@ -242,16 +248,19 @@ def run_table_maintenance(
         method="post",
         payload=payload,
         status_codes=202,
+        client="fabric_sp",
     )
 
-    f"{icons.in_progress} The table maintenance job for the '{table_name}' table in the '{lakehouse_name}' lakehouse within the '{workspace_name}' workspace has been initiated."
+    print(
+        f"{icons.in_progress} The table maintenance job for the '{table_name}' table in the '{lakehouse_name}' lakehouse within the '{workspace_name}' workspace has been initiated."
+    )
 
     status_url = response.headers.get("Location").split("fabric.microsoft.com")[1]
     status = None
     while status not in ["Completed", "Failed"]:
         response = _base_api(request=status_url)
         status = response.json().get("status")
-        time.sleep(10)
+        time.sleep(3)
 
     df = _get_item_job_instance(url=status_url)