semantic-link-labs 0.9.10__py3-none-any.whl → 0.10.0__py3-none-any.whl

sempy_labs/directlake/_update_directlake_model_lakehouse_connection.py

@@ -7,7 +7,7 @@ from sempy_labs._helper_functions import (
 )
 from sempy._utils._log import log
 from sempy_labs.tom import connect_semantic_model
-from typing import Optional
+from typing import Optional, List
 import sempy_labs._icons as icons
 from uuid import UUID
 import re
@@ -19,7 +19,9 @@ def _extract_expression_list(expression):
     """
 
     pattern_sql = r'Sql\.Database\s*\(\s*"([^"]+)"\s*,\s*"([^"]+)"\s*\)'
-    pattern_no_sql = r'AzureDataLakeStorage\s*\{\s*"server".*?:\s*onelake\.dfs\.fabric\.microsoft\.com"\s*,\s*"path"\s*:\s*"/([\da-fA-F-]+)\s*/\s*([\da-fA-F-]+)\s*/"\s*\}'
+    pattern_no_sql = (
+        r'AzureStorage\.DataLake\(".*?/([0-9a-fA-F\-]{36})/([0-9a-fA-F\-]{36})"'
+    )
 
     match_sql = re.search(pattern_sql, expression)
     match_no_sql = re.search(pattern_no_sql, expression)
@@ -102,6 +104,7 @@ def update_direct_lake_model_connection(
     source_type: str = "Lakehouse",
     source_workspace: Optional[str | UUID] = None,
     use_sql_endpoint: bool = True,
+    tables: Optional[str | List[str]] = None,
 ):
     """
     Remaps a Direct Lake semantic model's SQL Endpoint connection to a new lakehouse/warehouse.
@@ -126,12 +129,19 @@ def update_direct_lake_model_connection(
     use_sql_endpoint : bool, default=True
         If True, the SQL Endpoint will be used for the connection.
         If False, Direct Lake over OneLake will be used.
+    tables : str | List[str], default=None
+        The name(s) of the table(s) to update in the Direct Lake semantic model.
+        If None, all tables will be updated (if there is only one expression).
+        If multiple tables are specified, they must be provided as a list.
     """
     if use_sql_endpoint:
         icons.sll_tags.append("UpdateDLConnection_SQL")
     else:
         icons.sll_tags.append("UpdateDLConnection_DLOL")
 
+    if isinstance(tables, str):
+        tables = [tables]
+
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
     (dataset_name, dataset_id) = resolve_dataset_name_and_id(dataset, workspace_id)
 
@@ -174,7 +184,12 @@ def update_direct_lake_model_connection(
             )
 
         # Update the single connection expression
-        if len(expressions) == 1:
+        if len(expressions) > 1 and not tables:
+            print(
+                f"{icons.info} Multiple expressions found in the model. Please specify the tables to update using the 'tables parameter."
+            )
+            return
+        elif len(expressions) == 1 and not tables:
             expr = expressions[0]
             tom.model.Expressions[expr].Expression = shared_expression
 
@@ -182,6 +197,41 @@ def update_direct_lake_model_connection(
                 f"{icons.green_dot} The expression in the '{dataset_name}' semantic model within the '{workspace_name}' workspace has been updated to point to the '{source}' {source_type.lower()} in the '{source_workspace}' workspace."
             )
         else:
-            print(
-                f"{icons.info} Multiple expressions found in the model. Please use the update_direct_lake_partition_entity function to update specific tables."
+            import sempy
+
+            sempy.fabric._client._utils._init_analysis_services()
+            import Microsoft.AnalysisServices.Tabular as TOM
+
+            expr_list = _extract_expression_list(shared_expression)
+
+            expr_name = next(
+                (name for name, exp in expression_dict.items() if exp == expr_list),
+                None,
             )
+
+            # If the expression does not already exist, create it
+            def generate_unique_name(existing_names):
+                i = 1
+                while True:
+                    candidate = f"DatabaseQuery{i}"
+                    if candidate not in existing_names:
+                        return candidate
+                    i += 1
+
+            if not expr_name:
+                expr_name = generate_unique_name(expressions)
+                tom.add_expression(name=expr_name, expression=shared_expression)
+
+            all_tables = [t.Name for t in tom.model.Tables]
+            for t_name in tables:
+                if t_name not in all_tables:
+                    raise ValueError(
+                        f"{icons.red_dot} The table '{t_name}' does not exist in the '{dataset_name}' semantic model within the '{workspace_name}' workspace."
+                    )
+                p = next(p for p in tom.model.Tables[t_name].Partitions)
+                if p.Mode != TOM.ModeType.DirectLake:
+                    raise ValueError(
+                        f"{icons.red_dot} The table '{t_name}' in the '{dataset_name}' semantic model within the '{workspace_name}' workspace is not in Direct Lake mode. This function is only applicable to Direct Lake tables."
+                    )
+
+                p.Source.ExpressionSource = tom.model.Expressions[expr_name]

sempy_labs/dotnet_lib/dotnet.runtime.config.json

@@ -0,0 +1,10 @@
+{
+  "runtimeOptions": {
+    "tfm": "net6.0",
+    "framework": {
+      "name": "Microsoft.NETCore.App",
+      "version": "6.0.0"
+    },
+    "rollForward": "Major"
+  }
+}

sempy_labs/lakehouse/__init__.py

@@ -21,6 +21,15 @@ from sempy_labs.lakehouse._blobs import (
     recover_lakehouse_object,
     list_blobs,
 )
+from sempy_labs.lakehouse._livy_sessions import (
+    list_livy_sessions,
+)
+from sempy_labs.lakehouse._helper import (
+    is_v_ordered,
+    delete_lakehouse,
+    update_lakehouse,
+    load_table,
+)
 
 __all__ = [
     "get_lakehouse_columns",
@@ -36,4 +45,9 @@ __all__ = [
     "list_shortcuts",
     "recover_lakehouse_object",
     "list_blobs",
+    "list_livy_sessions",
+    "is_v_ordered",
+    "delete_lakehouse",
+    "update_lakehouse",
+    "load_table",
 ]

sempy_labs/lakehouse/_blobs.py

@@ -11,6 +11,7 @@ from typing import Optional, List
 import sempy_labs._icons as icons
 import xml.etree.ElementTree as ET
 import pandas as pd
+from sempy.fabric.exceptions import FabricHTTPException
 
 
 def _request_blob_api(
@@ -18,6 +19,7 @@ def _request_blob_api(
     method: str = "get",
     payload: Optional[dict] = None,
     status_codes: int | List[int] = 200,
+    uses_pagination: bool = False,
 ):
 
     import requests
@@ -31,21 +33,41 @@ def _request_blob_api(
 
     headers = {
         "Authorization": f"Bearer {token}",
-        "Content-Type": "application/json",
+        "Content-Type": "application/xml",
         "x-ms-version": "2025-05-05",
     }
 
-    response = requests.request(
-        method.upper(),
-        f"https://onelake.blob.fabric.microsoft.com/{request}",
-        headers=headers,
-        json=payload,
-    )
+    base_url = "https://onelake.blob.fabric.microsoft.com/"
+    full_url = f"{base_url}{request}"
+    results = []
+
+    while True:
+        response = requests.request(
+            method.upper(),
+            full_url,
+            headers=headers,
+            data=payload if method.lower() != "get" else None,
+        )
+
+        if response.status_code not in status_codes:
+            raise FabricHTTPException(response)
+
+        if not uses_pagination:
+            return response
+
+        # Parse XML to find blobs and NextMarker
+        root = ET.fromstring(response.content)
+        results.append(root)
+
+        next_marker = root.findtext(".//NextMarker")
+        if not next_marker:
+            break  # No more pages
 
-    if response.status_code not in status_codes:
-        raise FabricHTTPException(response)
+        # Append the marker to the original request (assuming query string format)
+        delimiter = "&" if "?" in request else "?"
+        full_url = f"{base_url}{request}{delimiter}marker={next_marker}"
 
-    return response
+    return results
 
 
 @log
@@ -90,12 +112,6 @@ def list_blobs(
             )
         path_prefix = f"{workspace_id}/{lakehouse_id}/{container}"
 
-    response = _request_blob_api(
-        request=f"{path_prefix}?restype=container&comp=list&include=deleted"
-    )
-    root = ET.fromstring(response.content)
-    response_json = _xml_to_dict(root)
-
     columns = {
         "Blob Name": "str",
         "Is Deleted": "bool",
@@ -122,37 +138,55 @@ def list_blobs(
 
     df = _create_dataframe(columns=columns)
 
-    for blob in (
-        response_json.get("EnumerationResults", {}).get("Blobs", {}).get("Blob", {})
-    ):
-        p = blob.get("Properties", {})
-        new_data = {
-            "Blob Name": blob.get("Name"),
-            "Is Deleted": blob.get("Deleted", False),
-            "Deletion Id": blob.get("DeletionId"),
-            "Creation Time": p.get("Creation-Time"),
-            "Expiry Time": p.get("Expiry-Time"),
-            "Etag": p.get("Etag"),
-            "Resource Type": p.get("ResourceType"),
-            "Content Length": p.get("Content-Length"),
-            "Content Type": p.get("Content-Type"),
-            "Content Encoding": p.get("Content-Encoding"),
-            "Content Language": p.get("Content-Language"),
-            "Content CRC64": p.get("Content-CRC64"),
-            "Content MD5": p.get("Content-MD5"),
-            "Cache Control": p.get("Cache-Control"),
-            "Content Disposition": p.get("Content-Disposition"),
-            "Blob Type": p.get("BlobType"),
-            "Access Tier": p.get("AccessTier"),
-            "Access Tier Inferred": p.get("AccessTierInferred"),
-            "Server Encrypted": p.get("ServerEncrypted"),
-            "Deleted Time": p.get("DeletedTime"),
-            "Remaining Retention Days": p.get("RemainingRetentionDays"),
-        }
-
-        df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
-
-    _update_dataframe_datatypes(dataframe=df, column_map=columns)
+    url = f"{path_prefix}?restype=container&comp=list&include=deleted"
+
+    responses = _request_blob_api(
+        request=url,
+        uses_pagination=True,
+    )
+
+    dfs = []
+    for root in responses:
+        response_json = _xml_to_dict(root)
+
+        blobs = (
+            response_json.get("EnumerationResults", {}).get("Blobs", {}).get("Blob", [])
+        )
+
+        if isinstance(blobs, dict):
+            blobs = [blobs]
+
+        for blob in blobs:
+            p = blob.get("Properties", {})
+            new_data = {
+                "Blob Name": blob.get("Name"),
+                "Is Deleted": blob.get("Deleted", False),
+                "Deletion Id": blob.get("DeletionId"),
+                "Creation Time": p.get("Creation-Time"),
+                "Expiry Time": p.get("Expiry-Time"),
+                "Etag": p.get("Etag"),
+                "Resource Type": p.get("ResourceType"),
+                "Content Length": p.get("Content-Length"),
+                "Content Type": p.get("Content-Type"),
+                "Content Encoding": p.get("Content-Encoding"),
+                "Content Language": p.get("Content-Language"),
+                "Content CRC64": p.get("Content-CRC64"),
+                "Content MD5": p.get("Content-MD5"),
+                "Cache Control": p.get("Cache-Control"),
+                "Content Disposition": p.get("Content-Disposition"),
+                "Blob Type": p.get("BlobType"),
+                "Access Tier": p.get("AccessTier"),
+                "Access Tier Inferred": p.get("AccessTierInferred"),
+                "Server Encrypted": p.get("ServerEncrypted"),
+                "Deleted Time": p.get("DeletedTime"),
+                "Remaining Retention Days": p.get("RemainingRetentionDays"),
+            }
+
+            dfs.append(pd.DataFrame(new_data, index=[0]))
+
+    if dfs:
+        df = pd.concat(dfs, ignore_index=True)
+        _update_dataframe_datatypes(dataframe=df, column_map=columns)
 
     return df
 
@@ -182,7 +216,7 @@ def recover_lakehouse_object(
     workspace_id = resolve_workspace_id(workspace)
     lakehouse_id = resolve_lakehouse_id(lakehouse, workspace_id)
 
-    blob_path_prefix = f"{lakehouse_id}/{file_path}"
+    blob_name = f"{lakehouse_id}/{file_path}"
 
     container = file_path.split("/")[0]
     if container not in ["Tables", "Files"]:
@@ -190,42 +224,23 @@ def recover_lakehouse_object(
             f"{icons.red_dot} Invalid container '{container}' within the file_path parameter. Expected 'Tables' or 'Files'."
         )
 
-    df = list_blobs(lakehouse=lakehouse, workspace=workspace, container=container)
+    # Undelete the blob
+    print(f"{icons.in_progress} Attempting to recover the '{blob_name}' blob...")
 
-    for _, r in df.iterrows():
-        blob_name = r.get("Blob Name")
-        is_deleted = r.get("Is Deleted")
-        if blob_name.startswith(blob_path_prefix) and is_deleted:
-            print(f"{icons.in_progress} Restoring the '{blob_name}' blob...")
-            _request_blob_api(
-                request=f"{workspace_id}/{lakehouse_id}/{file_path}?comp=undelete",
-                method="put",
+    try:
+        _request_blob_api(
+            request=f"{workspace_id}/{lakehouse_id}/{file_path}?comp=undelete",
+            method="put",
+        )
+        print(
+            f"{icons.green_dot} The '{blob_name}' blob recover attempt was successful."
+        )
+    except FabricHTTPException as e:
+        if e.status_code == 404:
+            print(
+                f"{icons.warning} The '{blob_name}' blob was not found. No action taken."
+            )
+        else:
+            print(
+                f"{icons.red_dot} An error occurred while recovering the '{blob_name}' blob: {e}"
             )
-            print(f"{icons.green_dot} The '{blob_name}' blob has been restored.")
-
-
-def _get_user_delegation_key():
-
-    # https://learn.microsoft.com/rest/api/storageservices/get-user-delegation-key
-
-    from datetime import datetime, timedelta, timezone
-
-    utc_now = datetime.now(timezone.utc)
-    start_time = utc_now + timedelta(minutes=2)
-    expiry_time = start_time + timedelta(minutes=45)
-    start_str = start_time.strftime("%Y-%m-%dT%H:%M:%SZ")
-    expiry_str = expiry_time.strftime("%Y-%m-%dT%H:%M:%SZ")
-
-    payload = f"""<?xml version="1.0" encoding="utf-8"?>
-    <KeyInfo>
-        <Start>{start_str}</Start>
-        <Expiry>{expiry_str}</Expiry>
-    </KeyInfo>"""
-
-    response = _request_blob_api(
-        request="restype=service&comp=userdelegationkey",
-        method="post",
-        payload=payload,
-    )
-
-    return response.content

sempy_labs/lakehouse/_get_lakehouse_tables.py

@@ -9,8 +9,6 @@ from sempy_labs._helper_functions import (
     save_as_delta_table,
     _base_api,
     _create_dataframe,
-    resolve_workspace_id,
-    resolve_lakehouse_id,
     _read_delta_table,
     _get_delta_table,
     _mount,
@@ -85,16 +83,6 @@ def get_lakehouse_tables(
     if count_rows:  # Setting countrows defaults to extended=True
         extended = True
 
-    if (
-        workspace_id != resolve_workspace_id()
-        and lakehouse_id != resolve_lakehouse_id()
-        and count_rows
-    ):
-        raise ValueError(
-            f"{icons.red_dot} If 'count_rows' is set to True, you must run this function against the default lakehouse attached to the notebook. "
-            "Count rows runs a spark query and cross-workspace spark queries are currently not supported."
-        )
-
     responses = _base_api(
         request=f"v1/workspaces/{workspace_id}/lakehouses/{lakehouse_id}/tables",
         uses_pagination=True,
@@ -123,7 +111,7 @@ def get_lakehouse_tables(
     if extended:
         sku_value = get_sku_size(workspace_id)
         guardrail = get_directlake_guardrails_for_sku(sku_value)
-        local_path = _mount()
+        local_path = _mount(lakehouse=lakehouse_id, workspace=workspace_id)
 
         df["Files"], df["Row Groups"], df["Table Size"] = None, None, None
         if count_rows:

sempy_labs/lakehouse/_helper.py

@@ -0,0 +1,211 @@
+from uuid import UUID
+from typing import Optional, Literal
+import pyarrow.dataset as ds
+from sempy_labs._helper_functions import (
+    _mount,
+    delete_item,
+    _base_api,
+    resolve_workspace_name_and_id,
+    resolve_lakehouse_name_and_id,
+)
+from sempy._utils._log import log
+import sempy_labs._icons as icons
+import os
+
+
+@log
+def is_v_ordered(
+    table_name: str,
+    lakehouse: Optional[str | UUID] = None,
+    workspace: Optional[str | UUID] = None,
+    schema: Optional[str] = None,
+) -> bool:
+    """
+    Checks if a delta table is v-ordered.
+
+    Parameters
+    ----------
+    table_name : str
+        The name of the table to check.
+    lakehouse : str | uuid.UUID, default=None
+        The Fabric lakehouse name or ID.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID used by the lakehouse.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    schema : str, optional
+        The schema of the table to check. If not provided, the default schema is used.
+
+    Returns
+    -------
+    bool
+        True if the table is v-ordered, False otherwise.
+    """
+
+    local_path = _mount(lakehouse=lakehouse, workspace=workspace)
+    table_path = (
+        f"{local_path}/Tables/{schema}/{table_name}"
+        if schema
+        else f"{local_path}/Tables/{table_name}"
+    )
+    ds_schema = ds.dataset(table_path).schema.metadata
+
+    return any(b"vorder" in key for key in ds_schema.keys())
+
+
+def delete_lakehouse(
+    lakehouse: str | UUID, workspace: Optional[str | UUID] = None
+) -> None:
+    """
+    Deletes a lakehouse.
+
+    This is a wrapper function for the following API: `Items - Delete Lakehouse <https://learn.microsoft.com/rest/api/fabric/lakehouse/items/delete-lakehouse>`_.
+
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
+    Parameters
+    ----------
+    lakehouse : str | uuid.UUID
+        The name or ID of the lakehouse to delete.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID used by the lakehouse.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    delete_item(item=lakehouse, item_type="lakehouse", workspace=workspace)
+
+
+def update_lakehouse(
+    name: Optional[str] = None,
+    description: Optional[str] = None,
+    lakehouse: Optional[str | UUID] = None,
+    workspace: Optional[str | UUID] = None,
+):
+    """
+    Updates a lakehouse.
+
+    This is a wrapper function for the following API: `Items - Update Lakehouse <https://learn.microsoft.com/rest/api/fabric/lakehouse/items/update-lakehouse>`_.
+
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
+    Parameters
+    ----------
+    name: str, default=None
+        The new name of the lakehouse.
+        Defaults to None which does not update the name.
+    description: str, default=None
+        The new description of the lakehouse.
+        Defaults to None which does not update the description.
+    lakehouse : str | uuid.UUID, default=None
+        The name or ID of the lakehouse to update.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID used by the lakehouse.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    if not name and not description:
+        raise ValueError(
+            f"{icons.red_dot} Either name or description must be provided."
+        )
+
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    (lakehouse_name, lakehouse_id) = resolve_lakehouse_name_and_id(
+        lakehouse, workspace_id
+    )
+
+    payload = {}
+    if name:
+        payload["displayName"] = name
+    if description:
+        payload["description"] = description
+
+    _base_api(
+        request=f"/v1/workspaces/{workspace_id}/lakehouses/{lakehouse_id}",
+        method="patch",
+        client="fabric_sp",
+        payload=payload,
+    )
+
+    print(
+        f"{icons.green_dot} The '{lakehouse_name}' lakehouse within the '{workspace_name}' workspace has been updated accordingly."
+    )
+
+
+@log
+def load_table(
+    table_name: str,
+    file_path: str,
+    mode: Literal["Overwrite", "Append"],
+    lakehouse: Optional[str | UUID] = None,
+    workspace: Optional[str | UUID] = None,
+):
+    """
+    Loads a table into a lakehouse. Currently only files are supported, not folders.
+
+    This is a wrapper function for the following API: `Tables - Load Table <https://learn.microsoft.com/rest/api/fabric/lakehouse/tables/load-table>`_.
+
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
+    Parameters
+    ----------
+    table_name : str
+        The name of the table to load.
+    file_path : str
+        The path to the data to load.
+    mode : Literal["Overwrite", "Append"]
+        The mode to use when loading the data.
+        "Overwrite" will overwrite the existing data.
+        "Append" will append the data to the existing data.
+    lakehouse : str | uuid.UUID, default=None
+        The name or ID of the lakehouse to load the table into.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID used by the lakehouse.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    (lakehouse_name, lakehouse_id) = resolve_lakehouse_name_and_id(
+        lakehouse, workspace_id
+    )
+
+    file_extension = os.path.splitext(file_path)[1]
+
+    payload = {
+        "relativePath": file_path,
+        "pathType": "File",
+        "mode": mode,
+        "formatOptions": {},
+    }
+
+    if file_extension == ".csv":
+        payload["formatOptions"] = {"format": "Csv", "header": True, "delimiter": ","}
+    elif file_extension == ".parquet":
+        payload["formatOptions"] = {
+            "format": "Parquet",
+            "header": True,
+        }
+    # Solve for loading folders
+    # elif file_extension == '':
+    #    payload['pathType'] = "Folder"
+    #    payload["recursive"] = recursive
+    #    payload['formatOptions']
+    else:
+        raise NotImplementedError()
+
+    _base_api(
+        request=f"/v1/workspaces/{workspace_id}/lakehouses/{lakehouse_id}/tables/{table_name}/load",
+        client="fabric_sp",
+        method="post",
+        status_codes=202,
+        lro_return_status_code=True,
+    )
+
+    print(
+        f"{icons.green_dot} The '{table_name}' table has been loaded into the '{lakehouse_name}' lakehouse within the '{workspace_name}' workspace."
+    )

sempy_labs/lakehouse/_lakehouse.py

@@ -113,7 +113,7 @@ def vacuum_lakehouse_tables(
     Parameters
     ----------
     tables : str | List[str] | None
-        The table(s) to vacuum. If no tables are specified, all tables in the lakehouse will be optimized.
+        The table(s) to vacuum. If no tables are specified, all tables in the lakehouse will be vacuumed.
     lakehouse : str | uuid.UUID, default=None
         The Fabric lakehouse name or ID.
         Defaults to None which resolves to the lakehouse attached to the notebook.