semantic-link-labs 0.9.11__py3-none-any.whl → 0.10.1__py3-none-any.whl

sempy_labs/_helper_functions.py

@@ -17,6 +17,8 @@ import numpy as np
 from IPython.display import display, HTML
 import requests
 import sempy_labs._authentication as auth
+from jsonpath_ng.ext import parse
+from jsonpath_ng.jsonpath import Fields, Index
 
 
 def _build_url(url: str, params: dict) -> str:
@@ -2270,3 +2272,166 @@ def file_exists(file_path: str) -> bool:
     import notebookutils
 
     return len(notebookutils.fs.ls(file_path)) > 0
+
+
+def generate_number_guid():
+
+    guid = uuid.uuid4()
+    return str(guid.int & ((1 << 64) - 1))
+
+
+def get_url_content(url: str):
+
+    if "github.com" in url and "/blob/" in url:
+        url = url.replace("github.com", "raw.githubusercontent.com")
+        url = url.replace("/blob/", "/")
+
+    response = requests.get(url)
+    if response.ok:
+        try:
+            data = response.json()  # Only works if the response is valid JSON
+        except ValueError:
+            data = response.text  # Fallback: get raw text content
+        return data
+    else:
+        print(f"Failed to fetch raw content: {response.status_code}")
+
+
+def generate_hex(length: int = 10) -> str:
+    """
+    Generate a random hex string of the specified length. Used for generating IDs for report objects (page, visual, bookmark etc.).
+    """
+    import secrets
+
+    return secrets.token_hex(length)
+
+
+def decode_payload(payload):
+
+    if is_base64(payload):
+        try:
+            decoded_payload = json.loads(base64.b64decode(payload).decode("utf-8"))
+        except Exception:
+            decoded_payload = base64.b64decode(payload)
+    elif isinstance(payload, dict):
+        decoded_payload = payload
+    else:
+        raise ValueError("Payload must be a dictionary or a base64 encoded value.")
+
+    return decoded_payload
+
+
+def is_base64(s):
+    try:
+        # Add padding if needed
+        s_padded = s + "=" * (-len(s) % 4)
+        decoded = base64.b64decode(s_padded, validate=True)
+        # Optional: check if re-encoding gives the original (excluding padding)
+        return base64.b64encode(decoded).decode().rstrip("=") == s.rstrip("=")
+    except Exception:
+        return False
+
+
+def get_jsonpath_value(
+    data, path, default=None, remove_quotes=False, fix_true: bool = False
+):
+    matches = parse(path).find(data)
+    result = matches[0].value if matches else default
+    if result and remove_quotes and isinstance(result, str):
+        if result.startswith("'") and result.endswith("'"):
+            result = result[1:-1]
+    if fix_true and isinstance(result, str):
+        if result.lower() == "true":
+            result = True
+        elif result.lower() == "false":
+            result = False
+    return result
+
+
+def set_json_value(payload: dict, json_path: str, json_value: str | dict | List):
+
+    jsonpath_expr = parse(json_path)
+    matches = jsonpath_expr.find(payload)
+
+    if matches:
+        # Update all matches
+        for match in matches:
+            parent = match.context.value
+            path = match.path
+            if isinstance(path, Fields):
+                parent[path.fields[0]] = json_value
+            elif isinstance(path, Index):
+                parent[path.index] = json_value
+    else:
+        # Handle creation
+        parts = json_path.lstrip("$").strip(".").split(".")
+        current = payload
+
+        for i, part in enumerate(parts):
+            is_last = i == len(parts) - 1
+
+            # Detect list syntax like "lockAspect[*]"
+            list_match = re.match(r"(\w+)\[\*\]", part)
+            if list_match:
+                list_key = list_match.group(1)
+                if list_key not in current or not isinstance(current[list_key], list):
+                    # Initialize with one dict element
+                    current[list_key] = [{}]
+
+                for item in current[list_key]:
+                    if is_last:
+                        # Last part, assign value
+                        item = json_value
+                    else:
+                        # Proceed to next level
+                        if not isinstance(item, dict):
+                            raise ValueError(
+                                f"Expected dict in list for key '{list_key}', got {type(item)}"
+                            )
+                        next_part = ".".join(parts[i + 1 :])
+                        set_json_value(item, "$." + next_part, json_value)
+                return payload
+            else:
+                if part not in current or not isinstance(current[part], dict):
+                    current[part] = {} if not is_last else json_value
+                elif is_last:
+                    current[part] = json_value
+                current = current[part]
+
+    return payload
+
+
+def remove_json_value(path: str, payload: dict, json_path: str, verbose: bool = True):
+
+    if not isinstance(payload, dict):
+        raise ValueError(
+            f"{icons.red_dot} Cannot apply json_path to non-dictionary payload in '{path}'."
+        )
+
+    jsonpath_expr = parse(json_path)
+    matches = jsonpath_expr.find(payload)
+
+    if not matches and verbose:
+        print(
+            f"{icons.red_dot} No match found for '{json_path}' in '{path}'. Skipping."
+        )
+        return payload
+
+    for match in matches:
+        parent = match.context.value
+        path_expr = match.path
+
+        if isinstance(path_expr, Fields):
+            key = path_expr.fields[0]
+            if key in parent:
+                del parent[key]
+                if verbose:
+                    print(f"{icons.green_dot} Removed key '{key}' from '{path}'.")
+        elif isinstance(path_expr, Index):
+            index = path_expr.index
+            if isinstance(parent, list) and 0 <= index < len(parent):
+                parent.pop(index)
+                if verbose:
+                    print(f"{icons.green_dot} Removed index [{index}] from '{path}'.")
+
+    return payload

sempy_labs/_list_functions.py

@@ -642,49 +642,6 @@ def list_lakehouses(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
     return df
 
 
-def list_sql_endpoints(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
-    """
-    Shows the SQL endpoints within a workspace.
-
-    Parameters
-    ----------
-    workspace : str | uuid.UUID, default=None
-        The Fabric workspace name or ID.
-        Defaults to None which resolves to the workspace of the attached lakehouse
-        or if no lakehouse attached, resolves to the workspace of the notebook.
-
-    Returns
-    -------
-    pandas.DataFrame
-        A pandas dataframe showing the SQL endpoints within a workspace.
-    """
-
-    columns = {
-        "SQL Endpoint Id": "string",
-        "SQL Endpoint Name": "string",
-        "Description": "string",
-    }
-    df = _create_dataframe(columns=columns)
-
-    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
-
-    responses = _base_api(
-        request=f"/v1/workspaces/{workspace_id}/sqlEndpoints", uses_pagination=True
-    )
-
-    for r in responses:
-        for v in r.get("value", []):
-
-            new_data = {
-                "SQL Endpoint Id": v.get("id"),
-                "SQL Endpoint Name": v.get("displayName"),
-                "Description": v.get("description"),
-            }
-            df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
-
-    return df
-
-
 def list_datamarts(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
     """
     Shows the datamarts within a workspace.

sempy_labs/_notebooks.py

@@ -159,6 +159,7 @@ def import_notebook_from_web(
             notebook_content=response.content,
             workspace=workspace_id,
             description=description,
+            format="ipynb",
         )
     elif len(dfI_filt) > 0 and overwrite:
         print(f"{icons.info} Overwrite of notebooks is currently not supported.")
@@ -202,9 +203,8 @@ def create_notebook(
         otherwise notebook_content should be GIT friendly format
     """
 
-    notebook_payload = base64.b64encode(notebook_content.encode("utf-8")).decode(
-        "utf-8"
-    )
+    notebook_payload = base64.b64encode(notebook_content).decode("utf-8")
+
     definition_payload = {
         "parts": [
             {

sempy_labs/_semantic_models.py

@@ -8,6 +8,8 @@ from sempy_labs._helper_functions import (
     resolve_workspace_name_and_id,
     resolve_dataset_name_and_id,
     delete_item,
+    resolve_dataset_id,
+    resolve_workspace_id,
 )
 import sempy_labs._icons as icons
 import re
@@ -227,3 +229,102 @@ def update_semantic_model_refresh_schedule(
     print(
         f"{icons.green_dot} Refresh schedule for the '{dataset_name}' within the '{workspace_name}' workspace has been updated."
     )
+
+
+def list_semantic_model_datasources(
+    dataset: str | UUID,
+    workspace: Optional[str | UUID] = None,
+    expand_details: bool = True,
+) -> pd.DataFrame:
+    """
+    Lists the data sources for the specified semantic model.
+
+    This is a wrapper function for the following API: `Datasets - Get Datasources In Group <https://learn.microsoft.com/rest/api/power-bi/datasets/get-datasources-in-group>`_.
+
+    Parameters
+    ----------
+    dataset : str | uuid.UUID
+        Name or ID of the semantic model.
+    workspace : str | uuid.UUID, default=None
+        The workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    expand_details : bool, default=True
+        If True, expands the connection details for each data source.
+
+    Returns
+    -------
+    pandas.DataFrame
+        DataFrame containing the data sources for the specified semantic model.
+    """
+
+    workspace_id = resolve_workspace_id(workspace)
+    dataset_id = resolve_dataset_id(dataset, workspace_id)
+
+    if expand_details:
+        columns = {
+            "Datasource Type": "str",
+            "Connection Server": "str",
+            "Connection Database": "str",
+            "Connection Path": "str",
+            "Connection Account": "str",
+            "Connection Domain": "str",
+            "Connection Kind": "str",
+            "Connection Email Address": "str",
+            "Connection URL": "str",
+            "Connection Class Info": "str",
+            "Connection Login Server": "str",
+            "Datasource Id": "str",
+            "Gateway Id": "str",
+        }
+    else:
+        columns = {
+            "Datasource Type": "str",
+            "Connection Details": "str",
+            "Datasource Id": "str",
+            "Gateway Id": "str",
+        }
+
+    df = _create_dataframe(columns)
+
+    response = _base_api(
+        request=f"/v1.0/myorg/groups/{workspace_id}/datasets/{dataset_id}/datasources",
+        client="fabric_sp",
+    )
+
+    dfs = []
+    for item in response.json().get("value", []):
+        ds_type = item.get("datasourceType")
+        conn_details = item.get("connectionDetails", {})
+        ds_id = item.get("datasourceId")
+        gateway_id = item.get("gatewayId")
+        if expand_details:
+            new_data = {
+                "Datasource Type": ds_type,
+                "Connection Server": conn_details.get("server"),
+                "Connection Database": conn_details.get("database"),
+                "Connection Path": conn_details.get("path"),
+                "Connection Account": conn_details.get("account"),
+                "Connection Domain": conn_details.get("domain"),
+                "Connection Kind": conn_details.get("kind"),
+                "Connection Email Address": conn_details.get("emailAddress"),
+                "Connection URL": conn_details.get("url"),
+                "Connection Class Info": conn_details.get("classInfo"),
+                "Connection Login Server": conn_details.get("loginServer"),
+                "Datasource Id": ds_id,
+                "Gateway Id": gateway_id,
+            }
+            dfs.append(pd.DataFrame(new_data, index=[0]))
+        else:
+            new_data = {
+                "Datasource Type": ds_type,
+                "Connection Details": conn_details,
+                "Datasource Id": ds_id,
+                "Gateway Id": gateway_id,
+            }
+            dfs.append(pd.DataFrame([new_data]))
+
+    if dfs:
+        df = pd.concat(dfs, ignore_index=True)
+
+    return df

sempy_labs/_sql.py

@@ -82,7 +82,7 @@ class ConnectBase:
             )
 
         # Set up the connection string
-        access_token = SynapseTokenProvider()()
+        access_token = SynapseTokenProvider()("sql")
         tokenstruct = _bytes2mswin_bstr(access_token.encode())
         if endpoint_type == "sqldatabase":
             conn_str = f"DRIVER={{ODBC Driver 18 for SQL Server}};SERVER={tds_endpoint};DATABASE={resource_name}-{resource_id};Encrypt=Yes;"

sempy_labs/_sql_endpoints.py

@@ -0,0 +1,185 @@
+from typing import Optional, Literal
+from uuid import UUID
+import pandas as pd
+from sempy_labs._helper_functions import (
+    _base_api,
+    _create_dataframe,
+    resolve_workspace_name_and_id,
+    resolve_item_name_and_id,
+    _update_dataframe_datatypes,
+)
+import sempy_labs._icons as icons
+
+
+def list_sql_endpoints(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
+    """
+    Shows the SQL endpoints within a workspace.
+
+    Parameters
+    ----------
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing the SQL endpoints within a workspace.
+    """
+
+    columns = {
+        "SQL Endpoint Id": "string",
+        "SQL Endpoint Name": "string",
+        "Description": "string",
+    }
+    df = _create_dataframe(columns=columns)
+
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+
+    responses = _base_api(
+        request=f"/v1/workspaces/{workspace_id}/sqlEndpoints", uses_pagination=True
+    )
+
+    for r in responses:
+        for v in r.get("value", []):
+
+            new_data = {
+                "SQL Endpoint Id": v.get("id"),
+                "SQL Endpoint Name": v.get("displayName"),
+                "Description": v.get("description"),
+            }
+            df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
+
+    return df
+
+
+def refresh_sql_endpoint_metadata(
+    item: str | UUID,
+    type: Literal["Lakehouse", "MirroredDatabase"],
+    workspace: Optional[str | UUID] = None,
+    tables: dict[str, list[str]] = None,
+) -> pd.DataFrame:
+    """
+    Refreshes the metadata of a SQL endpoint.
+
+    This is a wrapper function for the following API: `Items - Refresh Sql Endpoint Metadata <https://learn.microsoft.com/rest/api/fabric/sqlendpoint/items/refresh-sql-endpoint-metadata>`_.
+
+    Parameters
+    ----------
+    item : str | uuid.UUID
+        The name or ID of the item (Lakehouse or MirroredDatabase).
+    type : Literal['Lakehouse', 'MirroredDatabase']
+        The type of the item. Must be 'Lakehouse' or 'MirroredDatabase'.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    tables : dict[str, list[str]], default=None
+        A dictionary where the keys are schema names and the values are lists of table names.
+        If empty, all table metadata will be refreshed.
+
+        Example:
+        {
+            "dbo": ["DimDate", "DimGeography"],
+            "sls": ["FactSales", "FactBudget"],
+        }
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing the status of the metadata refresh operation.
+    """
+
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+
+    (item_name, item_id) = resolve_item_name_and_id(
+        item=item, type=type, workspace=workspace
+    )
+
+    if type == "Lakehouse":
+        response = _base_api(
+            request=f"/v1/workspaces/{workspace_id}/lakehouses/{item_id}",
+            client="fabric_sp",
+        )
+        sql_endpoint_id = (
+            response.json()
+            .get("properties", {})
+            .get("sqlEndpointProperties", {})
+            .get("id")
+        )
+    elif type == "MirroredDatabase":
+        response = _base_api(
+            request=f"/v1/workspaces/{workspace_id}/mirroredDatabases/{item_id}",
+            client="fabric_sp",
+        )
+        sql_endpoint_id = (
+            response.json()
+            .get("properties", {})
+            .get("sqlEndpointProperties", {})
+            .get("id")
+        )
+    else:
+        raise ValueError("Invalid type. Must be 'Lakehouse' or 'MirroredDatabase'.")
+
+    payload = {}
+    if tables:
+        payload = {
+            "tableDefinitions": [
+                {"schema": schema, "tableNames": tables}
+                for schema, tables in tables.items()
+            ]
+        }
+
+    result = _base_api(
+        request=f"v1/workspaces/{workspace_id}/sqlEndpoints/{sql_endpoint_id}/refreshMetadata?preview=true",
+        method="post",
+        status_codes=[200, 202],
+        lro_return_json=True,
+        payload=payload,
+    )
+
+    columns = {
+        "Table Name": "string",
+        "Status": "string",
+        "Start Time": "datetime",
+        "End Time": "datetime",
+        "Last Successful Sync Time": "datetime",
+        "Error Code": "string",
+        "Error Message": "string",
+    }
+
+    df = pd.json_normalize(result)
+
+    # Extract error code and message, set to None if no error
+    df['Error Code'] = df.get('error.errorCode', None)
+    df['Error Message'] = df.get('error.message', None)
+
+    # Friendly column renaming
+    df.rename(columns={
+        'tableName': 'Table Name',
+        'startDateTime': 'Start Time',
+        'endDateTime': 'End Time',
+        'status': 'Status',
+        'lastSuccessfulSyncDateTime': 'Last Successful Sync Time'
+    }, inplace=True)
+
+    # Drop the original 'error' column if present
+    df.drop(columns=[col for col in ['error'] if col in df.columns], inplace=True)
+
+    # Optional: Reorder columns
+    column_order = [
+        'Table Name', 'Status', 'Start Time', 'End Time',
+        'Last Successful Sync Time', 'Error Code', 'Error Message'
+    ]
+    df = df[column_order]
+
+    _update_dataframe_datatypes(df, columns)
+
+    printout = f"{icons.green_dot} The metadata of the SQL endpoint for the '{item_name}' {type.lower()} within the '{workspace_name}' workspace has been refreshed"
+    if tables:
+        print(f"{printout} for the following tables: {tables}.")
+    else:
+        print(f"{printout} for all tables.")
+
+    return df

sempy_labs/_user_delegation_key.py

@@ -0,0 +1,42 @@
+from sempy_labs.lakehouse._blobs import _request_blob_api
+from sempy_labs._helper_functions import (
+    _xml_to_dict,
+)
+from datetime import datetime, timedelta, timezone
+import xml.etree.ElementTree as ET
+
+
+def get_user_delegation_key():
+    """
+    Gets a key that can be used to sign a user delegation SAS (shared access signature). A user delegation SAS grants access to Azure Blob Storage resources by using Microsoft Entra credentials.
+
+    This is a wrapper function for the following API: `Get User Delegation Key <https://learn.microsoft.com/rest/api/storageservices/get-user-delegation-key>`_.
+
+    Returns
+    -------
+    str
+        The user delegation key value.
+    """
+
+    utc_now = datetime.now(timezone.utc)
+    start_time = utc_now + timedelta(minutes=2)
+    expiry_time = start_time + timedelta(minutes=60)
+    start_str = start_time.strftime("%Y-%m-%dT%H:%M:%SZ")
+    expiry_str = expiry_time.strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    payload = f"""<?xml version="1.0" encoding="utf-8"?>
+    <KeyInfo>
+        <Start>{start_str}</Start>
+        <Expiry>{expiry_str}</Expiry>
+    </KeyInfo>"""
+
+    response = _request_blob_api(
+        request="?restype=service&comp=userdelegationkey",
+        method="post",
+        payload=payload,
+    )
+
+    root = ET.fromstring(response.content)
+    response_json = _xml_to_dict(root)
+
+    return response_json.get("UserDelegationKey", {}).get("Value", None)

sempy_labs/_vpax.py

@@ -16,6 +16,7 @@ from sempy_labs._helper_functions import (
     file_exists,
     create_abfss_path_from_path,
 )
+from sempy._utils._log import log
 import sempy_labs._icons as icons
 import zipfile
 import requests
@@ -134,6 +135,7 @@ def init_vertipaq_analyzer():
     _vpa_initialized = True
 
 
+@log
 def create_vpax(
     dataset: str | UUID,
     workspace: Optional[str | UUID] = None,

sempy_labs/directlake/_update_directlake_model_lakehouse_connection.py

@@ -111,9 +111,9 @@ def update_direct_lake_model_connection(
 
     Parameters
     ----------
-    dataset : str | UUID
+    dataset : str | uuid.UUID
         Name or ID of the semantic model.
-    workspace : str | UUID, default=None
+    workspace : str | uuid.UUID, default=None
         The Fabric workspace name or ID in which the semantic model exists.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
@@ -122,7 +122,7 @@ def update_direct_lake_model_connection(
         Defaults to None which resolves to the lakehouse attached to the notebook.
     source_type : str, default="Lakehouse"
         The type of source for the Direct Lake semantic model. Valid options: "Lakehouse", "Warehouse".
-    source_workspace : str | UUID, default=None
+    source_workspace : str | uuid.UUID, default=None
         The Fabric workspace name or ID used by the lakehouse/warehouse.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.

sempy_labs/lakehouse/__init__.py

@@ -20,7 +20,6 @@ from sempy_labs.lakehouse._shortcuts import (
 from sempy_labs.lakehouse._blobs import (
     recover_lakehouse_object,
     list_blobs,
-    get_user_delegation_key,
 )
 from sempy_labs.lakehouse._livy_sessions import (
     list_livy_sessions,
@@ -51,5 +50,4 @@ __all__ = [
     "delete_lakehouse",
     "update_lakehouse",
     "load_table",
-    "get_user_delegation_key",
 ]

sempy_labs/lakehouse/_blobs.py

@@ -244,40 +244,3 @@ def recover_lakehouse_object(
             print(
                 f"{icons.red_dot} An error occurred while recovering the '{blob_name}' blob: {e}"
             )
-
-
-def get_user_delegation_key():
-    """
-    Gets a key that can be used to sign a user delegation SAS (shared access signature). A user delegation SAS grants access to Azure Blob Storage resources by using Microsoft Entra credentials.
-
-    This is a wrapper function for the following API: `Get User Delegation Key <https://learn.microsoft.com/rest/api/storageservices/get-user-delegation-key>`_.
-
-    Returns
-    -------
-    str
-        The user delegation key value.
-    """
-
-    from datetime import datetime, timedelta, timezone
-
-    utc_now = datetime.now(timezone.utc)
-    start_time = utc_now + timedelta(minutes=2)
-    expiry_time = start_time + timedelta(minutes=60)
-    start_str = start_time.strftime("%Y-%m-%dT%H:%M:%SZ")
-    expiry_str = expiry_time.strftime("%Y-%m-%dT%H:%M:%SZ")
-
-    payload = f"""<?xml version="1.0" encoding="utf-8"?>
-    <KeyInfo>
-        <Start>{start_str}</Start>
-        <Expiry>{expiry_str}</Expiry>
-    </KeyInfo>"""
-
-    response = _request_blob_api(
-        request="?restype=service&comp=userdelegationkey",
-        method="post",
-        payload=payload,
-    )
-
-    root = ET.fromstring(response.content)
-    response_json = _xml_to_dict(root)
-    return response_json.get("UserDelegationKey", {}).get("Value", None)