semantic-link-labs 0.9.10__py3-none-any.whl → 0.10.0__py3-none-any.whl

sempy_labs/_kusto.py

@@ -77,33 +77,35 @@ def query_kusto(
 
     df = pd.DataFrame(rows, columns=[col["ColumnName"] for col in columns_info])
 
-    for col_info in columns_info:
-        col_name = col_info["ColumnName"]
-        data_type = col_info["DataType"]
-
-        try:
-            if data_type == "DateTime":
-                df[col_name] = pd.to_datetime(df[col_name])
-            elif data_type in ["Int64", "Int32", "Long"]:
-                df[col_name] = (
-                    pd.to_numeric(df[col_name], errors="coerce")
-                    .fillna(0)
-                    .astype("int64")
-                )
-            elif data_type == "Real" or data_type == "Double":
-                df[col_name] = pd.to_numeric(df[col_name], errors="coerce")
-            else:
-                # Convert any other type to string, change as needed
-                df[col_name] = df[col_name].astype(str)
-        except Exception as e:
-            print(
-                f"{icons.yellow_dot} Could not convert column {col_name} to {data_type}, defaulting to string: {str(e)}"
-            )
-            df[col_name] = df[col_name].astype(str)
+    return df
+    # for col_info in columns_info:
+    #    col_name = col_info["ColumnName"]
+    #    data_type = col_info["DataType"]
+
+    #    try:
+    #        if data_type == "DateTime":
+    #            df[col_name] = pd.to_datetime(df[col_name])
+    #        elif data_type in ["Int64", "Int32", "Long"]:
+    #            df[col_name] = (
+    #                pd.to_numeric(df[col_name], errors="coerce")
+    #                .fillna(0)
+    #                .astype("int64")
+    #            )
+    #        elif data_type == "Real" or data_type == "Double":
+    #            df[col_name] = pd.to_numeric(df[col_name], errors="coerce")
+    #        else:
+    #            # Convert any other type to string, change as needed
+    #            df[col_name] = df[col_name].astype(str)
+    #    except Exception as e:
+    #        print(
+    #            f"{icons.yellow_dot} Could not convert column {col_name} to {data_type}, defaulting to string: {str(e)}"
+    #        )
+    #        df[col_name] = df[col_name].astype(str)
 
     return df
 
 
+@log
 def query_workspace_monitoring(
     query: str, workspace: Optional[str | UUID] = None, language: str = "kql"
 ) -> pd.DataFrame:

sempy_labs/_list_functions.py

@@ -41,54 +41,32 @@ def get_object_level_security(
 
     from sempy_labs.tom import connect_semantic_model
 
-    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
-    (dataset_name, dataset_id) = resolve_dataset_name_and_id(dataset, workspace_id)
-
     columns = {
         "Role Name": "string",
         "Object Type": "string",
         "Table Name": "string",
         "Object Name": "string",
+        "Metadata Permission": "string",
     }
     df = _create_dataframe(columns=columns)
 
     with connect_semantic_model(
-        dataset=dataset_id, readonly=True, workspace=workspace_id
+        dataset=dataset, readonly=True, workspace=workspace
     ) as tom:
 
         for r in tom.model.Roles:
             for tp in r.TablePermissions:
-                if len(tp.FilterExpression) == 0:
-                    columnCount = 0
-                    try:
-                        columnCount = len(tp.ColumnPermissions)
-                    except Exception:
-                        pass
-                    objectType = "Table"
-                    if columnCount == 0:
-                        new_data = {
-                            "Role Name": r.Name,
-                            "Object Type": objectType,
-                            "Table Name": tp.Name,
-                            "Object Name": tp.Name,
-                        }
-                        df = pd.concat(
-                            [df, pd.DataFrame(new_data, index=[0])], ignore_index=True
-                        )
-                    else:
-                        objectType = "Column"
-                        for cp in tp.ColumnPermissions:
-                            new_data = {
-                                "Role Name": r.Name,
-                                "Object Type": objectType,
-                                "Table Name": tp.Name,
-                                "Object Name": cp.Name,
-                            }
-                            df = pd.concat(
-                                [df, pd.DataFrame(new_data, index=[0])],
-                                ignore_index=True,
-                            )
-
+                for cp in tp.ColumnPermissions:
+                    new_data = {
+                        "Role Name": r.Name,
+                        "Object Type": "Column",
+                        "Table Name": tp.Name,
+                        "Object Name": cp.Name,
+                        "Metadata Permission": cp.Permission,
+                    }
+                    df = pd.concat(
+                        [df, pd.DataFrame(new_data, index=[0])], ignore_index=True
+                    )
         return df
 
 

sempy_labs/_model_bpa_rules.py

@@ -674,8 +674,18 @@ def model_bpa_rules(
                 "Provide format string for 'Date' columns",
                 lambda obj, tom: (re.search(r"date", obj.Name, flags=re.IGNORECASE))
                 and (obj.DataType == TOM.DataType.DateTime)
-                and (obj.FormatString != "mm/dd/yyyy"),
-                'Columns of type "DateTime" that have "Month" in their names should be formatted as "mm/dd/yyyy".',
+                and (
+                    obj.FormatString.lower()
+                    not in [
+                        "mm/dd/yyyy",
+                        "mm-dd-yyyy",
+                        "dd/mm/yyyy",
+                        "dd-mm-yyyy",
+                        "yyyy-mm-dd",
+                        "yyyy/mm/dd",
+                    ]
+                ),
+                'Columns of type "DateTime" that have "Date" in their names should be formatted.',
             ),
             (
                 "Formatting",
@@ -789,7 +799,7 @@ def model_bpa_rules(
                 "Formatting",
                 "Column",
                 "Warning",
-                'Provide format string for "Month" columns',
+                "Provide format string for 'Month' columns",
                 lambda obj, tom: re.search(r"month", obj.Name, flags=re.IGNORECASE)
                 and obj.DataType == TOM.DataType.DateTime
                 and obj.FormatString != "MMMM yyyy",

sempy_labs/_notebooks.py

@@ -7,6 +7,7 @@ import requests
 from sempy._utils._log import log
 from sempy_labs._helper_functions import (
     resolve_workspace_name_and_id,
+    resolve_workspace_id,
     _decode_b64,
     _base_api,
     resolve_item_id,
@@ -20,13 +21,20 @@ _notebook_prefix = "notebook-content."
 
 
 def _get_notebook_definition_base(
-    notebook_name: str, workspace: Optional[str | UUID] = None
+    notebook_name: str,
+    workspace: Optional[str | UUID] = None,
+    format: Optional[str] = None,
 ) -> pd.DataFrame:
 
-    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    workspace_id = resolve_workspace_id(workspace)
     item_id = resolve_item_id(item=notebook_name, type="Notebook", workspace=workspace)
+
+    url = f"v1/workspaces/{workspace_id}/notebooks/{item_id}/getDefinition"
+    if format == "ipynb":
+        url += f"?format={format}"
+
     result = _base_api(
-        request=f"v1/workspaces/{workspace_id}/notebooks/{item_id}/getDefinition",
+        request=url,
         method="post",
         lro_return_json=True,
         status_codes=None,
@@ -53,7 +61,10 @@ def _get_notebook_type(
 
 
 def get_notebook_definition(
-    notebook_name: str, workspace: Optional[str | UUID] = None, decode: bool = True
+    notebook_name: str,
+    workspace: Optional[str | UUID] = None,
+    decode: bool = True,
+    format: Optional[str] = None,
 ) -> str:
     """
     Obtains the notebook definition.
@@ -71,6 +82,9 @@ def get_notebook_definition(
     decode : bool, default=True
         If True, decodes the notebook definition file into .ipynb format.
         If False, obtains the notebook definition file in base64 format.
+    format : str, default=None
+        The only supported value is ipynb
+        If provided the format will be in standard .ipynb otherwise the format will be in source code format which is GIT friendly ipynb
 
     Returns
     -------
@@ -79,7 +93,7 @@ def get_notebook_definition(
     """
 
     df_items = _get_notebook_definition_base(
-        notebook_name=notebook_name, workspace=workspace
+        notebook_name=notebook_name, workspace=workspace, format=format
     )
     df_items_filt = df_items[df_items["path"].str.startswith(_notebook_prefix)]
     payload = df_items_filt["payload"].iloc[0]
@@ -163,6 +177,7 @@ def create_notebook(
     type: str = "py",
     description: Optional[str] = None,
     workspace: Optional[str | UUID] = None,
+    format: Optional[str] = None,
 ):
     """
     Creates a new notebook with a definition within a workspace.
@@ -182,20 +197,27 @@ def create_notebook(
         The name or ID of the workspace.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
+    format : str, default=None
+        If 'ipynb' is provided than notebook_content should be standard ipynb format
+        otherwise notebook_content should be GIT friendly format
     """
 
-    notebook_payload = base64.b64encode(notebook_content).decode("utf-8")
+    notebook_payload = base64.b64encode(notebook_content.encode("utf-8")).decode(
+        "utf-8"
+    )
     definition_payload = {
-        "format": "ipynb",
         "parts": [
             {
-                "path": f"{_notebook_prefix}.{type}",
+                "path": f"{_notebook_prefix}{type}",
                 "payload": notebook_payload,
                 "payloadType": "InlineBase64",
             }
         ],
     }
 
+    if format == "ipynb":
+        definition_payload["format"] = "ipynb"
+
     create_item(
         name=name,
         type="Notebook",
@@ -206,7 +228,10 @@ def create_notebook(
 
 
 def update_notebook_definition(
-    name: str, notebook_content: str, workspace: Optional[str | UUID] = None
+    name: str,
+    notebook_content: str,
+    workspace: Optional[str | UUID] = None,
+    format: Optional[str] = None,
 ):
     """
     Updates an existing notebook with a new definition.
@@ -221,10 +246,15 @@ def update_notebook_definition(
         The name or ID of the workspace.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
+    format : str, default=None
+        If 'ipynb' is provided than notebook_content should be standard ipynb format
+        otherwise notebook_content should be GIT friendly format
     """
 
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
-    notebook_payload = base64.b64encode(notebook_content)
+    notebook_payload = base64.b64encode(notebook_content.encode("utf-8")).decode(
+        "utf-8"
+    )
     item_id = resolve_item_id(item=name, type="Notebook", workspace=workspace)
     type = _get_notebook_type(notebook_name=name, workspace=workspace)
 
@@ -232,7 +262,7 @@ def update_notebook_definition(
         "definition": {
             "parts": [
                 {
-                    "path": f"{_notebook_prefix}.{type}",
+                    "path": f"{_notebook_prefix}{type}",
                     "payload": notebook_payload,
                     "payloadType": "InlineBase64",
                 }
@@ -240,6 +270,9 @@ def update_notebook_definition(
         },
     }
 
+    if format == "ipynb":
+        payload["definition"]["format"] = "ipynb"
+
     _base_api(
         request=f"v1/workspaces/{workspace_id}/notebooks/{item_id}/updateDefinition",
         payload=payload,

sempy_labs/_semantic_models.py

@@ -1,5 +1,5 @@
 from uuid import UUID
-from typing import Optional
+from typing import Optional, List
 import pandas as pd
 from sempy_labs._helper_functions import (
     _create_dataframe,
@@ -10,6 +10,7 @@ from sempy_labs._helper_functions import (
     delete_item,
 )
 import sempy_labs._icons as icons
+import re
 
 
 def get_semantic_model_refresh_schedule(
@@ -135,3 +136,94 @@ def delete_semantic_model(dataset: str | UUID, workspace: Optional[str | UUID] =
     """
 
     delete_item(item=dataset, type="SemanticModel", workspace=workspace)
+
+
+def update_semantic_model_refresh_schedule(
+    dataset: str | UUID,
+    days: Optional[str | List[str]] = None,
+    times: Optional[str | List[str]] = None,
+    time_zone: Optional[str] = None,
+    workspace: Optional[str | UUID] = None,
+):
+    """
+    Updates the refresh schedule for the specified dataset from the specified workspace.
+
+    This is a wrapper function for the following API: `Datasets - Update Refresh Schedule In Group <https://learn.microsoft.com/rest/api/power-bi/datasets/update-refresh-schedule-in-group>`_.
+
+    Parameters
+    ----------
+    dataset : str | uuid.UUID
+        Name or ID of the semantic model.
+    days : str | list[str], default=None
+        The days of the week to refresh the dataset.
+        Valid values are: "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday".
+        Defaults to None which means the refresh schedule will not be updated.
+    times : str | list[str], default=None
+        The times of the day to refresh the dataset.
+        Valid format is "HH:MM" (24-hour format).
+        Defaults to None which means the refresh schedule will not be updated.
+    time_zone : str, default=None
+        The time zone to use for the refresh schedule.
+        Defaults to None which means the refresh schedule will not be updated.
+    workspace : str | uuid.UUID, default=None
+        The workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    (dataset_name, dataset_id) = resolve_dataset_name_and_id(dataset, workspace)
+
+    payload = {"value": {}}
+
+    def is_valid_time_format(time_str):
+        pattern = r"^(?:[01]\d|2[0-3]):[0-5]\d$"
+        return re.match(pattern, time_str) is not None
+
+    weekdays = [
+        "Monday",
+        "Tuesday",
+        "Wednesday",
+        "Thursday",
+        "Friday",
+        "Sunday",
+        "Saturday",
+    ]
+    if days:
+        if isinstance(days, str):
+            days = [days]
+            for i in range(len(days)):
+                days[i] = days[i].capitalize()
+                if days[i] not in weekdays:
+                    raise ValueError(
+                        f"{icons.red_dot} Invalid day '{days[i]}'. Valid days are: {weekdays}"
+                    )
+        payload["value"]["days"] = days
+    if times:
+        if isinstance(times, str):
+            times = [times]
+            for i in range(len(times)):
+                if not is_valid_time_format(times[i]):
+                    raise ValueError(
+                        f"{icons.red_dot} Invalid time '{times[i]}'. Valid time format is 'HH:MM' (24-hour format)."
+                    )
+        payload["value"]["times"] = times
+    if time_zone:
+        payload["value"]["localTimeZoneId"] = time_zone
+
+    if not payload.get("value"):
+        print(
+            f"{icons.info} No changes were made to the refresh schedule for the '{dataset_name}' within the '{workspace_name}' workspace."
+        )
+        return
+
+    _base_api(
+        request=f"/v1.0/myorg/groups/{workspace_id}/datasets/{dataset_id}/refreshSchedule",
+        method="patch",
+        client="fabric_sp",
+        payload=payload,
+    )
+
+    print(
+        f"{icons.green_dot} Refresh schedule for the '{dataset_name}' within the '{workspace_name}' workspace has been updated."
+    )

sempy_labs/_sql.py

@@ -82,7 +82,7 @@ class ConnectBase:
             )
 
         # Set up the connection string
-        access_token = SynapseTokenProvider()()
+        access_token = SynapseTokenProvider()("sql")
         tokenstruct = _bytes2mswin_bstr(access_token.encode())
         if endpoint_type == "sqldatabase":
             conn_str = f"DRIVER={{ODBC Driver 18 for SQL Server}};SERVER={tds_endpoint};DATABASE={resource_name}-{resource_id};Encrypt=Yes;"
@@ -185,7 +185,7 @@ class ConnectWarehouse(ConnectBase):
 class ConnectLakehouse(ConnectBase):
     def __init__(
         self,
-        lakehouse: str | UUID,
+        lakehouse: Optional[str | UUID] = None,
         workspace: Optional[Union[str, UUID]] = None,
         timeout: int = 30,
     ):
@@ -194,8 +194,9 @@ class ConnectLakehouse(ConnectBase):
 
         Parameters
         ----------
-        lakehouse : str | uuid.UUID
+        lakehouse : str | uuid.UUID, default=None
             The name or ID of the Fabric lakehouse.
+            Defaults to None which resolves to the lakehouse attached to the notebook.
         workspace : str | uuid.UUID, default=None
             The name or ID of the workspace.
             Defaults to None which resolves to the workspace of the attached lakehouse

sempy_labs/_tags.py

@@ -0,0 +1,194 @@
+from sempy_labs._helper_functions import (
+    _base_api,
+    _create_dataframe,
+    _update_dataframe_datatypes,
+    resolve_item_name_and_id,
+    resolve_workspace_name_and_id,
+    _is_valid_uuid,
+)
+import pandas as pd
+from typing import Optional, List
+from uuid import UUID
+import sempy_labs._icons as icons
+
+
+def list_tags() -> pd.DataFrame:
+    """
+    Shows a list of all the tenant's tags.
+
+    This is a wrapper function for the following API: `Tags - List Tags <https://learn.microsoft.com/rest/api/fabric/core/tags/list-tags>`_.
+
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing a list of all the tenant's tags.
+    """
+
+    columns = {
+        "Tag Name": "string",
+        "Tag Id": "string",
+    }
+    df = _create_dataframe(columns=columns)
+
+    responses = _base_api(
+        request="/v1/tags",
+        uses_pagination=True,
+        client="fabric_sp",
+    )
+
+    dfs = []
+
+    for r in responses:
+        for v in r.get("value", []):
+            new_data = {
+                "Tag Name": v.get("displayName"),
+                "Tag Id": v.get("id"),
+            }
+            dfs.append(pd.DataFrame(new_data, index=[0]))
+
+    if dfs:
+        df = pd.concat(dfs, ignore_index=True)
+        _update_dataframe_datatypes(dataframe=df, column_map=columns)
+
+    return df
+
+
+def resolve_tags(tags: str | List[str]) -> List[str]:
+    """
+    Resolves the tags to a list of strings.
+
+    Parameters
+    ----------
+    tags : str | List[str]
+        The tags to resolve.
+
+    Returns
+    -------
+    List[str]
+        A list of resolved tags.
+    """
+
+    if isinstance(tags, str):
+        tags = [tags]
+
+    if all(_is_valid_uuid(tag) for tag in tags):
+        return tags
+
+    df = list_tags()
+
+    tag_list = []
+    for tag in tags:
+        if _is_valid_uuid(tag):
+            tag_list.append(tag)
+        else:
+            df_filt = df[df["Tag Name"] == tag]
+            if df_filt.empty:
+                raise ValueError(f"Tag '{tag}' not found in the tenant's tags.")
+            tag_id = df_filt["Tag Id"].iloc[0]
+            tag_list.append(tag_id)
+
+    return tag_list
+
+
+def apply_tags(
+    item: str | UUID,
+    type: str,
+    tags: str | UUID | List[str | UUID],
+    workspace: Optional[str | UUID] = None,
+):
+    """
+    Shows a list of all the tenant's tags.
+
+    This is a wrapper function for the following API: `Tags - Apply Tags <https://learn.microsoft.com/rest/api/fabric/core/tags/apply-tags>`_.
+
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
+    Parameters
+    ----------
+    item : str | uuid.UUID
+        The name or ID of the item to apply tags to.
+    type : str
+        The type of the item to apply tags to. For example: "Lakehouse".
+    tags : str | uuid.UUID | List[str | uuid.UUID]
+        The name or ID of the tag(s) to apply to the item.
+    workspace : str | uuid.UUID, default=None
+        The workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    (item_name, item_id) = resolve_item_name_and_id(item, type, workspace_id)
+
+    if isinstance(tags, str):
+        tags = [tags]
+
+    tag_list = resolve_tags(tags)
+
+    payload = {
+        "tags": tag_list,
+    }
+
+    _base_api(
+        request=f"/v1/workspaces/{workspace_id}/items/{item_id}/applyTags",
+        client="fabric_sp",
+        method="post",
+        payload=payload,
+    )
+
+    print(
+        f"{icons.green_dot} Tags {tags} applied to the '{item_name}' {type.lower()} within the '{workspace_name}' workspace"
+    )
+
+
+def unapply_tags(
+    item: str | UUID,
+    type: str,
+    tags: str | UUID | List[str | UUID],
+    workspace: Optional[str | UUID] = None,
+):
+    """
+    Shows a list of all the tenant's tags.
+
+    This is a wrapper function for the following API: `Tags - Unapply Tags <https://learn.microsoft.com/rest/api/fabric/core/tags/unapply-tags>`_.
+
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
+    Parameters
+    ----------
+    item : str | uuid.UUID
+        The name or ID of the item to apply tags to.
+    type : str
+        The type of the item to apply tags to. For example: "Lakehouse".
+    tags : str | uuid.UUID | List[str | uuid.UUID]
+        The name or ID of the tag(s) to apply to the item.
+    workspace : str | uuid.UUID, default=None
+        The workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    (item_name, item_id) = resolve_item_name_and_id(item, type, workspace_id)
+
+    if isinstance(tags, str):
+        tags = [tags]
+
+    tag_list = resolve_tags(tags)
+
+    payload = {
+        "tags": tag_list,
+    }
+
+    _base_api(
+        request=f"/v1/workspaces/{workspace_id}/items/{item_id}/unapplyTags",
+        client="fabric_sp",
+        method="post",
+        payload=payload,
+    )
+
+    print(
+        f"{icons.green_dot} Tags {tags} applied to the '{item_name}' {type.lower()} within the '{workspace_name}' workspace"
+    )

sempy_labs/_user_delegation_key.py

@@ -0,0 +1,42 @@
+from sempy_labs.lakehouse._blobs import _request_blob_api
+from sempy_labs._helper_functions import (
+    _xml_to_dict,
+)
+from datetime import datetime, timedelta, timezone
+import xml.etree.ElementTree as ET
+
+
+def get_user_delegation_key():
+    """
+    Gets a key that can be used to sign a user delegation SAS (shared access signature). A user delegation SAS grants access to Azure Blob Storage resources by using Microsoft Entra credentials.
+
+    This is a wrapper function for the following API: `Get User Delegation Key <https://learn.microsoft.com/rest/api/storageservices/get-user-delegation-key>`_.
+
+    Returns
+    -------
+    str
+        The user delegation key value.
+    """
+
+    utc_now = datetime.now(timezone.utc)
+    start_time = utc_now + timedelta(minutes=2)
+    expiry_time = start_time + timedelta(minutes=60)
+    start_str = start_time.strftime("%Y-%m-%dT%H:%M:%SZ")
+    expiry_str = expiry_time.strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    payload = f"""<?xml version="1.0" encoding="utf-8"?>
+    <KeyInfo>
+        <Start>{start_str}</Start>
+        <Expiry>{expiry_str}</Expiry>
+    </KeyInfo>"""
+
+    response = _request_blob_api(
+        request="?restype=service&comp=userdelegationkey",
+        method="post",
+        payload=payload,
+    )
+
+    root = ET.fromstring(response.content)
+    response_json = _xml_to_dict(root)
+
+    return response_json.get("UserDelegationKey", {}).get("Value", None)