semantic-link-labs 0.9.9__py3-none-any.whl → 0.9.11__py3-none-any.whl

sempy_labs/_kusto.py

@@ -0,0 +1,137 @@
+import requests
+import pandas as pd
+from sempy.fabric.exceptions import FabricHTTPException
+from sempy._utils._log import log
+import sempy_labs._icons as icons
+from typing import Optional
+from uuid import UUID
+from sempy_labs._kql_databases import _resolve_cluster_uri
+from sempy_labs._helper_functions import resolve_item_id
+
+
+@log
+def query_kusto(
+    query: str,
+    kql_database: str | UUID,
+    workspace: Optional[str | UUID] = None,
+    language: str = "kql",
+) -> pd.DataFrame:
+    """
+    Runs a KQL query against a KQL database.
+
+    Parameters
+    ----------
+    query : str
+        The query (supports KQL or SQL - make sure to specify the language parameter accordingly).
+    kql_database : str | uuid.UUID
+        The KQL database name or ID.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    language : str, default="kql"
+        The language of the query. Currently "kql' and "sql" are supported.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing the result of the KQL query.
+    """
+
+    import notebookutils
+
+    language = language.lower()
+    if language not in ["kql", "sql"]:
+        raise ValueError(
+            f"{icons._red_dot} Invalid language '{language}'. Only 'kql' and 'sql' are supported."
+        )
+
+    cluster_uri = _resolve_cluster_uri(kql_database=kql_database, workspace=workspace)
+    token = notebookutils.credentials.getToken(cluster_uri)
+
+    headers = {
+        "Authorization": f"Bearer {token}",
+        "Content-Type": "application/json",
+        "Accept": "application/json",
+    }
+
+    kql_database_id = resolve_item_id(
+        item=kql_database, type="KQLDatabase", workspace=workspace
+    )
+    payload = {"db": kql_database_id, "csl": query}
+    if language == "sql":
+        payload["properties"] = {"Options": {"query_language": "sql"}}
+
+    response = requests.post(
+        f"{cluster_uri}/v1/rest/query",
+        headers=headers,
+        json=payload,
+    )
+
+    if response.status_code != 200:
+        raise FabricHTTPException(response)
+
+    results = response.json()
+    columns_info = results["Tables"][0]["Columns"]
+    rows = results["Tables"][0]["Rows"]
+
+    df = pd.DataFrame(rows, columns=[col["ColumnName"] for col in columns_info])
+
+    return df
+    # for col_info in columns_info:
+    #    col_name = col_info["ColumnName"]
+    #    data_type = col_info["DataType"]
+
+    #    try:
+    #        if data_type == "DateTime":
+    #            df[col_name] = pd.to_datetime(df[col_name])
+    #        elif data_type in ["Int64", "Int32", "Long"]:
+    #            df[col_name] = (
+    #                pd.to_numeric(df[col_name], errors="coerce")
+    #                .fillna(0)
+    #                .astype("int64")
+    #            )
+    #        elif data_type == "Real" or data_type == "Double":
+    #            df[col_name] = pd.to_numeric(df[col_name], errors="coerce")
+    #        else:
+    #            # Convert any other type to string, change as needed
+    #            df[col_name] = df[col_name].astype(str)
+    #    except Exception as e:
+    #        print(
+    #            f"{icons.yellow_dot} Could not convert column {col_name} to {data_type}, defaulting to string: {str(e)}"
+    #        )
+    #        df[col_name] = df[col_name].astype(str)
+
+    return df
+
+
+@log
+def query_workspace_monitoring(
+    query: str, workspace: Optional[str | UUID] = None, language: str = "kql"
+) -> pd.DataFrame:
+    """
+    Runs a query against the Fabric workspace monitoring database. Workspace monitoring must be enabled on the workspace to use this function.
+
+    Parameters
+    ----------
+    query : str
+        The query (supports KQL or SQL - make sure to specify the language parameter accordingly).
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    language : str, default="kql"
+        The language of the query. Currently "kql' and "sql" are supported.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing the result of the query.
+    """
+
+    return query_kusto(
+        query=query,
+        kql_database="Monitoring KQL database",
+        workspace=workspace,
+        language=language,
+    )

sempy_labs/_list_functions.py

@@ -41,54 +41,32 @@ def get_object_level_security(
 
     from sempy_labs.tom import connect_semantic_model
 
-    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
-    (dataset_name, dataset_id) = resolve_dataset_name_and_id(dataset, workspace_id)
-
     columns = {
         "Role Name": "string",
         "Object Type": "string",
         "Table Name": "string",
         "Object Name": "string",
+        "Metadata Permission": "string",
     }
     df = _create_dataframe(columns=columns)
 
     with connect_semantic_model(
-        dataset=dataset_id, readonly=True, workspace=workspace_id
+        dataset=dataset, readonly=True, workspace=workspace
     ) as tom:
 
         for r in tom.model.Roles:
             for tp in r.TablePermissions:
-                if len(tp.FilterExpression) == 0:
-                    columnCount = 0
-                    try:
-                        columnCount = len(tp.ColumnPermissions)
-                    except Exception:
-                        pass
-                    objectType = "Table"
-                    if columnCount == 0:
-                        new_data = {
-                            "Role Name": r.Name,
-                            "Object Type": objectType,
-                            "Table Name": tp.Name,
-                            "Object Name": tp.Name,
-                        }
-                        df = pd.concat(
-                            [df, pd.DataFrame(new_data, index=[0])], ignore_index=True
-                        )
-                    else:
-                        objectType = "Column"
-                        for cp in tp.ColumnPermissions:
-                            new_data = {
-                                "Role Name": r.Name,
-                                "Object Type": objectType,
-                                "Table Name": tp.Name,
-                                "Object Name": cp.Name,
-                            }
-                            df = pd.concat(
-                                [df, pd.DataFrame(new_data, index=[0])],
-                                ignore_index=True,
-                            )
-
+                for cp in tp.ColumnPermissions:
+                    new_data = {
+                        "Role Name": r.Name,
+                        "Object Type": "Column",
+                        "Table Name": tp.Name,
+                        "Object Name": cp.Name,
+                        "Metadata Permission": cp.Permission,
+                    }
+                    df = pd.concat(
+                        [df, pd.DataFrame(new_data, index=[0])], ignore_index=True
+                    )
         return df
 
 
@@ -240,7 +218,11 @@ def list_tables(
                         "Columns": sum(
                             1 for c in t.Columns if str(c.Type) != "RowNumber"
                         ),
-                        "% DB": round((total_size / model_size) * 100, 2),
+                        "% DB": (
+                            round((total_size / model_size) * 100, 2)
+                            if model_size not in (0, None, float("nan"))
+                            else 0.0
+                        ),
                     }
                 )
 

sempy_labs/_model_bpa_rules.py

@@ -674,8 +674,18 @@ def model_bpa_rules(
                 "Provide format string for 'Date' columns",
                 lambda obj, tom: (re.search(r"date", obj.Name, flags=re.IGNORECASE))
                 and (obj.DataType == TOM.DataType.DateTime)
-                and (obj.FormatString != "mm/dd/yyyy"),
-                'Columns of type "DateTime" that have "Month" in their names should be formatted as "mm/dd/yyyy".',
+                and (
+                    obj.FormatString.lower()
+                    not in [
+                        "mm/dd/yyyy",
+                        "mm-dd-yyyy",
+                        "dd/mm/yyyy",
+                        "dd-mm-yyyy",
+                        "yyyy-mm-dd",
+                        "yyyy/mm/dd",
+                    ]
+                ),
+                'Columns of type "DateTime" that have "Date" in their names should be formatted.',
             ),
             (
                 "Formatting",
@@ -789,7 +799,7 @@ def model_bpa_rules(
                 "Formatting",
                 "Column",
                 "Warning",
-                'Provide format string for "Month" columns',
+                "Provide format string for 'Month' columns",
                 lambda obj, tom: re.search(r"month", obj.Name, flags=re.IGNORECASE)
                 and obj.DataType == TOM.DataType.DateTime
                 and obj.FormatString != "MMMM yyyy",

sempy_labs/_notebooks.py

@@ -7,6 +7,7 @@ import requests
 from sempy._utils._log import log
 from sempy_labs._helper_functions import (
     resolve_workspace_name_and_id,
+    resolve_workspace_id,
     _decode_b64,
     _base_api,
     resolve_item_id,
@@ -20,13 +21,20 @@ _notebook_prefix = "notebook-content."
 
 
 def _get_notebook_definition_base(
-    notebook_name: str, workspace: Optional[str | UUID] = None
+    notebook_name: str,
+    workspace: Optional[str | UUID] = None,
+    format: Optional[str] = None,
 ) -> pd.DataFrame:
 
-    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    workspace_id = resolve_workspace_id(workspace)
     item_id = resolve_item_id(item=notebook_name, type="Notebook", workspace=workspace)
+
+    url = f"v1/workspaces/{workspace_id}/notebooks/{item_id}/getDefinition"
+    if format == "ipynb":
+        url += f"?format={format}"
+
     result = _base_api(
-        request=f"v1/workspaces/{workspace_id}/notebooks/{item_id}/getDefinition",
+        request=url,
         method="post",
         lro_return_json=True,
         status_codes=None,
@@ -53,7 +61,10 @@ def _get_notebook_type(
 
 
 def get_notebook_definition(
-    notebook_name: str, workspace: Optional[str | UUID] = None, decode: bool = True
+    notebook_name: str,
+    workspace: Optional[str | UUID] = None,
+    decode: bool = True,
+    format: Optional[str] = None,
 ) -> str:
     """
     Obtains the notebook definition.
@@ -71,6 +82,9 @@ def get_notebook_definition(
     decode : bool, default=True
         If True, decodes the notebook definition file into .ipynb format.
         If False, obtains the notebook definition file in base64 format.
+    format : str, default=None
+        The only supported value is ipynb
+        If provided the format will be in standard .ipynb otherwise the format will be in source code format which is GIT friendly ipynb
 
     Returns
     -------
@@ -79,7 +93,7 @@ def get_notebook_definition(
     """
 
     df_items = _get_notebook_definition_base(
-        notebook_name=notebook_name, workspace=workspace
+        notebook_name=notebook_name, workspace=workspace, format=format
     )
     df_items_filt = df_items[df_items["path"].str.startswith(_notebook_prefix)]
     payload = df_items_filt["payload"].iloc[0]
@@ -163,6 +177,7 @@ def create_notebook(
     type: str = "py",
     description: Optional[str] = None,
     workspace: Optional[str | UUID] = None,
+    format: Optional[str] = None,
 ):
     """
     Creates a new notebook with a definition within a workspace.
@@ -182,20 +197,27 @@ def create_notebook(
         The name or ID of the workspace.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
+    format : str, default=None
+        If 'ipynb' is provided than notebook_content should be standard ipynb format
+        otherwise notebook_content should be GIT friendly format
     """
 
-    notebook_payload = base64.b64encode(notebook_content).decode("utf-8")
+    notebook_payload = base64.b64encode(notebook_content.encode("utf-8")).decode(
+        "utf-8"
+    )
     definition_payload = {
-        "format": "ipynb",
         "parts": [
             {
-                "path": f"{_notebook_prefix}.{type}",
+                "path": f"{_notebook_prefix}{type}",
                 "payload": notebook_payload,
                 "payloadType": "InlineBase64",
             }
         ],
     }
 
+    if format == "ipynb":
+        definition_payload["format"] = "ipynb"
+
     create_item(
         name=name,
         type="Notebook",
@@ -206,7 +228,10 @@ def create_notebook(
 
 
 def update_notebook_definition(
-    name: str, notebook_content: str, workspace: Optional[str | UUID] = None
+    name: str,
+    notebook_content: str,
+    workspace: Optional[str | UUID] = None,
+    format: Optional[str] = None,
 ):
     """
     Updates an existing notebook with a new definition.
@@ -221,10 +246,15 @@ def update_notebook_definition(
         The name or ID of the workspace.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
+    format : str, default=None
+        If 'ipynb' is provided than notebook_content should be standard ipynb format
+        otherwise notebook_content should be GIT friendly format
     """
 
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
-    notebook_payload = base64.b64encode(notebook_content)
+    notebook_payload = base64.b64encode(notebook_content.encode("utf-8")).decode(
+        "utf-8"
+    )
     item_id = resolve_item_id(item=name, type="Notebook", workspace=workspace)
     type = _get_notebook_type(notebook_name=name, workspace=workspace)
 
@@ -232,7 +262,7 @@ def update_notebook_definition(
         "definition": {
             "parts": [
                 {
-                    "path": f"{_notebook_prefix}.{type}",
+                    "path": f"{_notebook_prefix}{type}",
                     "payload": notebook_payload,
                     "payloadType": "InlineBase64",
                 }
@@ -240,6 +270,9 @@ def update_notebook_definition(
         },
     }
 
+    if format == "ipynb":
+        payload["definition"]["format"] = "ipynb"
+
     _base_api(
         request=f"v1/workspaces/{workspace_id}/notebooks/{item_id}/updateDefinition",
         payload=payload,

sempy_labs/_semantic_models.py

@@ -1,5 +1,5 @@
 from uuid import UUID
-from typing import Optional
+from typing import Optional, List
 import pandas as pd
 from sempy_labs._helper_functions import (
     _create_dataframe,
@@ -10,6 +10,7 @@ from sempy_labs._helper_functions import (
     delete_item,
 )
 import sempy_labs._icons as icons
+import re
 
 
 def get_semantic_model_refresh_schedule(
@@ -135,3 +136,94 @@ def delete_semantic_model(dataset: str | UUID, workspace: Optional[str | UUID] =
     """
 
     delete_item(item=dataset, type="SemanticModel", workspace=workspace)
+
+
+def update_semantic_model_refresh_schedule(
+    dataset: str | UUID,
+    days: Optional[str | List[str]] = None,
+    times: Optional[str | List[str]] = None,
+    time_zone: Optional[str] = None,
+    workspace: Optional[str | UUID] = None,
+):
+    """
+    Updates the refresh schedule for the specified dataset from the specified workspace.
+
+    This is a wrapper function for the following API: `Datasets - Update Refresh Schedule In Group <https://learn.microsoft.com/rest/api/power-bi/datasets/update-refresh-schedule-in-group>`_.
+
+    Parameters
+    ----------
+    dataset : str | uuid.UUID
+        Name or ID of the semantic model.
+    days : str | list[str], default=None
+        The days of the week to refresh the dataset.
+        Valid values are: "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday".
+        Defaults to None which means the refresh schedule will not be updated.
+    times : str | list[str], default=None
+        The times of the day to refresh the dataset.
+        Valid format is "HH:MM" (24-hour format).
+        Defaults to None which means the refresh schedule will not be updated.
+    time_zone : str, default=None
+        The time zone to use for the refresh schedule.
+        Defaults to None which means the refresh schedule will not be updated.
+    workspace : str | uuid.UUID, default=None
+        The workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    (dataset_name, dataset_id) = resolve_dataset_name_and_id(dataset, workspace)
+
+    payload = {"value": {}}
+
+    def is_valid_time_format(time_str):
+        pattern = r"^(?:[01]\d|2[0-3]):[0-5]\d$"
+        return re.match(pattern, time_str) is not None
+
+    weekdays = [
+        "Monday",
+        "Tuesday",
+        "Wednesday",
+        "Thursday",
+        "Friday",
+        "Sunday",
+        "Saturday",
+    ]
+    if days:
+        if isinstance(days, str):
+            days = [days]
+            for i in range(len(days)):
+                days[i] = days[i].capitalize()
+                if days[i] not in weekdays:
+                    raise ValueError(
+                        f"{icons.red_dot} Invalid day '{days[i]}'. Valid days are: {weekdays}"
+                    )
+        payload["value"]["days"] = days
+    if times:
+        if isinstance(times, str):
+            times = [times]
+            for i in range(len(times)):
+                if not is_valid_time_format(times[i]):
+                    raise ValueError(
+                        f"{icons.red_dot} Invalid time '{times[i]}'. Valid time format is 'HH:MM' (24-hour format)."
+                    )
+        payload["value"]["times"] = times
+    if time_zone:
+        payload["value"]["localTimeZoneId"] = time_zone
+
+    if not payload.get("value"):
+        print(
+            f"{icons.info} No changes were made to the refresh schedule for the '{dataset_name}' within the '{workspace_name}' workspace."
+        )
+        return
+
+    _base_api(
+        request=f"/v1.0/myorg/groups/{workspace_id}/datasets/{dataset_id}/refreshSchedule",
+        method="patch",
+        client="fabric_sp",
+        payload=payload,
+    )
+
+    print(
+        f"{icons.green_dot} Refresh schedule for the '{dataset_name}' within the '{workspace_name}' workspace has been updated."
+    )

sempy_labs/_sql.py

@@ -185,7 +185,7 @@ class ConnectWarehouse(ConnectBase):
 class ConnectLakehouse(ConnectBase):
     def __init__(
         self,
-        lakehouse: str | UUID,
+        lakehouse: Optional[str | UUID] = None,
         workspace: Optional[Union[str, UUID]] = None,
         timeout: int = 30,
     ):
@@ -194,8 +194,9 @@ class ConnectLakehouse(ConnectBase):
 
         Parameters
         ----------
-        lakehouse : str | uuid.UUID
+        lakehouse : str | uuid.UUID, default=None
             The name or ID of the Fabric lakehouse.
+            Defaults to None which resolves to the lakehouse attached to the notebook.
         workspace : str | uuid.UUID, default=None
             The name or ID of the workspace.
             Defaults to None which resolves to the workspace of the attached lakehouse