semantic-link-labs 0.12.8__py3-none-any.whl

sempy_labs/_kql_databases.py

@@ -0,0 +1,149 @@
+import pandas as pd
+from typing import Optional
+from sempy_labs._helper_functions import (
+    _base_api,
+    _create_dataframe,
+    delete_item,
+    create_item,
+    resolve_item_id,
+    resolve_workspace_id,
+)
+from uuid import UUID
+import sempy_labs._icons as icons
+from sempy._utils._log import log
+
+
+@log
+def list_kql_databases(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
+    """
+    Shows the KQL databases within a workspace.
+
+    This is a wrapper function for the following API: `Items - List KQL Databases <https://learn.microsoft.com/rest/api/fabric/kqldatabase/items/list-kql-databases>`_.
+
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
+    Parameters
+    ----------
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing the KQL databases within a workspace.
+    """
+
+    columns = {
+        "KQL Database Name": "string",
+        "KQL Database Id": "string",
+        "Description": "string",
+        "Parent Eventhouse Item Id": "string",
+        "Query Service URI": "string",
+        "Ingestion Service URI": "string",
+        "Database Type": "string",
+    }
+    df = _create_dataframe(columns=columns)
+
+    workspace_id = resolve_workspace_id(workspace)
+
+    responses = _base_api(
+        request=f"v1/workspaces/{workspace_id}/kqlDatabases",
+        uses_pagination=True,
+        client="fabric_sp",
+    )
+
+    rows = []
+    for r in responses:
+        for v in r.get("value", []):
+            prop = v.get("properties", {})
+            rows.append(
+                {
+                    "KQL Database Name": v.get("displayName"),
+                    "KQL Database Id": v.get("id"),
+                    "Description": v.get("description"),
+                    "Parent Eventhouse Item Id": prop.get("parentEventhouseItemId"),
+                    "Query Service URI": prop.get("queryServiceUri"),
+                    "Ingestion Service URI": prop.get("ingestionServiceUri"),
+                    "Database Type": prop.get("databaseType"),
+                }
+            )
+
+    if rows:
+        df = pd.DataFrame(rows, columns=list(columns.keys()))
+
+    return df
+
+
+@log
+def _create_kql_database(
+    name: str, description: Optional[str] = None, workspace: Optional[str | UUID] = None
+):
+    """
+    Creates a KQL database.
+
+    This is a wrapper function for the following API: `Items - Create KQL Database <https://learn.microsoft.com/rest/api/fabric/kqldatabase/items/create-kql-database>`_.
+
+    Parameters
+    ----------
+    name: str
+        Name of the KQL database.
+    description : str, default=None
+        A description of the environment.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    create_item(
+        name=name, description=description, type="KQLDatabase", workspace=workspace
+    )
+
+
+@log
+def delete_kql_database(
+    kql_database: str | UUID,
+    workspace: Optional[str | UUID] = None,
+    **kwargs,
+):
+    """
+    Deletes a KQL database.
+
+    This is a wrapper function for the following API: `Items - Delete KQL Database <https://learn.microsoft.com/rest/api/fabric/kqldatabase/items/delete-kql-database>`_.
+
+    Parameters
+    ----------
+    kql_database: str | uuid.UUID
+        Name or ID of the KQL database.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    if "name" in kwargs:
+        kql_database = kwargs["name"]
+        print(
+            f"{icons.warning} The 'name' parameter is deprecated. Please use 'kql_database' instead."
+        )
+
+    delete_item(item=kql_database, type="KQLDatabase", workspace=workspace)
+
+
+@log
+def _resolve_cluster_uri(
+    kql_database: str | UUID, workspace: Optional[str | UUID] = None
+) -> str:
+
+    workspace_id = resolve_workspace_id(workspace=workspace)
+    item_id = resolve_item_id(
+        item=kql_database, type="KQLDatabase", workspace=workspace
+    )
+    response = _base_api(
+        request=f"/v1/workspaces/{workspace_id}/kqlDatabases/{item_id}",
+        client="fabric_sp",
+    )
+
+    return response.json().get("properties", {}).get("queryServiceUri")

sempy_labs/_kql_querysets.py

@@ -0,0 +1,124 @@
+import pandas as pd
+import sempy_labs._icons as icons
+from typing import Optional
+from sempy_labs._helper_functions import (
+    resolve_workspace_id,
+    _base_api,
+    _create_dataframe,
+    delete_item,
+    create_item,
+)
+from uuid import UUID
+from sempy._utils._log import log
+
+
+@log
+def list_kql_querysets(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
+    """
+    Shows the KQL querysets within a workspace.
+
+    This is a wrapper function for the following API: `Items - List KQL Querysets <https://learn.microsoft.com/rest/api/fabric/kqlqueryset/items/list-kql-querysets>`_.
+
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
+    Parameters
+    ----------
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing the KQL querysets within a workspace.
+    """
+
+    columns = {
+        "KQL Queryset Name": "string",
+        "KQL Queryset Id": "string",
+        "Description": "string",
+    }
+    df = _create_dataframe(columns=columns)
+
+    workspace_id = resolve_workspace_id(workspace)
+
+    responses = _base_api(
+        request=f"v1/workspaces/{workspace_id}/kqlQuerysets",
+        uses_pagination=True,
+        client="fabric_sp",
+    )
+
+    rows = []
+    for r in responses:
+        for v in r.get("value", []):
+            rows.append(
+                {
+                    "KQL Queryset Name": v.get("displayName"),
+                    "KQL Queryset Id": v.get("id"),
+                    "Description": v.get("description"),
+                }
+            )
+
+    if rows:
+        df = pd.DataFrame(rows, columns=list(columns.keys()))
+
+    return df
+
+
+@log
+def create_kql_queryset(
+    name: str, description: Optional[str] = None, workspace: Optional[str | UUID] = None
+):
+    """
+    Creates a KQL queryset.
+
+    This is a wrapper function for the following API: `Items - Create KQL Queryset <https://learn.microsoft.com/rest/api/fabric/kqlqueryset/items/create-kql-queryset>`_.
+
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
+    Parameters
+    ----------
+    name: str
+        Name of the KQL queryset.
+    description : str, default=None
+        A description of the environment.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    create_item(
+        name=name, description=description, type="KQLQueryset", workspace=workspace
+    )
+
+
+@log
+def delete_kql_queryset(
+    kql_queryset: str | UUID, workspace: Optional[str | UUID] = None, **kwargs
+):
+    """
+    Deletes a KQL queryset.
+
+    This is a wrapper function for the following API: `Items - Delete KQL Queryset <https://learn.microsoft.com/rest/api/fabric/kqlqueryset/items/delete-kql-queryset>`_.
+
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
+    Parameters
+    ----------
+    kql_queryset: str | uuid.UUID
+        Name or ID of the KQL queryset.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    if "name" in kwargs:
+        kql_queryset = kwargs["name"]
+        print(
+            f"{icons.warning} The 'name' parameter is deprecated. Please use 'kql_queryset' instead."
+        )
+
+    delete_item(item=kql_queryset, type="KQLQueryset", workspace=workspace)

sempy_labs/_kusto.py

@@ -0,0 +1,137 @@
+import requests
+import pandas as pd
+from sempy.fabric.exceptions import FabricHTTPException
+from sempy._utils._log import log
+import sempy_labs._icons as icons
+from typing import Optional
+from uuid import UUID
+from sempy_labs._kql_databases import _resolve_cluster_uri
+from sempy_labs._helper_functions import resolve_item_id
+
+
+@log
+def query_kusto(
+    query: str,
+    kql_database: str | UUID,
+    workspace: Optional[str | UUID] = None,
+    language: str = "kql",
+) -> pd.DataFrame:
+    """
+    Runs a KQL query against a KQL database.
+
+    Parameters
+    ----------
+    query : str
+        The query (supports KQL or SQL - make sure to specify the language parameter accordingly).
+    kql_database : str | uuid.UUID
+        The KQL database name or ID.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    language : str, default="kql"
+        The language of the query. Currently "kql' and "sql" are supported.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing the result of the KQL query.
+    """
+
+    import notebookutils
+
+    language = language.lower()
+    if language not in ["kql", "sql"]:
+        raise ValueError(
+            f"{icons._red_dot} Invalid language '{language}'. Only 'kql' and 'sql' are supported."
+        )
+
+    cluster_uri = _resolve_cluster_uri(kql_database=kql_database, workspace=workspace)
+    token = notebookutils.credentials.getToken(cluster_uri)
+
+    headers = {
+        "Authorization": f"Bearer {token}",
+        "Content-Type": "application/json",
+        "Accept": "application/json",
+    }
+
+    kql_database_id = resolve_item_id(
+        item=kql_database, type="KQLDatabase", workspace=workspace
+    )
+    payload = {"db": kql_database_id, "csl": query}
+    if language == "sql":
+        payload["properties"] = {"Options": {"query_language": "sql"}}
+
+    response = requests.post(
+        f"{cluster_uri}/v1/rest/query",
+        headers=headers,
+        json=payload,
+    )
+
+    if response.status_code != 200:
+        raise FabricHTTPException(response)
+
+    results = response.json()
+    columns_info = results["Tables"][0]["Columns"]
+    rows = results["Tables"][0]["Rows"]
+
+    df = pd.DataFrame(rows, columns=[col["ColumnName"] for col in columns_info])
+
+    return df
+    # for col_info in columns_info:
+    #    col_name = col_info["ColumnName"]
+    #    data_type = col_info["DataType"]
+
+    #    try:
+    #        if data_type == "DateTime":
+    #            df[col_name] = pd.to_datetime(df[col_name])
+    #        elif data_type in ["Int64", "Int32", "Long"]:
+    #            df[col_name] = (
+    #                pd.to_numeric(df[col_name], errors="coerce")
+    #                .fillna(0)
+    #                .astype("int64")
+    #            )
+    #        elif data_type == "Real" or data_type == "Double":
+    #            df[col_name] = pd.to_numeric(df[col_name], errors="coerce")
+    #        else:
+    #            # Convert any other type to string, change as needed
+    #            df[col_name] = df[col_name].astype(str)
+    #    except Exception as e:
+    #        print(
+    #            f"{icons.yellow_dot} Could not convert column {col_name} to {data_type}, defaulting to string: {str(e)}"
+    #        )
+    #        df[col_name] = df[col_name].astype(str)
+
+    return df
+
+
+@log
+def query_workspace_monitoring(
+    query: str, workspace: Optional[str | UUID] = None, language: str = "kql"
+) -> pd.DataFrame:
+    """
+    Runs a query against the Fabric workspace monitoring database. Workspace monitoring must be enabled on the workspace to use this function.
+
+    Parameters
+    ----------
+    query : str
+        The query (supports KQL or SQL - make sure to specify the language parameter accordingly).
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    language : str, default="kql"
+        The language of the query. Currently "kql' and "sql" are supported.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing the result of the query.
+    """
+
+    return query_kusto(
+        query=query,
+        kql_database="Monitoring KQL database",
+        workspace=workspace,
+        language=language,
+    )

sempy_labs/_labels.py

@@ -0,0 +1,124 @@
+import sempy.fabric as fabric
+import requests
+import pandas as pd
+from typing import Optional, Union
+from uuid import UUID
+from sempy.fabric.exceptions import FabricHTTPException
+from sempy._utils._log import log
+from sempy_labs._helper_functions import (
+    _get_url_prefix,
+)
+
+
+@log
+def list_item_labels(workspace: Optional[Union[str, UUID]] = None) -> pd.DataFrame:
+    """
+    List all items within a workspace and shows their sensitivity labels.
+
+    NOTE: This function uses an internal API and is subject to change/break without notice.
+
+    Parameters
+    ----------
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing a list of all items within a workspace and their sensitivity labels.
+    """
+
+    import notebookutils
+
+    token = notebookutils.credentials.getToken("pbi")
+    headers = {"Authorization": f"Bearer {token}"}
+
+    # Item types handled in special payload fields
+    grouped_types = {
+        "dashboards": "Dashboard",
+        "reports": "Report",
+        "models": "SemanticModel",
+        "dataflows": "Dataflow",
+        "datamarts": "Datamart",
+    }
+
+    # All other item types go into 'artifacts'
+    fabric_items = [
+        "Datamart",
+        "Lakehouse",
+        "Eventhouse",
+        "Environment",
+        "KQLDatabase",
+        "KQLQueryset",
+        "KQLDashboard",
+        "DataPipeline",
+        "Notebook",
+        "SparkJobDefinition",
+        "MLExperiment",
+        "MLModel",
+        "Warehouse",
+        "Eventstream",
+        "SQLEndpoint",
+        "MirroredWarehouse",
+        "MirroredDatabase",
+        "Reflex",
+        "GraphQLApi",
+        "MountedDataFactory",
+        "SQLDatabase",
+        "CopyJob",
+        "VariableLibrary",
+        "Dataflow",
+        "ApacheAirflowJob",
+        "WarehouseSnapshot",
+        "DigitalTwinBuilder",
+        "DigitalTwinBuilderFlow",
+        "MirroredAzureDatabricksCatalog",
+        "DataAgent",
+        "UserDataFunction",
+    ]
+
+    dfI = fabric.list_items(workspace=workspace)
+
+    payload = {
+        key: [{"artifactId": i} for i in dfI[dfI["Type"] == value]["Id"].tolist()]
+        for key, value in grouped_types.items()
+    }
+
+    # Add generic artifact types
+    artifact_ids = dfI[dfI["Type"].isin(fabric_items)]["Id"].tolist()
+    if artifact_ids:
+        payload["artifacts"] = [{"artifactId": i} for i in artifact_ids]
+
+    prefix = _get_url_prefix()
+
+    response = requests.post(
+        f"{prefix}/metadata/informationProtection/artifacts",
+        json=payload,
+        headers=headers,
+    )
+    if response.status_code != 200:
+        raise FabricHTTPException(f"Failed to retrieve labels: {response.text}")
+    result = response.json()
+
+    label_keys = [
+        "artifactInformationProtections",
+        "datasetInformationProtections",
+        "reportInformationProtections",
+        "dashboardInformationProtections",
+    ]
+
+    rows = [
+        {
+            "Id": item.get("artifactObjectId"),
+            "Label Id": item.get("labelId"),
+            "Label Name": item.get("name"),
+            "Parent Label Name": item.get("parent", {}).get("name"),
+            "Label Description": item.get("tooltip"),
+        }
+        for key in label_keys
+        for item in result.get(key, [])
+    ]
+
+    df_labels = pd.DataFrame(rows)
+    return dfI.merge(df_labels, on="Id", how="left")