semantic-link-labs 0.9.9__py3-none-any.whl → 0.9.11__py3-none-any.whl

sempy_labs/_warehouses.py

@@ -53,11 +53,11 @@ def create_warehouse(
             "defaultCollation"
         ] = "Latin1_General_100_CI_AS_KS_WS_SC_UTF8"
 
-    response = _base_api(
+    result = _base_api(
         request=f"/v1/workspaces/{workspace_id}/warehouses",
         payload=payload,
         method="post",
-        lro_return_status_code=True,
+        lro_return_json=True,
         status_codes=[201, 202],
     )
 
@@ -65,7 +65,7 @@ def create_warehouse(
         f"{icons.green_dot} The '{warehouse}' warehouse has been created within the '{workspace_name}' workspace."
     )
 
-    return response.get("id")
+    return result.get("id")
 
 
 def list_warehouses(workspace: Optional[str | UUID] = None) -> pd.DataFrame:

sempy_labs/admin/__init__.py

@@ -38,6 +38,7 @@ from sempy_labs.admin._capacities import (
     get_capacity_assignment_status,
     get_capacity_state,
     list_capacity_users,
+    get_refreshables,
 )
 from sempy_labs.admin._tenant import (
     list_tenant_settings,
@@ -80,6 +81,14 @@ from sempy_labs.admin._external_data_share import (
 from sempy_labs.admin._git import (
     list_git_connections,
 )
+from sempy_labs.admin._dataflows import (
+    export_dataflow,
+)
+from sempy_labs.admin._tags import (
+    list_tags,
+    create_tags,
+    delete_tag,
+)
 
 __all__ = [
     "list_items",
@@ -133,4 +142,9 @@ __all__ = [
     "list_capacity_users",
     "list_user_subscriptions",
     "list_report_subscriptions",
+    "get_refreshables",
+    "export_dataflow",
+    "list_tags",
+    "create_tags",
+    "delete_tag",
 ]

sempy_labs/admin/_artifacts.py

@@ -31,7 +31,7 @@ def list_unused_artifacts(workspace: Optional[str | UUID] = None) -> pd.DataFram
         "Artifact Name": "string",
         "Artifact Id": "string",
         "Artifact Type": "string",
-        "Artifact Size in MB": "int",
+        "Artifact Size in MB": "string",
         "Created Date Time": "datetime",
         "Last Accessed Date Time": "datetime",
     }
@@ -47,8 +47,8 @@ def list_unused_artifacts(workspace: Optional[str | UUID] = None) -> pd.DataFram
     for r in responses:
         for i in r.get("unusedArtifactEntities", []):
             new_data = {
-                "Artifact Name": i.get("artifactId"),
-                "Artifact Id": i.get("displayName"),
+                "Artifact Name": i.get("displayName"),
+                "Artifact Id": i.get("artifactId"),
                 "Artifact Type": i.get("artifactType"),
                 "Artifact Size in MB": i.get("artifactSizeInMB"),
                 "Created Date Time": i.get("createdDateTime"),

sempy_labs/admin/_capacities.py

@@ -5,6 +5,7 @@ from typing import Optional, Tuple
 from sempy._utils._log import log
 from sempy_labs._helper_functions import (
     _base_api,
+    _build_url,
     _create_dataframe,
     _update_dataframe_datatypes,
     _is_valid_uuid,
@@ -57,6 +58,24 @@ def _resolve_capacity_name_and_id(
     return capacity_name, capacity_id
 
 
+def _resolve_capacity_id(
+    capacity: str | UUID,
+) -> UUID:
+
+    if _is_valid_uuid(capacity):
+        capacity_id = capacity
+    else:
+        dfC = list_capacities(capacity=capacity)
+        if dfC.empty:
+            raise ValueError(
+                f"{icons.red_dot} The '{capacity}' capacity was not found."
+            )
+
+        capacity_id = dfC["Capacity Id"].iloc[0]
+
+    return capacity_id
+
+
 def _list_capacities_meta() -> pd.DataFrame:
     """
     Shows the a list of capacities and their properties. This function is the admin version.
@@ -221,7 +240,7 @@ def list_capacities(
         "Sku": "string",
         "Region": "string",
         "State": "string",
-        "Admins": "string",
+        "Admins": "list",
     }
     df = _create_dataframe(columns=columns)
 
@@ -309,3 +328,144 @@ def list_capacity_users(capacity: str | UUID) -> pd.DataFrame:
     _update_dataframe_datatypes(dataframe=df, column_map=columns)
 
     return df
+
+
+@log
+def get_refreshables(
+    top: Optional[int] = None,
+    expand: Optional[str] = None,
+    filter: Optional[str] = None,
+    skip: Optional[int] = None,
+    capacity: Optional[str | UUID] = None,
+) -> pd.DataFrame | dict:
+    """
+    Returns a list of refreshables for the organization within a capacity.
+
+    Power BI retains a seven-day refresh history for each dataset, up to a maximum of sixty refreshes.
+
+    This is a wrapper function for the following API: `Admin - Get Refreshables <https://learn.microsoft.com/rest/api/power-bi/admin/get-refreshables>`_.
+
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
+    Parameters
+    ----------
+    top : int, default=None
+        Returns only the first n results.
+    expand : str, default=None
+        Accepts a comma-separated list of data types, which will be expanded inline in the response. Supports capacities and groups.
+    filter : str, default=None
+        Returns a subset of a results based on Odata filter query parameter condition.
+    skip : int, default=None
+        Skips the first n results. Use with top to fetch results beyond the first 1000.
+    capacity : str | uuid.UUID, default=None
+        The capacity name or ID to filter. If None, all capacities are returned.
+
+    Returns
+    -------
+    pandas.DataFrame
+        Returns a list of refreshables for the organization within a capacity.
+    """
+
+    columns = {
+        "Workspace Id": "string",
+        "Workspace Name": "string",
+        "Item Id": "string",
+        "Item Name": "string",
+        "Item Kind": "string",
+        "Capacity Id": "string",
+        "Capacity Name": "string",
+        "Capacity SKU": "string",
+        "Refresh Count": "int",
+        "Refresh Failures": "int",
+        "Average Duration": "float",
+        "Median Duration": "float",
+        "Refreshes Per Day": "int",
+        "Refresh Type": "string",
+        "Start Time": "string",
+        "End Time": "string",
+        "Status": "string",
+        "Request Id": "string",
+        "Service Exception Json": "string",
+        "Extended Status": "dict",
+        "Refresh Attempts": "list",
+        "Refresh Schedule Days": "list",
+        "Refresh Schedule Times": "list",
+        "Refresh Schedule Enabled": "bool",
+        "Refresh Schedule Local Timezone Id": "string",
+        "Refresh Schedule Notify Option": "string",
+        "Configured By": "list",
+    }
+
+    df = _create_dataframe(columns=columns)
+
+    params = {}
+    url = (
+        "/v1.0/myorg/admin/capacities/refreshables"
+        if capacity is None
+        else f"/v1.0/myorg/admin/capacities/{_resolve_capacity_id(capacity=capacity)}/refreshables"
+    )
+
+    if top is not None:
+        params["$top"] = top
+
+    if expand is not None:
+        params["$expand"] = expand
+
+    if filter is not None:
+        params["$filter"] = filter
+
+    if skip is not None:
+        params["$skip"] = skip
+
+    url = _build_url(url, params)
+
+    responses = _base_api(request=url, client="fabric_sp")
+
+    refreshables = []
+
+    for i in responses.json().get("value", []):
+        last_refresh = i.get("lastRefresh", {})
+        refresh_schedule = i.get("refreshSchedule", {})
+        new_data = {
+            "Workspace Id": i.get("group", {}).get("id"),
+            "Workspace Name": i.get("group", {}).get("name"),
+            "Item Id": i.get("id"),
+            "Item Name": i.get("name"),
+            "Item Kind": i.get("kind"),
+            "Capacity Id": (
+                i.get("capacity", {}).get("id").lower()
+                if i.get("capacity", {}).get("id")
+                else None
+            ),
+            "Capacity Name": i.get("capacity", {}).get("displayName"),
+            "Capacity SKU": i.get("capacity", {}).get("sku"),
+            "Refresh Count": i.get("refreshCount", 0),
+            "Refresh Failures": i.get("refreshFailures", 0),
+            "Average Duration": i.get("averageDuration", 0),
+            "Median Duration": i.get("medianDuration", 0),
+            "Refreshes Per Day": i.get("refreshesPerDay", 0),
+            "Refresh Type": last_refresh.get("refreshType"),
+            "Start Time": last_refresh.get("startTime"),
+            "End Time": last_refresh.get("endTime"),
+            "Status": last_refresh.get("status"),
+            "Request Id": last_refresh.get("requestId"),
+            "Service Exception Json": last_refresh.get("serviceExceptionJson"),
+            "Extended Status": last_refresh.get("extendedStatus"),
+            "Refresh Attempts": last_refresh.get("refreshAttempts"),
+            "Refresh Schedule Days": refresh_schedule.get("days"),
+            "Refresh Schedule Times": refresh_schedule.get("times"),
+            "Refresh Schedule Enabled": refresh_schedule.get("enabled"),
+            "Refresh Schedule Local Timezone Id": refresh_schedule.get(
+                "localTimeZoneId"
+            ),
+            "Refresh Schedule Notify Option": refresh_schedule.get("notifyOption"),
+            "Configured By": i.get("configuredBy"),
+        }
+
+        refreshables.append(new_data)
+
+    if len(refreshables) > 0:
+        df = pd.DataFrame(refreshables)
+        _update_dataframe_datatypes(dataframe=df, column_map=columns)
+
+    return df

sempy_labs/admin/_dataflows.py

@@ -0,0 +1,45 @@
+from typing import Optional
+from sempy_labs._helper_functions import (
+    _base_api,
+)
+from sempy_labs.admin._items import (
+    _resolve_item_id,
+)
+from uuid import UUID
+from sempy._utils._log import log
+
+
+@log
+def export_dataflow(
+    dataflow: str | UUID,
+    workspace: Optional[str | UUID] = None,
+) -> dict:
+    """
+    Shows a list of datasets for the organization.
+
+    This is a wrapper function for the following API: `Admin - Dataflows ExportDataflowAsAdmin <https://learn.microsoft.com/rest/api/power-bi/admin/dataflows-export-dataflow-as-admin>`_.
+
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
+    Parameters
+    ----------
+    dataflow : str | UUID, default=None
+        The dataflow Name or Id.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or id.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+        Only used if given a dataflow name and not an id.
+
+    Returns
+    -------
+    dict
+        Exported Json file.
+    """
+    dataflow_id = _resolve_item_id(item=dataflow, type="dataflow", workspace=workspace)
+
+    url = f"/v1.0/myorg/admin/dataflows/{dataflow_id}/export"
+
+    response = _base_api(request=url, client="fabric_sp")
+
+    return response.json()

sempy_labs/admin/_items.py

@@ -17,20 +17,26 @@ from sempy_labs._helper_functions import (
 
 
 def _resolve_item_id(
-    item_name: str,
+    item: str,
     type: Optional[str] = None,
     workspace: Optional[str | UUID] = None,
 ) -> UUID:
+    if _is_valid_uuid(item):
+        item_id = item
 
-    dfI = list_items(workspace=workspace, type=type)
-    dfI_filt = dfI[dfI["Item Name"] == item_name]
+    else:
+        workspace_id = _resolve_workspace_name_and_id(workspace)[1]
+        dfI = list_items(workspace=workspace_id, type=type)
+        dfI_filt = dfI[dfI["Item Name"] == item]
 
-    if len(dfI_filt) == 0:
-        raise ValueError(
-            f"The '{item_name}' {type} does not exist within the '{workspace}' workspace or is not of type '{type}'."
-        )
+        if len(dfI_filt) == 0:
+            raise ValueError(
+                f"The '{item}' {type} does not exist within the '{workspace}' workspace or is not of type '{type}'."
+            )
+
+        item_id = dfI_filt["Item Id"].iloc[0]
 
-    return dfI_filt["Item Id"].iloc[0]
+    return item_id
 
 
 def _resolve_item_name_and_id(
@@ -84,9 +90,8 @@ def list_items(
     capacity : str | uuid.UUID, default=None
         The capacity name or id.
     workspace : str | uuid.UUID, default=None
-        The Fabric workspace name.
-        Defaults to None which resolves to the workspace of the attached lakehouse
-        or if no lakehouse attached, resolves to the workspace of the notebook.
+        The Fabric workspace name or id.
+        Defaults to None which looks into all the workspaces.
     state : str, default=None
         The item state.
     type : str, default=None

sempy_labs/admin/_tags.py

@@ -0,0 +1,126 @@
+from sempy_labs._helper_functions import (
+    _base_api,
+    _is_valid_uuid,
+)
+from uuid import UUID
+from sempy_labs._tags import list_tags
+import sempy_labs._icons as icons
+from typing import List
+
+
+def resolve_tag_id(tag: str | UUID):
+
+    if _is_valid_uuid(tag):
+        tag_id = tag
+    else:
+        df = list_tags()
+        df[df["Tag Name"] == tag]
+        if df.empty:
+            raise ValueError(f"{icons.red_dot} The '{tag}' tag does not exist.")
+        tag_id = df.iloc[0]["Tag Id"]
+
+    return tag_id
+
+
+def create_tags(tags: str | List[str]):
+    """
+    Creates a new tag or tags.
+
+    This is a wrapper function for the following API: `Tags - Bulk Create Tags <https://learn.microsoft.com/rest/api/fabric/admin/tags/bulk-create-tags>`_.
+
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
+    Parameters
+    ----------
+    tags : str | List[str]
+        The name of the tag or tags to create.
+    """
+
+    if isinstance(tags, str):
+        tags = [tags]
+
+    # Check the length of the tags
+    for tag in tags:
+        if len(tag) > 40:
+            raise ValueError(
+                f"{icons.red_dot} The '{tag}' tag name is too long. It must be 40 characters or less."
+            )
+
+    # Check if the tags already exist
+    df = list_tags()
+    existing_names = df["Tag Name"].tolist()
+    existing_ids = df["Tag Id"].tolist()
+
+    available_tags = [
+        tag for tag in tags if tag not in existing_names and tag not in existing_ids
+    ]
+    unavailable_tags = [
+        tag for tag in tags if tag in existing_names or tag in existing_ids
+    ]
+
+    print(f"{icons.warning} The following tags already exist: {unavailable_tags}")
+    if not available_tags:
+        print(f"{icons.info} No new tags to create.")
+        return
+
+    payload = [{"displayName": name} for name in available_tags]
+
+    for tag in tags:
+        _base_api(
+            request="/v1/admin/bulkCreateTags",
+            client="fabric_sp",
+            method="post",
+            payload=payload,
+            status_codes=201,
+        )
+
+    print(f"{icons.green_dot} The '{available_tags}' tag(s) have been created.")
+
+
+def delete_tag(tag: str | UUID):
+    """
+    Deletes a tag.
+
+    This is a wrapper function for the following API: `Tags - Delete Tag <https://learn.microsoft.com/rest/api/fabric/admin/tags/delete-tag>`_.
+
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
+    Parameters
+    ----------
+    tag : str | uuid.UUID
+        The name or ID of the tag to delete.
+    """
+
+    tag_id = resolve_tag_id(tag)
+
+    _base_api(request=f"/v1/admin/tags/{tag_id}", client="fabric_sp", method="delete")
+
+    print(f"{icons.green_dot} The '{tag}' tag has been deleted.")
+
+
+def update_tag(name: str, tag: str | UUID):
+    """
+    Updates the name of a tag.
+
+    This is a wrapper function for the following API: `Tags - Update Tag <https://learn.microsoft.com/rest/api/fabric/admin/tags/update-tag>`_.
+
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
+    Parameters
+    ----------
+    name : str
+        The new name of the tag.
+    tag : str | uuid.UUID
+        The name or ID of the tag to update.
+    """
+
+    tag_id = resolve_tag_id(tag)
+
+    _base_api(
+        request=f"/v1/admin/tags/{tag_id}",
+        client="fabric_sp",
+        method="patch",
+        payload={"displayName": name},
+    )
+
+    print(f"{icons.green_dot} The '{tag}' tag has been renamed to '{name}'.")

sempy_labs/admin/_tenant.py

@@ -32,7 +32,7 @@ def list_tenant_settings() -> pd.DataFrame:
         "Enabled": "bool",
         "Can Specify Security Groups": "bool",
         "Tenant Setting Group": "string",
-        "Enabled Security Groups": "string",
+        "Enabled Security Groups": "list",
     }
     df = _create_dataframe(columns=columns)
 
@@ -86,9 +86,9 @@ def list_capacity_tenant_settings_overrides(
         "Setting Title": "string",
         "Setting Enabled": "bool",
         "Can Specify Security Groups": "bool",
-        "Enabled Security Groups": "string",
+        "Enabled Security Groups": "list",
         "Tenant Setting Group": "string",
-        "Tenant Setting Properties": "string",
+        "Tenant Setting Properties": "list",
         "Delegate to Workspace": "bool",
         "Delegated From": "string",
     }
@@ -395,7 +395,7 @@ def list_workspaces_tenant_settings_overrides() -> pd.DataFrame:
         "Title": "string",
         "Enabled": "bool",
         "Can Specify Security Groups": "bool",
-        "Enabled Security Groups": "string",
+        "Enabled Security Groups": "list",
         "Tenant Setting Group": "string",
         "Delegated From": "string",
     }
@@ -454,7 +454,7 @@ def list_domain_tenant_settings_overrides() -> pd.DataFrame:
         "Title": "string",
         "Enabled": "bool",
         "Can Specify Security Groups": "bool",
-        "Enabled Security Groups": "string",
+        "Enabled Security Groups": "list",
         "Tenant Setting Group": "string",
         "Delegated To Workspace": "bool",
         "Delegated From": "string",

sempy_labs/directlake/_generate_shared_expression.py

@@ -3,6 +3,7 @@ from sempy_labs._helper_functions import (
     _base_api,
     resolve_lakehouse_name_and_id,
     resolve_item_name_and_id,
+    _get_fabric_context_setting,
 )
 from typing import Optional
 import sempy_labs._icons as icons
@@ -56,34 +57,36 @@ def generate_shared_expression(
             item=item_name, type=item_type, workspace=workspace_id
         )
 
-    item_type_rest = f"{item_type.lower()}s"
-    response = _base_api(
-        request=f"/v1/workspaces/{workspace_id}/{item_type_rest}/{item_id}"
-    )
-
-    prop = response.json().get("properties")
+    if use_sql_endpoint:
+        item_type_rest = f"{item_type.lower()}s"
+        response = _base_api(
+            request=f"/v1/workspaces/{workspace_id}/{item_type_rest}/{item_id}"
+        )
 
-    if item_type == "Lakehouse":
-        sqlprop = prop.get("sqlEndpointProperties")
-        sqlEPCS = sqlprop.get("connectionString")
-        sqlepid = sqlprop.get("id")
-        provStatus = sqlprop.get("provisioningStatus")
-    elif item_type == "Warehouse":
-        sqlEPCS = prop.get("connectionString")
-        sqlepid = item_id
-        provStatus = None
+        prop = response.json().get("properties")
 
-    if provStatus == "InProgress":
-        raise ValueError(
-            f"{icons.red_dot} The SQL Endpoint for the '{item_name}' lakehouse within the '{workspace_name}' workspace has not yet been provisioned. Please wait until it has been provisioned."
-        )
+        if item_type == "Lakehouse":
+            sqlprop = prop.get("sqlEndpointProperties")
+            sqlEPCS = sqlprop.get("connectionString")
+            sqlepid = sqlprop.get("id")
+            provStatus = sqlprop.get("provisioningStatus")
+        elif item_type == "Warehouse":
+            sqlEPCS = prop.get("connectionString")
+            sqlepid = item_id
+            provStatus = None
 
-    start_expr = "let\n\tdatabase = "
-    end_expr = "\nin\n\tdatabase"
-    mid_expr = f'Sql.Database("{sqlEPCS}", "{sqlepid}")'
+        if provStatus == "InProgress":
+            raise ValueError(
+                f"{icons.red_dot} The SQL Endpoint for the '{item_name}' {item_type.lower()} within the '{workspace_name}' workspace has not yet been provisioned. Please wait until it has been provisioned."
+            )
 
-    # Build DL/OL expression
-    if not use_sql_endpoint and item_type == "Lakehouse":
-        return f'AzureDataLakeStorage{{"server":"onelake.dfs.fabric.microsoft.com","path":"/{workspace_id}/{item_id}/"}}'
-    else:
+        start_expr = "let\n\tdatabase = "
+        end_expr = "\nin\n\tdatabase"
+        mid_expr = f'Sql.Database("{sqlEPCS}", "{sqlepid}")'
         return f"{start_expr}{mid_expr}{end_expr}"
+    else:
+        # Build DL/OL expression
+        env = _get_fabric_context_setting("spark.trident.pbienv").lower()
+        env = "" if env == "prod" else f"{env}-"
+
+        return f"""let\n\tSource = AzureStorage.DataLake("https://{env}onelake.dfs.fabric.microsoft.com/{workspace_id}/{item_id}")\nin\n\tSource"""