semantic-link-labs 0.10.0__py3-none-any.whl → 0.11.0__py3-none-any.whl

sempy_labs/_dataflows.py

@@ -6,10 +6,16 @@ from sempy_labs._helper_functions import (
     _base_api,
     _create_dataframe,
     resolve_workspace_name,
+    resolve_workspace_id,
+    _decode_b64,
+    _conv_b64,
+    get_jsonpath_value,
 )
 from typing import Optional, Tuple
 import sempy_labs._icons as icons
 from uuid import UUID
+from jsonpath_ng.ext import parse
+import json
 
 
 def list_dataflows(workspace: Optional[str | UUID] = None):
@@ -29,34 +35,55 @@ def list_dataflows(workspace: Optional[str | UUID] = None):
         A pandas dataframe showing the dataflows which exist within a workspace.
     """
 
-    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    workspace_id = resolve_workspace_id(workspace)
 
     columns = {
         "Dataflow Id": "string",
         "Dataflow Name": "string",
+        "Description": "string",
         "Configured By": "string",
         "Users": "string",
-        "Generation": "int",
+        "Generation": "string",
     }
     df = _create_dataframe(columns=columns)
 
-    response = _base_api(request=f"/v1.0/myorg/groups/{workspace_id}/dataflows")
-
-    data = []  # Collect rows here
+    response = _base_api(
+        request=f"/v1.0/myorg/groups/{workspace_id}/dataflows", client="fabric_sp"
+    )
 
+    dfs = []
     for v in response.json().get("value", []):
+        gen = v.get("generation")
         new_data = {
             "Dataflow Id": v.get("objectId"),
             "Dataflow Name": v.get("name"),
+            "Description": "",
             "Configured By": v.get("configuredBy"),
-            "Users": v.get("users", []),
-            "Generation": v.get("generation"),
+            "Users": ", ".join(v.get("users", [])),
+            "Generation": "Gen2" if gen == 2 else "Gen1",
         }
-        data.append(new_data)
+        dfs.append(pd.DataFrame(new_data, index=[0]))
 
-    if data:
-        df = pd.DataFrame(data)
+    responses = _base_api(
+        request=f"/v1/workspaces/{workspace_id}/dataflows",
+        client="fabric_sp",
+        uses_pagination=True,
+    )
+    for r in responses:
+        for v in r.get("value", []):
+            gen = v.get("generation")
+            new_data = {
+                "Dataflow Id": v.get("id"),
+                "Dataflow Name": v.get("displayName"),
+                "Description": v.get("description"),
+                "Configured By": "",
+                "Users": "",
+                "Generation": "Gen2 CI/CD",
+            }
+            dfs.append(pd.DataFrame(new_data, index=[0]))
 
+    if dfs:
+        df = pd.concat(dfs, ignore_index=True)
         _update_dataframe_datatypes(dataframe=df, column_map=columns)
 
     return df
@@ -162,8 +189,10 @@ def list_upstream_dataflows(
     """
 
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
-    (dataflow_name, dataflow_id) = _resolve_dataflow_name_and_id(
-        dataflow=dataflow, workspace=workspace_id
+    (dataflow_name, dataflow_id, dataflow_generation) = (
+        _resolve_dataflow_name_and_id_and_generation(
+            dataflow=dataflow, workspace=workspace_id
+        )
     )
 
     columns = {
@@ -188,7 +217,7 @@ def list_upstream_dataflows(
             tgt_dataflow_id = v.get("targetDataflowId")
             tgt_workspace_id = v.get("groupId")
             tgt_workspace_name = resolve_workspace_name(workspace_id=tgt_workspace_id)
-            (tgt_dataflow_name, _) = _resolve_dataflow_name_and_id(
+            (tgt_dataflow_name, _, _) = _resolve_dataflow_name_and_id_and_generation(
                 dataflow=tgt_dataflow_id, workspace=tgt_workspace_id
             )
 
@@ -215,9 +244,9 @@ def list_upstream_dataflows(
     return df
 
 
-def _resolve_dataflow_name_and_id(
+def _resolve_dataflow_name_and_id_and_generation(
     dataflow: str | UUID, workspace: Optional[str | UUID] = None
-) -> Tuple[str, UUID]:
+) -> Tuple[str, UUID, str]:
 
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
 
@@ -228,12 +257,250 @@ def _resolve_dataflow_name_and_id(
     else:
         dfD_filt = dfD[dfD["Dataflow Name"] == dataflow]
 
-    if len(dfD_filt) == 0:
+    if dfD_filt.empty:
         raise ValueError(
             f"{icons.red_dot} The '{dataflow}' dataflow does not exist within the '{workspace_name}' workspace."
         )
 
     dataflow_id = dfD_filt["Dataflow Id"].iloc[0]
     dataflow_name = dfD_filt["Dataflow Name"].iloc[0]
+    dataflow_generation = dfD_filt["Generation"].iloc[0]
+
+    return (dataflow_name, dataflow_id, dataflow_generation)
+
+
+def get_dataflow_definition(
+    dataflow: str | UUID,
+    workspace: Optional[str | UUID] = None,
+    decode: bool = True,
+) -> dict:
+    """
+    Obtains the definition of a dataflow. This supports Gen1, Gen2 and Gen 2 CI/CD dataflows.
+
+    This is a wrapper function for the following API: `Dataflows - Get Dataflow <https://learn.microsoft.com/rest/api/power-bi/dataflows/get-dataflow>`_.
+
+    Parameters
+    ----------
+    dataflow : str | uuid.UUID
+        The name or ID of the dataflow.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name.
+        Defaults to None, which resolves to the workspace of the attached lakehouse
+        or if no lakehouse is attached, resolves to the workspace of the notebook.
+    decode : bool, optional
+        If True, decodes the dataflow definition file.
+
+    Returns
+    -------
+    dict
+        The dataflow definition.
+    """
+
+    workspace_id = resolve_workspace_id(workspace)
+
+    (dataflow_name, dataflow_id, dataflow_generation) = (
+        _resolve_dataflow_name_and_id_and_generation(
+            dataflow=dataflow, workspace=workspace_id
+        )
+    )
+
+    if dataflow_generation == "Gen2 CI/CD":
+        result = _base_api(
+            request=f"/v1/workspaces/{workspace_id}/items/{dataflow_id}/getDefinition",
+            client="fabric_sp",
+            method="post",
+            lro_return_json=True,
+            status_codes=[200, 202],
+        )
+
+        if decode:
+            # Decode the payload from base64
+            definition = {"definition": {"parts": []}}
+
+            for part in result.get("definition", {}).get("parts", []):
+                path = part.get("path")
+                payload = part.get("payload")
+                decoded_payload = _decode_b64(payload)
+                definition["definition"]["parts"].append(
+                    {"path": path, "payload": decoded_payload}
+                )
+            return definition
+        else:
+            return result
+    else:
+        result = _base_api(
+            request=f"/v1.0/myorg/groups/{workspace_id}/dataflows/{dataflow_id}",
+            client="fabric_sp",
+            method="get",
+        ).json()
+
+        return result
+
+
+def upgrade_dataflow(
+    dataflow: str | UUID,
+    workspace: Optional[str | UUID] = None,
+    new_dataflow_name: Optional[str] = None,
+    new_dataflow_workspace: Optional[str | UUID] = None,
+):
+    """
+    Creates a Dataflow Gen2 CI/CD item based on the mashup definition from an existing Gen1/Gen2 dataflow. After running this function, update the connections in the dataflow to ensure the data can be properly refreshed.
 
-    return dataflow_name, dataflow_id
+    Parameters
+    ----------
+    dataflow : str | uuid.UUID
+        The name or ID of the dataflow.
+    workspace : str | uuid.UUID, default=None
+        The workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    new_dataflow_name: str, default=None
+        Name of the new dataflow.
+    new_dataflow_workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID of the dataflow to be created.
+        Defaults to None which resolves to the existing workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    # Resolve the workspace name and ID
+    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
+
+    # Resolve the dataflow name and ID
+    (dataflow_name, dataflow_id, dataflow_generation) = (
+        _resolve_dataflow_name_and_id_and_generation(dataflow, workspace_id)
+    )
+
+    if dataflow_generation == "Gen2 CI/CD":
+        # Print an error message that the dataflow is already a native Fabric item
+        print(
+            f"{icons.info} The dataflow '{dataflow_name}' is already a Fabric native Dataflow Gen2 item. No changes made."
+        )
+        return
+
+    (new_dataflow_workspace, new_dataflow_workspace_id) = resolve_workspace_name_and_id(
+        new_dataflow_workspace
+    )
+
+    # If no new dataflow name is provided, use the existing dataflow name
+    if not new_dataflow_name:
+        new_dataflow_name = dataflow_name
+
+    # Get dataflow definition
+    definition = get_dataflow_definition(dataflow, workspace_id)
+
+    # Check for linked table references
+    matches = (
+        parse("$['pbi:mashup'].connectionOverrides[*].kind").find(definition) or []
+    )
+    if any(match.value in {"PowerPlatformDataflows", "PowerBI"} for match in matches):
+        print(
+            f"""{icons.red_dot} The dataflow '{dataflow_name}' contains a linked table reference to an existing dataflow as a connection source and will not be upgraded. No changes were made.
+    - To track the upstream lineage of linked tables across dataflows use the list_upstream_dataflows function.
+    - To automatically remove the tables and upgrade the existing dataflow use the upgrade_powerbippdf_dataflow function."""
+        )
+        return
+
+    description = get_jsonpath_value(data=definition, path="$.description")
+
+    payload = {
+        "displayName": new_dataflow_name,
+    }
+    if description:
+        payload["description"] = description
+
+    # Query Groups
+    matches = parse("$.annotations[?(@.name=='pbi:QueryGroups')].value").find(
+        definition
+    )
+    query_groups_value = json.loads(matches[0].value) if matches else []
+
+    # Prepare the dataflow definition
+    query_metadata = {
+        "formatVersion": "202502",
+        "computeEngineSettings": {},  # How to set this?
+        "name": new_dataflow_name,
+        "queryGroups": query_groups_value,
+        "documentLocale": get_jsonpath_value(data=definition, path="$.culture"),
+        "queriesMetadata": get_jsonpath_value(
+            data=definition, path="$['pbi:mashup'].queriesMetadata"
+        ),
+        "fastCombine": get_jsonpath_value(
+            data=definition, path="$['pbi:mashup'].fastCombine", default=False
+        ),
+        "allowNativeQueries": get_jsonpath_value(
+            data=definition, path="$['pbi:mashup'].allowNativeQueries", default=False
+        ),
+        # "connections": [],
+    }
+
+    mashup_doc = get_jsonpath_value(data=definition, path="$['pbi:mashup'].document")
+
+    # Add the dataflow definition to the payload
+    new_definition = {
+        "parts": [
+            {
+                "path": "queryMetadata.json",
+                "payload": _conv_b64(query_metadata),
+                "payloadType": "InlineBase64",
+            },
+            {
+                "path": "mashup.pq",
+                "payload": _conv_b64(mashup_doc, json_dumps=False),
+                "payloadType": "InlineBase64",
+            },
+        ]
+    }
+
+    create_dataflow(
+        name=new_dataflow_name,
+        workspace=new_dataflow_workspace,
+        definition=new_definition,
+    )
+
+
+def create_dataflow(
+    name: str,
+    workspace: Optional[str | UUID] = None,
+    description: Optional[str] = None,
+    definition: Optional[dict] = None,
+):
+    """
+    Creates a native Fabric Dataflow Gen2 CI/CD item.
+
+    Parameters
+    ----------
+    name : str
+        The name the dataflow.
+    workspace : str | uuid.UUID, default=None
+        The workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    description : str, default=None
+        The description of the dataflow.
+    definition : dict, default=None
+        The definition of the dataflow in the form of a dictionary.
+    """
+
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+
+    payload = {
+        "displayName": name,
+    }
+    if description:
+        payload["description"] = description
+
+    if definition:
+        payload["definition"] = definition
+
+    _base_api(
+        request=f"/v1/workspaces/{workspace_id}/dataflows",
+        method="post",
+        payload=payload,
+        client="fabric_sp",
+        lro_return_json=True,
+        status_codes=[201, 202],
+    )
+
+    print(
+        f"{icons.green_dot} The dataflow '{name}' has been created within the '{workspace_name}' workspace."
+    )

sempy_labs/_daxformatter.py

@@ -0,0 +1,80 @@
+import requests
+from typing import List, Optional
+from sempy_labs._a_lib_info import lib_name, lib_version
+from sempy._utils._log import log
+
+
+@log
+def _format_dax(
+    expressions: str | List[str],
+    skip_space_after_function_name: bool = False,
+    metadata: Optional[List[dict]] = None,
+) -> List[str]:
+
+    if isinstance(expressions, str):
+        expressions = [expressions]
+        metadata = [metadata] if metadata else [{}]
+
+    # Add variable assignment to each expression
+    expressions = [f"x :={item}" for item in expressions]
+
+    url = "https://daxformatter.azurewebsites.net/api/daxformatter/daxtextformatmulti"
+
+    payload = {
+        "Dax": expressions,
+        "MaxLineLength": 0,
+        "SkipSpaceAfterFunctionName": skip_space_after_function_name,
+        "ListSeparator": ",",
+        "DecimalSeparator": ".",
+    }
+
+    headers = {
+        "Accept": "application/json, text/javascript, */*; q=0.01",
+        "Accept-Encoding": "gzip,deflate",
+        "Accept-Language": "en-US,en;q=0.8",
+        "Content-Type": "application/json; charset=UTF-8",
+        "Host": "daxformatter.azurewebsites.net",
+        "Expect": "100-continue",
+        "Connection": "Keep-Alive",
+        "CallerApp": lib_name,
+        "CallerVersion": lib_version,
+    }
+
+    response = requests.post(url, json=payload, headers=headers)
+    result = []
+    for idx, dax in enumerate(response.json()):
+        formatted_dax = dax.get("formatted")
+        errors = dax.get("errors")
+        if errors:
+            meta = metadata[idx] if metadata and idx < len(metadata) else {}
+            obj_name = meta.get("name", "Unknown")
+            table_name = meta.get("table", "Unknown")
+            obj_type = meta.get("type", "Unknown")
+            if obj_type == "calculated_tables":
+                raise ValueError(
+                    f"DAX formatting failed for the '{obj_name}' calculated table: {errors}"
+                )
+            elif obj_type == "calculated_columns":
+                raise ValueError(
+                    f"DAX formatting failed for the '{table_name}'[{obj_name}] calculated column: {errors}"
+                )
+            elif obj_type == "calculation_items":
+                raise ValueError(
+                    f"DAX formatting failed for the '{table_name}'[{obj_name}] calculation item: {errors}"
+                )
+            elif obj_type == "measures":
+                raise ValueError(
+                    f"DAX formatting failed for the '{obj_name}' measure: {errors}"
+                )
+            elif obj_type == "rls":
+                raise ValueError(
+                    f"DAX formatting failed for the row level security expression on the '{table_name}' table within the '{obj_name}' role: {errors}"
+                )
+            else:
+                NotImplementedError()
+        else:
+            if formatted_dax.startswith("x :="):
+                formatted_dax = formatted_dax[4:]
+            formatted_dax = formatted_dax.strip()
+            result.append(formatted_dax)
+    return result

sempy_labs/_delta_analyzer_history.py

@@ -55,7 +55,10 @@ def delta_analyzer_history(
 
     table_path = create_abfss_path(lakehouse_id, workspace_id, table_name, schema)
     local_path = _mount(lakehouse=lakehouse, workspace=workspace)
-    table_path_local = f"{local_path}/Tables/{table_name}"
+    if schema:  # use schema if specified
+        table_path_local = f"{local_path}/Tables/{schema}/{table_name}"
+    else:
+        table_path_local = f"{local_path}/Tables/{table_name}"
     delta_table_path = f"{table_path}/_delta_log"
 
     files = notebookutils.fs.ls(delta_table_path)

sempy_labs/_deployment_pipelines.py

@@ -7,8 +7,10 @@ from sempy_labs._helper_functions import (
 )
 import sempy_labs._icons as icons
 from uuid import UUID
+from sempy._utils._log import log
 
 
+@log
 def list_deployment_pipelines() -> pd.DataFrame:
     """
     Shows a list of deployment pipelines the user can access.
@@ -46,6 +48,7 @@ def list_deployment_pipelines() -> pd.DataFrame:
     return df
 
 
+@log
 def list_deployment_pipeline_stages(deployment_pipeline: str | UUID) -> pd.DataFrame:
     """
     Shows the specified deployment pipeline stages.
@@ -104,6 +107,7 @@ def list_deployment_pipeline_stages(deployment_pipeline: str | UUID) -> pd.DataF
     return df
 
 
+@log
 def list_deployment_pipeline_stage_items(
     deployment_pipeline: str | UUID,
     stage: str | UUID,

sempy_labs/_documentation.py

@@ -2,8 +2,10 @@ import sempy
 import sempy.fabric as fabric
 import pandas as pd
 from typing import List, Optional
+from sempy._utils._log import log
 
 
+@log
 def list_all_items(workspaces: Optional[str | List[str]] = None):
 
     df = pd.DataFrame(
@@ -41,6 +43,7 @@ def list_all_items(workspaces: Optional[str | List[str]] = None):
     return df
 
 
+@log
 def data_dictionary(dataset: str, workspace: Optional[str | None] = None):
 
     from sempy_labs.tom import connect_semantic_model

sempy_labs/_environments.py

@@ -11,8 +11,10 @@ from sempy_labs._helper_functions import (
     create_item,
 )
 from uuid import UUID
+from sempy._utils._log import log
 
 
+@log
 def create_environment(
     environment: str,
     description: Optional[str] = None,
@@ -43,6 +45,7 @@ def create_environment(
     )
 
 
+@log
 def list_environments(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
     """
     Shows the environments within a workspace.
@@ -85,6 +88,7 @@ def list_environments(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
         client="fabric_sp",
     )
 
+    dfs = []
     for r in responses:
         for v in r.get("value", []):
             pub = v.get("properties", {}).get("publishDetails", {})
@@ -103,11 +107,15 @@ def list_environments(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
                 .get("sparkSettings", {})
                 .get("state"),
             }
-            df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
+            dfs.append(pd.DataFrame(new_data, index=[0]))
+
+    if dfs:
+        df = pd.concat(dfs, ignore_index=True)
 
     return df
 
 
+@log
 def delete_environment(environment: str | UUID, workspace: Optional[str | UUID] = None):
     """
     Deletes a Fabric environment.
@@ -127,6 +135,7 @@ def delete_environment(environment: str | UUID, workspace: Optional[str | UUID]
     delete_item(item=environment, type="Environment", workspace=workspace)
 
 
+@log
 def publish_environment(
     environment: str | UUID, workspace: Optional[str | UUID] = None
 ):

sempy_labs/_eventhouses.py

@@ -1,20 +1,20 @@
 import pandas as pd
 from typing import Optional
 from sempy_labs._helper_functions import (
-    resolve_workspace_name_and_id,
     _base_api,
-    resolve_item_id,
     _create_dataframe,
     _conv_b64,
-    _decode_b64,
     delete_item,
     create_item,
     get_item_definition,
+    resolve_workspace_id,
 )
 from uuid import UUID
 import sempy_labs._icons as icons
+from sempy._utils._log import log
 
 
+@log
 def create_eventhouse(
     name: str,
     definition: Optional[dict],
@@ -66,6 +66,7 @@ def create_eventhouse(
     )
 
 
+@log
 def list_eventhouses(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
     """
     Shows the eventhouses within a workspace.
@@ -94,7 +95,7 @@ def list_eventhouses(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
     }
     df = _create_dataframe(columns=columns)
 
-    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    workspace_id = resolve_workspace_id(workspace)
 
     responses = _base_api(
         request=f"/v1/workspaces/{workspace_id}/eventhouses",
@@ -102,6 +103,7 @@ def list_eventhouses(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
         client="fabric_sp",
     )
 
+    dfs = []
     for r in responses:
         for v in r.get("value", []):
             new_data = {
@@ -109,11 +111,15 @@ def list_eventhouses(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
                 "Eventhouse Id": v.get("id"),
                 "Description": v.get("description"),
             }
-            df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
+            dfs.append(pd.DataFrame(new_data, index=[0]))
+
+    if dfs:
+        df = pd.concat(dfs, ignore_index=True)
 
     return df
 
 
+@log
 def delete_eventhouse(name: str, workspace: Optional[str | UUID] = None):
     """
     Deletes a Fabric eventhouse.
@@ -133,6 +139,7 @@ def delete_eventhouse(name: str, workspace: Optional[str | UUID] = None):
     delete_item(item=name, type="Eventhouse", workspace=workspace)
 
 
+@log
 def get_eventhouse_definition(
     eventhouse: str | UUID,
     workspace: Optional[str | UUID] = None,

sempy_labs/_eventstreams.py

@@ -1,16 +1,18 @@
 import pandas as pd
 from typing import Optional
 from sempy_labs._helper_functions import (
-    resolve_workspace_name_and_id,
     _base_api,
     delete_item,
     _create_dataframe,
     create_item,
+    resolve_workspace_id,
 )
 from uuid import UUID
 import sempy_labs._icons as icons
+from sempy._utils._log import log
 
 
+@log
 def list_eventstreams(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
     """
     Shows the eventstreams within a workspace.
@@ -37,11 +39,12 @@ def list_eventstreams(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
     }
     df = _create_dataframe(columns=columns)
 
-    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    workspace_id = resolve_workspace_id(workspace)
     responses = _base_api(
         request=f"/v1/workspaces/{workspace_id}/eventstreams", uses_pagination=True
     )
 
+    dfs = []
     for r in responses:
         for v in r.get("value", []):
             new_data = {
@@ -49,11 +52,15 @@ def list_eventstreams(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
                 "Eventstream Id": v.get("id"),
                 "Description": v.get("description"),
             }
-            df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
+            dfs.append(pd.DataFrame(new_data, index=[0]))
+
+    if dfs:
+        df = pd.concat(dfs, ignore_index=True)
 
     return df
 
 
+@log
 def create_eventstream(
     name: str, description: Optional[str] = None, workspace: Optional[str | UUID] = None
 ):
@@ -79,6 +86,7 @@ def create_eventstream(
     )
 
 
+@log
 def delete_eventstream(
     eventstream: str | UUID, workspace: Optional[str | UUID] = None, **kwargs
 ):