semantic-link-labs 0.9.4__py3-none-any.whl → 0.9.5__py3-none-any.whl

sempy_labs/lakehouse/_shortcuts.py

@@ -1,16 +1,19 @@
 import sempy.fabric as fabric
+import pandas as pd
 from sempy_labs._helper_functions import (
-    resolve_lakehouse_name,
-    resolve_lakehouse_id,
+    resolve_lakehouse_name_and_id,
     resolve_workspace_name_and_id,
     _base_api,
+    _create_dataframe,
 )
+from sempy._utils._log import log
 from typing import Optional
 import sempy_labs._icons as icons
-from sempy.fabric.exceptions import FabricHTTPException
 from uuid import UUID
+from sempy.fabric.exceptions import FabricHTTPException
 
 
+@log
 def create_shortcut_onelake(
     table_name: str,
     source_lakehouse: str,
@@ -62,21 +65,18 @@ def create_shortcut_onelake(
     (source_workspace_name, source_workspace_id) = resolve_workspace_name_and_id(
         source_workspace
     )
-    source_lakehouse_id = resolve_lakehouse_id(source_lakehouse, source_workspace_id)
-    source_lakehouse_name = fabric.resolve_item_name(
-        item_id=source_lakehouse_id, type="Lakehouse", workspace=source_workspace_id
-    )
 
-    (destination_workspace_name, destination_workspace_id) = resolve_workspace_name_and_id(
-        destination_workspace
+    (source_lakehouse_name, source_lakehouse_id) = resolve_lakehouse_name_and_id(
+        lakehouse=source_lakehouse, workspace=source_workspace_id
     )
-    destination_lakehouse_id = resolve_lakehouse_id(
-        destination_lakehouse, destination_workspace
+
+    (destination_workspace_name, destination_workspace_id) = (
+        resolve_workspace_name_and_id(destination_workspace)
     )
-    destination_lakehouse_name = fabric.resolve_item_name(
-        item_id=destination_lakehouse_id,
-        type="Lakehouse",
-        workspace=destination_workspace_id,
+    (destination_lakehouse_name, destination_lakehouse_id) = (
+        resolve_lakehouse_name_and_id(
+            lakehouse=destination_lakehouse, workspace=destination_workspace_id
+        )
     )
 
     if shortcut_name is None:
@@ -84,18 +84,39 @@ def create_shortcut_onelake(
 
     source_full_path = f"{source_path}/{table_name}"
 
+    actual_shortcut_name = shortcut_name.replace(" ", "")
+
     payload = {
         "path": destination_path,
-        "name": shortcut_name.replace(" ", ""),
+        "name": actual_shortcut_name,
         "target": {
             "oneLake": {
-                "workspaceId": source_workspace_id,
                 "itemId": source_lakehouse_id,
                 "path": source_full_path,
+                "workspaceId": source_workspace_id,
             }
         },
     }
 
+    # Check if the shortcut already exists
+    try:
+        response = _base_api(
+            request=f"/v1/workspaces/{destination_workspace_id}/items/{destination_lakehouse_id}/shortcuts/{destination_path}/{actual_shortcut_name}"
+        )
+        response_json = response.json()
+        del response_json["target"]["type"]
+        if response_json.get("target") == payload.get("target"):
+            print(
+                f"{icons.info} The '{actual_shortcut_name}' shortcut already exists in the '{destination_lakehouse_name}' lakehouse within the '{destination_workspace_name}' workspace."
+            )
+            return
+        else:
+            raise ValueError(
+                f"{icons.red_dot} The '{actual_shortcut_name}' shortcut already exists in the '{destination_lakehouse_name} lakehouse within the '{destination_workspace_name}' workspace but has a different source."
+            )
+    except FabricHTTPException:
+        pass
+
     _base_api(
         request=f"/v1/workspaces/{destination_workspace_id}/items/{destination_lakehouse_id}/shortcuts",
         payload=payload,
@@ -146,17 +167,14 @@ def create_shortcut(
 
     sourceTitle = source_titles[source]
 
-    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
-
-    if lakehouse is None:
-        lakehouse_id = fabric.get_lakehouse_id()
-    else:
-        lakehouse_id = resolve_lakehouse_id(lakehouse, workspace)
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    (lakehouse_name, lakehouse_id) = resolve_lakehouse_name_and_id(
+        lakehouse=lakehouse, workspace=workspace_id
+    )
 
-    client = fabric.FabricRestClient()
     shortcutActualName = shortcut_name.replace(" ", "")
 
-    request_body = {
+    payload = {
         "path": "Tables",
         "name": shortcutActualName,
         "target": {
@@ -168,22 +186,16 @@ def create_shortcut(
         },
     }
 
-    try:
-        response = client.post(
-            f"/v1/workspaces/{workspace_id}/items/{lakehouse_id}/shortcuts",
-            json=request_body,
-        )
-        if response.status_code == 201:
-            print(
-                f"{icons.green_dot} The shortcut '{shortcutActualName}' was created in the '{lakehouse}' lakehouse within"
-                f" the '{workspace} workspace. It is based on the '{subpath}' table in '{sourceTitle}'."
-            )
-        else:
-            print(response.status_code)
-    except Exception as e:
-        raise ValueError(
-            f"{icons.red_dot} Failed to create a shortcut for the '{shortcut_name}' table."
-        ) from e
+    _base_api(
+        request=f"/v1/workspaces/{workspace_id}/items/{lakehouse_id}/shortcuts",
+        method="post",
+        payload=payload,
+        status_codes=201,
+    )
+    print(
+        f"{icons.green_dot} The shortcut '{shortcutActualName}' was created in the '{lakehouse_name}' lakehouse within"
+        f" the '{workspace_name}' workspace. It is based on the '{subpath}' table in '{sourceTitle}'."
+    )
 
 
 def delete_shortcut(
@@ -203,7 +215,7 @@ def delete_shortcut(
         The name of the shortcut.
     shortcut_path : str = "Tables"
         The path of the shortcut to be deleted. Must start with either "Files" or "Tables". Examples: Tables/FolderName/SubFolderName; Files/FolderName/SubFolderName.
-    lakehouse : str, default=None
+    lakehouse : str | uuid.UUID, default=None
         The Fabric lakehouse name in which the shortcut resides.
         Defaults to None which resolves to the lakehouse attached to the notebook.
     workspace : str | UUID, default=None
@@ -213,20 +225,15 @@ def delete_shortcut(
     """
 
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    (lakehouse_name, lakehouse_id) = resolve_lakehouse_name_and_id(
+        lakehouse=lakehouse, workspace=workspace_id
+    )
 
-    if lakehouse is None:
-        lakehouse_id = fabric.get_lakehouse_id()
-        lakehouse = resolve_lakehouse_name(lakehouse_id, workspace_id)
-    else:
-        lakehouse_id = resolve_lakehouse_id(lakehouse, workspace_id)
-
-    client = fabric.FabricRestClient()
-    response = client.delete(
-        f"/v1/workspaces/{workspace_id}/items/{lakehouse_id}/shortcuts/{shortcut_path}/{shortcut_name}"
+    _base_api(
+        request=f"/v1/workspaces/{workspace_id}/items/{lakehouse_id}/shortcuts/{shortcut_path}/{shortcut_name}",
+        method="delete",
     )
 
-    if response.status_code != 200:
-        raise FabricHTTPException(response)
     print(
         f"{icons.green_dot} The '{shortcut_name}' shortcut in the '{lakehouse}' within the '{workspace_name}' workspace has been deleted."
     )
@@ -258,3 +265,135 @@ def reset_shortcut_cache(workspace: Optional[str | UUID] = None):
     print(
         f"{icons.green_dot} The shortcut cache has been reset for the '{workspace_name}' workspace."
     )
+
+
+@log
+def list_shortcuts(
+    lakehouse: Optional[str | UUID] = None,
+    workspace: Optional[str | UUID] = None,
+    path: Optional[str] = None,
+) -> pd.DataFrame:
+    """
+    Shows all shortcuts which exist in a Fabric lakehouse and their properties.
+
+    Parameters
+    ----------
+    lakehouse : str | uuid.UUID, default=None
+        The Fabric lakehouse name or ID.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    workspace : str | uuid.UUID, default=None
+        The name or ID of the Fabric workspace in which lakehouse resides.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    path: str, default=None
+        The path within lakehouse where to look for shortcuts. If provied, must start with either "Files" or "Tables". Examples: Tables/FolderName/SubFolderName; Files/FolderName/SubFolderName.
+        Defaults to None which will retun all shortcuts on the given lakehouse
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing all the shortcuts which exist in the specified lakehouse.
+    """
+
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    (lakehouse_name, lakehouse_id) = resolve_lakehouse_name_and_id(
+        lakehouse=lakehouse, workspace=workspace_id
+    )
+
+    columns = {
+        "Shortcut Name": "string",
+        "Shortcut Path": "string",
+        "Source Type": "string",
+        "Source Workspace Id": "string",
+        "Source Workspace Name": "string",
+        "Source Item Id": "string",
+        "Source Item Name": "string",
+        "Source Item Type": "string",
+        "OneLake Path": "string",
+        "Connection Id": "string",
+        "Location": "string",
+        "Bucket": "string",
+        "SubPath": "string",
+        "Source Properties Raw": "string",
+    }
+    df = _create_dataframe(columns=columns)
+
+    # To improve performance create a dataframe to cache all items for a given workspace
+    itm_clms = {
+        "Id": "string",
+        "Display Name": "string",
+        "Description": "string",
+        "Type": "string",
+        "Workspace Id": "string",
+    }
+    source_items_df = _create_dataframe(columns=itm_clms)
+
+    url = f"/v1/workspaces/{workspace_id}/items/{lakehouse_id}/shortcuts"
+
+    if path is not None:
+        url += f"?parentPath={path}"
+
+    responses = _base_api(
+        request=url,
+        uses_pagination=True,
+    )
+
+    sources = {
+        "AdlsGen2": "adlsGen2",
+        "AmazonS3": "amazonS3",
+        "Dataverse": "dataverse",
+        "ExternalDataShare": "externalDataShare",
+        "GoogleCloudStorage": "googleCloudStorage",
+        "OneLake": "oneLake",
+        "S3Compatible": "s3Compatible",
+    }
+
+    for r in responses:
+        for i in r.get("value", []):
+            tgt = i.get("target", {})
+            tgt_type = tgt.get("type")
+            connection_id = tgt.get(sources.get(tgt_type), {}).get("connectionId")
+            location = tgt.get(sources.get(tgt_type), {}).get("location")
+            sub_path = tgt.get(sources.get(tgt_type), {}).get("subpath")
+            source_workspace_id = tgt.get(sources.get(tgt_type), {}).get("workspaceId")
+            source_item_id = tgt.get(sources.get(tgt_type), {}).get("itemId")
+            bucket = tgt.get(sources.get(tgt_type), {}).get("bucket")
+            source_workspace_name = (
+                fabric.resolve_workspace_name(source_workspace_id)
+                if source_workspace_id is not None
+                else None
+            )
+            # Cache and use it to getitem type and name
+            source_item_type = None
+            source_item_name = None
+            dfI = source_items_df[
+                source_items_df["Workspace Id"] == source_workspace_id
+            ]
+            if dfI.empty:
+                dfI = fabric.list_items(workspace=source_workspace_id)
+                source_items_df = pd.concat([source_items_df, dfI], ignore_index=True)
+
+            dfI_filt = dfI[dfI["Id"] == source_item_id]
+            if not dfI_filt.empty:
+                source_item_type = dfI_filt["Type"].iloc[0]
+                source_item_name = dfI_filt["Display Name"].iloc[0]
+
+            new_data = {
+                "Shortcut Name": i.get("name"),
+                "Shortcut Path": i.get("path"),
+                "Source Type": tgt_type,
+                "Source Workspace Id": source_workspace_id,
+                "Source Workspace Name": source_workspace_name,
+                "Source Item Id": source_item_id,
+                "Source Item Name": source_item_name,
+                "Source Item Type": source_item_type,
+                "OneLake Path": tgt.get(sources.get("oneLake"), {}).get("path"),
+                "Connection Id": connection_id,
+                "Location": location,
+                "Bucket": bucket,
+                "SubPath": sub_path,
+                "Source Properties Raw": str(tgt),
+            }
+            df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
+
+    return df

sempy_labs/report/_generate_report.py

@@ -11,6 +11,7 @@ from sempy_labs._helper_functions import (
     _update_dataframe_datatypes,
     _base_api,
     resolve_item_id,
+    get_item_definition,
 )
 import sempy_labs._icons as icons
 from sempy._utils._log import log
@@ -178,7 +179,9 @@ def update_report_from_reportjson(
 
 
 def get_report_definition(
-    report: str, workspace: Optional[str | UUID] = None, return_dataframe: bool = True
+    report: str | UUID,
+    workspace: Optional[str | UUID] = None,
+    return_dataframe: bool = True,
 ) -> pd.DataFrame | dict:
     """
     Gets the collection of definition files of a report.
@@ -187,8 +190,8 @@ def get_report_definition(
 
     Parameters
     ----------
-    report : str
-        Name of the report.
+    report : str | uuid.UUID
+        Name or ID of the report.
     workspace : str | uuid.UUID, default=None
         The Fabric workspace name or ID in which the report resides.
         Defaults to None which resolves to the workspace of the attached lakehouse
@@ -198,25 +201,14 @@ def get_report_definition(
 
     Returns
     -------
-    pandas.DataFrame | dict
+    pandas.DataFrame
         The collection of report definition files within a pandas dataframe.
     """
 
-    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
-    report_id = resolve_item_id(item=report, type="Report", workspace=workspace)
-
-    result = _base_api(
-        request=f"/v1/workspaces/{workspace_id}/reports/{report_id}/getDefinition",
-        method="post",
-        lro_return_json=True,
-        status_codes=None,
+    return get_item_definition(
+        item=report, type="Report", workspace=workspace, return_dataframe=True
     )
 
-    if return_dataframe:
-        return pd.json_normalize(result["definition"]["parts"])
-    else:
-        return result
-
 
 @log
 def create_model_bpa_report(

sempy_labs/report/_report_bpa.py

@@ -1,4 +1,3 @@
-import sempy.fabric as fabric
 from typing import Optional
 import pandas as pd
 import datetime
@@ -7,8 +6,7 @@ from sempy_labs.report import ReportWrapper, report_bpa_rules
 from sempy_labs._helper_functions import (
     format_dax_object_name,
     save_as_delta_table,
-    resolve_report_id,
-    resolve_lakehouse_name,
+    resolve_item_name_and_id,
     resolve_workspace_capacity,
     _get_column_aggregate,
     resolve_workspace_name_and_id,
@@ -54,9 +52,7 @@ def run_report_bpa(
         A pandas dataframe in HTML format showing report objects which violated the best practice analyzer rules.
     """
 
-    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
-
-    rpt = ReportWrapper(report=report, workspace=workspace_id)
+    rpt = ReportWrapper(report=report, workspace=workspace)
 
     dfCV = rpt.list_custom_visuals()
     dfP = rpt.list_pages()
@@ -149,7 +145,7 @@ def run_report_bpa(
             df_output["Description"] = row["Description"]
             df_output["URL"] = row["URL"]
             df_output["Report URL"] = helper.get_web_url(
-                report=report, workspace=workspace_id
+                report=report, workspace=workspace
             )
 
             page_mapping_dict = dfP.set_index("Page Display Name")["Page URL"].to_dict()
@@ -205,31 +201,28 @@ def run_report_bpa(
 
         now = datetime.datetime.now()
         delta_table_name = "reportbparesults"
-        lakehouse_id = fabric.get_lakehouse_id()
-        lake_workspace = fabric.resolve_workspace_name()
-        lakehouse = resolve_lakehouse_name(
-            lakehouse_id=lakehouse_id, workspace=lake_workspace
-        )
-
-        lakeT = get_lakehouse_tables(lakehouse=lakehouse, workspace=lake_workspace)
+        lakeT = get_lakehouse_tables()
         lakeT_filt = lakeT[lakeT["Table Name"] == delta_table_name]
 
         if len(lakeT_filt) == 0:
             runId = 1
         else:
-            max_run_id = _get_column_aggregate(
-                lakehouse=lakehouse, table_name=delta_table_name
-            )
+            max_run_id = _get_column_aggregate(table_name=delta_table_name)
             runId = max_run_id + 1
 
+        (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+        (report_name, report_id) = resolve_item_name_and_id(
+            item=report, type="Report", workspace=workspace_id
+        )
+
         export_df = finalDF.copy()
         capacity_id, capacity_name = resolve_workspace_capacity(workspace=workspace_id)
         export_df["Capacity Name"] = capacity_name
         export_df["Capacity Id"] = capacity_id
         export_df["Workspace Name"] = workspace_name
         export_df["Workspace Id"] = workspace_id
-        export_df["Report Name"] = report
-        export_df["Report Id"] = resolve_report_id(report, workspace_id)
+        export_df["Report Name"] = report_name
+        export_df["Report Id"] = report_id
         export_df["RunId"] = runId
         export_df["Timestamp"] = now
         export_df["RunId"] = export_df["RunId"].astype(int)

sempy_labs/tom/_model.py

@@ -1172,8 +1172,8 @@ class TOMWrapper:
             Name of the table.
         entity_name : str
             Name of the lakehouse/warehouse table.
-        expression : TOM Object, default=None
-            The expression used by the table.
+        expression : str, default=None
+            The name of the expression used by the partition.
             Defaults to None which resolves to the 'DatabaseQuery' expression.
         description : str, default=None
             A description for the partition.
@@ -1543,6 +1543,7 @@ class TOMWrapper:
         self,
         object: Union["TOM.Table", "TOM.Column", "TOM.Measure", "TOM.Hierarchy"],
         perspective_name: str,
+        include_all: bool = True,
     ):
         """
         Adds an object to a `perspective <https://learn.microsoft.com/dotnet/api/microsoft.analysisservices.perspective?view=analysisservices-dotnet>`_.
@@ -1553,6 +1554,8 @@ class TOMWrapper:
             An object (i.e. table/column/measure) within a semantic model.
         perspective_name : str
             Name of the perspective.
+        include_all : bool, default=True
+            Relevant to tables only, if set to True, includes all columns, measures, and hierarchies within that table in the perspective.
         """
         import Microsoft.AnalysisServices.Tabular as TOM
 
@@ -1578,6 +1581,8 @@ class TOMWrapper:
 
         if objectType == TOM.ObjectType.Table:
             pt = TOM.PerspectiveTable()
+            if include_all:
+                pt.IncludeAll = True
             pt.Table = object
             object.Model.Perspectives[perspective_name].PerspectiveTables.Add(pt)
         elif objectType == TOM.ObjectType.Column:
@@ -3517,14 +3522,14 @@ class TOMWrapper:
 
         return usingView
 
-    def has_incremental_refresh_policy(self, table_name: str):
+    def has_incremental_refresh_policy(self, object):
         """
         Identifies whether a table has an `incremental refresh <https://learn.microsoft.com/power-bi/connect-data/incremental-refresh-overview>`_ policy.
 
         Parameters
         ----------
-        table_name : str
-            Name of the table.
+        object : TOM Object
+            The TOM object within the semantic model. Accepts either a table or the model object.
 
         Returns
         -------
@@ -3532,13 +3537,21 @@ class TOMWrapper:
             An indicator whether a table has an incremental refresh policy.
         """
 
-        hasRP = False
-        rp = self.model.Tables[table_name].RefreshPolicy
-
-        if rp is not None:
-            hasRP = True
+        import Microsoft.AnalysisServices.Tabular as TOM
 
-        return hasRP
+        if object.ObjectType == TOM.ObjectType.Table:
+            if object.RefreshPolicy is not None:
+                return True
+            else:
+                return False
+        elif object.ObjectType == TOM.ObjectType.Model:
+            rp = False
+            for t in self.model.Tables:
+                if t.RefreshPolicy is not None:
+                    rp = True
+            return rp
+        else:
+            raise NotImplementedError
 
     def show_incremental_refresh_policy(self, table_name: str):
         """
@@ -4760,17 +4773,20 @@ class TOMWrapper:
 
         import Microsoft.AnalysisServices.Tabular as TOM
 
-        return (
+        bim = (
             json.loads(TOM.JsonScripter.ScriptCreate(self.model.Database))
             .get("create")
             .get("database")
         )
 
+        return bim
+
     def _reduce_model(self, perspective_name: str):
         """
         Reduces a model's objects based on a perspective. Adds the dependent objects within a perspective to that perspective.
         """
 
+        import Microsoft.AnalysisServices.Tabular as TOM
         from sempy_labs._model_dependencies import get_model_calc_dependencies
 
         fabric.refresh_tom_cache(workspace=self._workspace_id)
@@ -4852,7 +4868,7 @@ class TOMWrapper:
                     object=obj, perspective_name=perspective_name
                 ):
                     self.add_to_perspective(
-                        object=obj, perspective_name=perspective_name
+                        object=obj, perspective_name=perspective_name, include_all=False
                     )
                     added = True
             elif obj_type == "Measure":
@@ -4861,7 +4877,7 @@ class TOMWrapper:
                     object=obj, perspective_name=perspective_name
                 ):
                     self.add_to_perspective(
-                        object=obj, perspective_name=perspective_name
+                        object=obj, perspective_name=perspective_name, include_all=False
                     )
                     added = True
             elif obj_type == "Table":
@@ -4870,7 +4886,7 @@ class TOMWrapper:
                     object=obj, perspective_name=perspective_name
                 ):
                     self.add_to_perspective(
-                        object=obj, perspective_name=perspective_name
+                        object=obj, perspective_name=perspective_name, include_all=False
                     )
                     added = True
             if added:
@@ -4917,7 +4933,9 @@ class TOMWrapper:
             else:
                 for attr in ["Columns", "Measures", "Hierarchies"]:
                     for obj in getattr(t, attr):
-                        if not self.in_perspective(
+                        if attr == "Columns" and obj.Type == TOM.ColumnType.RowNumber:
+                            pass
+                        elif not self.in_perspective(
                             object=obj, perspective_name=perspective_name
                         ):
                             self.remove_object(object=obj)