semantic-link-labs 0.9.6__py3-none-any.whl → 0.9.7__py3-none-any.whl

sempy_labs/migration/_direct_lake_to_import.py

@@ -1,11 +1,16 @@
 import sempy
 from uuid import UUID
 import sempy_labs._icons as icons
+from typing import Optional
 
 
-def migrate_direct_lake_to_import(dataset: str | UUID, workspace: str | UUID):
+def migrate_direct_lake_to_import(
+    dataset: str | UUID,
+    workspace: Optional[str | UUID] = None,
+    mode: str = "import",
+):
     """
-    Migrates a semantic model from Direct Lake mode to import mode. After running this function, you must go to the semantic model settings and update the cloud connection. Not doing so will result in an inablity to refresh/use the semantic model.
+    Migrates a semantic model or specific table(s) from a Direct Lake mode to import or DirectQuery mode. After running this function, you must go to the semantic model settings and update the cloud connection. Not doing so will result in an inablity to refresh/use the semantic model.
 
     Parameters
     ----------
@@ -15,12 +20,29 @@ def migrate_direct_lake_to_import(dataset: str | UUID, workspace: str | UUID):
         The Fabric workspace name or ID.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
+    mode : str, default="import"
+        The mode to migrate to. Can be either "import" or "directquery".
     """
 
     sempy.fabric._client._utils._init_analysis_services()
     import Microsoft.AnalysisServices.Tabular as TOM
     from sempy_labs.tom import connect_semantic_model
 
+    modes = {
+        "import": "Import",
+        "directquery": "DirectQuery",
+        "dq": "DirectQuery",
+    }
+
+    # Resolve mode
+    mode = mode.lower()
+    actual_mode = modes.get(mode)
+    if actual_mode is None:
+        raise ValueError(f"Invalid mode '{mode}'. Must be one of {list(modes.keys())}.")
+
+    # if isinstance(tables, str):
+    #     tables = [tables]
+
     with connect_semantic_model(
         dataset=dataset, workspace=workspace, readonly=False
     ) as tom:
@@ -31,7 +53,14 @@ def migrate_direct_lake_to_import(dataset: str | UUID, workspace: str | UUID):
             )
             return
 
-        for t in tom.model.Tables:
+        # if tables is None:
+        table_list = [t for t in tom.model.Tables]
+        # else:
+        #    table_list = [t for t in tom.model.Tables if t.Name in tables]
+        # if not table_list:
+        #    raise ValueError(f"{icons.red_dot} No tables found to migrate.")
+
+        for t in table_list:
             table_name = t.Name
             if t.Partitions.Count == 1 and all(
                 p.Mode == TOM.ModeType.DirectLake for p in t.Partitions
@@ -51,16 +80,24 @@ def migrate_direct_lake_to_import(dataset: str | UUID, workspace: str | UUID):
                     table_name=table_name,
                     partition_name=partition_name,
                     expression=expression,
-                    mode="Import",
+                    mode=actual_mode,
                 )
                 # Remove Direct Lake partition
                 tom.remove_object(object=p)
+                # if tables is not None:
+                #    print(
+                #        f"{icons.green_dot} The '{table_name}' table has been migrated to '{actual_mode}' mode."
+                #    )
 
         tom.model.Model.DefaultMode = TOM.ModeType.Import
+    # if tables is None:
+    print(
+        f"{icons.green_dot} All tables which were in Direct Lake mode have been migrated to '{actual_mode}' mode."
+    )
 
-        # Check
-        # for t in tom.model.Tables:
-        #    if t.Partitions.Count == 1 and all(p.Mode == TOM.ModeType.Import for p in t.Partitions) and t.CalculationGroup is None:
-        #        p = next(p for p in t.Partitions)
-        #        print(p.Name)
-        #        print(p.Source.Expression)
+    # Check
+    # for t in tom.model.Tables:
+    #    if t.Partitions.Count == 1 and all(p.Mode == TOM.ModeType.Import for p in t.Partitions) and t.CalculationGroup is None:
+    #        p = next(p for p in t.Partitions)
+    #        print(p.Name)
+    #        print(p.Source.Expression)

sempy_labs/report/__init__.py

@@ -1,3 +1,6 @@
+from sempy_labs.report._save_report import (
+    save_report_as_pbip,
+)
 from sempy_labs.report._reportwrapper import (
     ReportWrapper,
 )
@@ -46,4 +49,5 @@ __all__ = [
     "run_report_bpa",
     "get_report_datasources",
     "download_report",
+    "save_report_as_pbip",
 ]

sempy_labs/report/_report_functions.py

@@ -116,9 +116,9 @@ def report_dependency_tree(workspace: Optional[str | UUID] = None):
     dfR.rename(columns={"Name": "Report Name"}, inplace=True)
     dfR = dfR[["Report Name", "Dataset Name"]]
 
-    report_icon = "\U0001F4F6"
-    dataset_icon = "\U0001F9CA"
-    workspace_icon = "\U0001F465"
+    report_icon = "\U0001f4f6"
+    dataset_icon = "\U0001f9ca"
+    workspace_icon = "\U0001f465"
 
     node_dict = {}
     rootNode = Node(workspace_name)

sempy_labs/report/_report_helper.py

@@ -236,15 +236,27 @@ def find_entity_property_pairs(data, result=None, keys_path=None):
         keys_path = []
 
     if isinstance(data, dict):
+        expression = data.get("Expression", {})
+        source_ref = (
+            expression.get("SourceRef", {}) if isinstance(expression, dict) else {}
+        )
+
         if (
-            "Entity" in data.get("Expression", {}).get("SourceRef", {})
+            isinstance(source_ref, dict)
+            and "Entity" in source_ref
             and "Property" in data
         ):
-            entity = data.get("Expression", {}).get("SourceRef", {}).get("Entity", {})
-            property_value = data.get("Property")
-            object_type = keys_path[-1].replace("HierarchyLevel", "Hierarchy")
+            entity = source_ref.get("Entity", "")
+            property_value = data.get("Property", "")
+
+            object_type = (
+                keys_path[-1].replace("HierarchyLevel", "Hierarchy")
+                if keys_path
+                else "Unknown"
+            )
             result[property_value] = (entity, object_type)
-            keys_path.pop()
+            if keys_path:
+                keys_path.pop()
 
         # Recursively search the rest of the dictionary
         for key, value in data.items():

sempy_labs/report/_reportwrapper.py

@@ -966,17 +966,25 @@ class ReportWrapper:
                 keys_path = []
 
             if isinstance(data, dict):
+                expression = data.get("Expression", {})
+                source_ref = (
+                    expression.get("SourceRef", {})
+                    if isinstance(expression, dict)
+                    else {}
+                )
                 if (
-                    "Entity" in data.get("Expression", {}).get("SourceRef", {})
+                    isinstance(source_ref, dict)
+                    and "Entity" in source_ref
                     and "Property" in data
                 ):
-                    entity = (
-                        data.get("Expression", {})
-                        .get("SourceRef", {})
-                        .get("Entity", {})
+                    entity = source_ref.get("Entity", "")
+                    property_value = data.get("Property", "")
+
+                    object_type = (
+                        keys_path[-1].replace("HierarchyLevel", "Hierarchy")
+                        if keys_path
+                        else "Unknown"
                     )
-                    property_value = data.get("Property", {})
-                    object_type = keys_path[-1].replace("HierarchyLevel", "Hierarchy")
                     is_agg = keys_path[-3] == "Aggregation"
                     is_viz_calc = keys_path[-3] == "NativeVisualCalculation"
                     is_sparkline = keys_path[-3] == "SparklineData"
@@ -987,7 +995,8 @@ class ReportWrapper:
                         is_viz_calc,
                         is_sparkline,
                     )
-                    keys_path.pop()
+                    if keys_path:
+                        keys_path.pop()
 
                 # Recursively search the rest of the dictionary
                 for key, value in data.items():

sempy_labs/report/_save_report.py

@@ -0,0 +1,147 @@
+import os
+import base64
+import json
+import sempy.fabric as fabric
+import sempy_labs._icons as icons
+from sempy_labs.report._generate_report import get_report_definition
+from sempy_labs._generate_semantic_model import get_semantic_model_definition
+from sempy_labs._helper_functions import (
+    _mount,
+    resolve_workspace_name_and_id,
+    resolve_item_name,
+    resolve_workspace_name,
+    resolve_item_name_and_id,
+)
+from uuid import UUID
+from sempy._utils._log import log
+from typing import Optional
+
+
+@log
+def save_report_as_pbip(
+    report: str | UUID,
+    workspace: Optional[str | UUID] = None,
+    thick_report: bool = True,
+    live_connect: bool = True,
+    lakehouse: Optional[str | UUID] = None,
+    lakehouse_workspace: Optional[str | UUID] = None,
+):
+    """
+    Saves a report as a .pbip file to the default lakehouse attached to the notebook.
+
+    Parameters
+    ----------
+    report : str | uuid.UUID
+        Name or ID of the Power BI report.
+    workspace : str | uuid.UUID, default=None
+        The name or ID of the Fabric workspace in which the report resides.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    thick_report : bool, default=True
+        If set to True, saves the report and underlying semantic model.
+        If set to False, saves just the report.
+    live_connect : bool, default=True
+        If set to True, saves a .pbip live-connected to the workspace in the Power BI / Fabric service.
+        If set to False, saves a .pbip with a local model, independent from the Power BI / Fabric service.
+    lakehouse : str | uuid.UUID, default=None
+        The Fabric lakehouse name or ID. This will be the lakehouse to which the report is saved.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    lakehouse_workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID used by the lakehouse.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    (report_workspace_name, report_workspace_id) = resolve_workspace_name_and_id(
+        workspace
+    )
+    (report_name, report_id) = resolve_item_name_and_id(
+        item=report, type="Report", workspace=workspace
+    )
+    indent = 2
+
+    local_path = _mount(lakehouse=lakehouse, workspace=lakehouse_workspace)
+    save_location = f"{local_path}/Files"
+
+    # Find semantic model info
+    dfR = fabric.list_reports(workspace=workspace)
+    dfR_filt = dfR[dfR["Id"] == report_id]
+    if dfR_filt.empty:
+        raise ValueError(
+            f"{icons.red_dot} The '{report} report does not exist within the '{report_workspace_name} workspace."
+        )
+
+    dataset_id = dfR_filt["Dataset Id"].iloc[0]
+    dataset_workspace_id = dfR_filt["Dataset Workspace Id"].iloc[0]
+    dataset_name = resolve_item_name(item_id=dataset_id, workspace=dataset_workspace_id)
+    dataset_workspace_name = resolve_workspace_name(dataset_workspace_id)
+    path_prefix = f"{save_location}/{report_workspace_name}/{report_name}/{report_name}"
+
+    # Local model not supported if the report and model are in different workspaces
+    if dataset_workspace_name != report_workspace_name and not live_connect:
+        live_connect = True
+        print(
+            f"{icons.warning} The '{report_name}' report from the '{report_workspace_name}' workspace is being saved as a live-connected report/model."
+        )
+
+    def add_files(name, type, object_workspace):
+
+        path_prefix_full = f"{path_prefix}.{type}"
+
+        if type == "Report":
+            dataframe = get_report_definition(report=name, workspace=workspace)
+        elif type == "SemanticModel":
+            dataframe = get_semantic_model_definition(
+                dataset=name, workspace=object_workspace
+            )
+        else:
+            raise NotImplementedError
+
+        # Create and save files based on dataset/report definition
+        for _, r in dataframe.iterrows():
+            path = r["path"]
+            file_content = base64.b64decode(r["payload"])
+            file_path = f"{path_prefix_full}/{path}"
+            os.makedirs(os.path.dirname(file_path), exist_ok=True)
+
+            # Update the definition.pbir file for local models
+            if not live_connect and type == "Report" and path == "definition.pbir":
+                file_content = {
+                    "version": "1.0",
+                    "datasetReference": {
+                        "byPath": {"path": f"../{report_name}.SemanticModel"},
+                        "byConnection": None,
+                    },
+                }
+
+                with open(file_path, "w") as f:
+                    json.dump(file_content, f, indent=indent)
+            else:
+                with open(file_path, "wb") as f:
+                    f.write(file_content)
+
+        # Create and save .pbip file for report, converting the file extension
+        if type == "Report":
+            # Standard .pbip file content
+            pbip = {
+                "version": "1.0",
+                "artifacts": [{"report": {"path": f"{report_name}.Report"}}],
+                "settings": {"enableAutoRecovery": True},
+            }
+            # Ensure the directory exists
+            os.makedirs(os.path.dirname(path_prefix), exist_ok=True)
+            # Write the .pbip file directly
+            pbip_final = f"{path_prefix}.pbip"
+            with open(pbip_final, "w") as file:
+                json.dump(pbip, file, indent=indent)
+
+    add_files(name=report_name, type="Report", object_workspace=workspace)
+    if thick_report:
+        add_files(
+            name=dataset_name,
+            type="SemanticModel",
+            object_workspace=dataset_workspace_name,
+        )
+    print(
+        f"{icons.green_dot} The '{report_name}' report within the '{report_workspace_name}' workspace has been saved to this location: {save_location}."
+    )

sempy_labs/tom/_model.py

@@ -11,6 +11,9 @@ from sempy_labs._helper_functions import (
     resolve_dataset_name_and_id,
     resolve_workspace_name_and_id,
     _base_api,
+    resolve_workspace_id,
+    resolve_item_id,
+    resolve_lakehouse_id,
 )
 from sempy_labs._list_functions import list_relationships
 from sempy_labs._refresh_semantic_model import refresh_semantic_model
@@ -84,7 +87,7 @@ class TOMWrapper:
         # No token provider (standard authentication)
         if self._token_provider is None:
             self._tom_server = fabric.create_tom_server(
-                readonly=readonly, workspace=workspace_id
+                dataset=dataset, readonly=readonly, workspace=workspace_id
             )
         # Service Principal Authentication for Azure AS via token provider
         elif self._is_azure_as:
@@ -2257,7 +2260,7 @@ class TOMWrapper:
 
         if validate:
             dax_query = f"""
-            define measure '{table_name}'[test] = 
+            define measure '{table_name}'[test] =
             var mn = MIN('{table_name}'[{column_name}])
             var ma = MAX('{table_name}'[{column_name}])
             var x = COUNTROWS(DISTINCT('{table_name}'[{column_name}]))
@@ -3309,14 +3312,16 @@ class TOMWrapper:
             .tolist()
         )
         cols = (
-            fil[fil["Referenced Object Type"] == "Column"][
+            fil[fil["Referenced Object Type"].isin(["Column", "Calc Column"])][
                 "Referenced Full Object Name"
             ]
             .unique()
             .tolist()
         )
         tbls = (
-            fil[fil["Referenced Object Type"] == "Table"]["Referenced Table"]
+            fil[fil["Referenced Object Type"].isin(["Table", "Calc Table"])][
+                "Referenced Table"
+            ]
             .unique()
             .tolist()
         )
@@ -3489,7 +3494,7 @@ class TOMWrapper:
                         tableList.append(c.Parent.Name)
                 if (
                     re.search(
-                        create_pattern(tableList, re.escape(obj.Name)),
+                        create_pattern(tableList, obj.Name),
                         expr,
                     )
                     is not None
@@ -4857,6 +4862,18 @@ class TOMWrapper:
 
         result_df = pd.DataFrame(columns=["Table Name", "Object Name", "Object Type"])
 
+        def add_to_result(table_name, object_name, object_type, dataframe):
+
+            new_data = {
+                "Table Name": table_name,
+                "Object Name": object_name,
+                "Object Type": object_type,
+            }
+
+            return pd.concat(
+                [dataframe, pd.DataFrame(new_data, index=[0])], ignore_index=True
+            )
+
         for _, r in filt.iterrows():
             added = False
             obj_type = r["Referenced Object Type"]
@@ -4890,15 +4907,7 @@ class TOMWrapper:
                     )
                     added = True
             if added:
-                new_data = {
-                    "Table Name": table_name,
-                    "Object Name": object_name,
-                    "Object Type": obj_type,
-                }
-
-                result_df = pd.concat(
-                    [result_df, pd.DataFrame(new_data, index=[0])], ignore_index=True
-                )
+                result_df = add_to_result(table_name, object_name, obj_type, result_df)
 
         # Reduce model...
 
@@ -4921,11 +4930,21 @@ class TOMWrapper:
                 self.add_to_perspective(
                     object=r.FromColumn, perspective_name=perspective_name
                 )
+
+                result_df = add_to_result(
+                    r.FromTable.Name, r.FromColumn.Name, "Column", result_df
+                )
             if not self.in_perspective(r.ToColumn, perspective_name=perspective_name):
+                table_name = r.ToTable.Name
+                object_name = r.ToColumn.Name
                 self.add_to_perspective(
                     object=r.ToColumn, perspective_name=perspective_name
                 )
 
+                result_df = add_to_result(
+                    r.ToTable.Name, r.ToColumn.Name, "Column", result_df
+                )
+
         # Remove objects not in the perspective
         for t in self.model.Tables:
             if not self.in_perspective(object=t, perspective_name=perspective_name):
@@ -4943,6 +4962,111 @@ class TOMWrapper:
         # Return the objects added to the perspective based on dependencies
         return result_df.drop_duplicates()
 
+    def convert_direct_lake_to_import(
+        self,
+        table_name: str,
+        entity_name: Optional[str] = None,
+        schema: Optional[str] = None,
+        source: Optional[str | UUID] = None,
+        source_type: str = "Lakehouse",
+        source_workspace: Optional[str | UUID] = None,
+    ):
+        """
+        Converts a Direct Lake table's partition to an import-mode partition.
+
+        The entity_name and schema parameters default to using the existing values in the Direct Lake partition. The source, source_type, and source_workspace
+        parameters do not default to existing values. This is because it may not always be possible to reconcile the source and its workspace.
+
+        Parameters
+        ----------
+        table_name : str
+            The table name.
+        entity_name : str, default=None
+            The entity name of the Direct Lake partition (the table name in the source).
+        schema : str, default=None
+            The schema of the source table. Defaults to None which resolves to the existing schema.
+        source : str | uuid.UUID, default=None
+            The source name or ID. This is the name or ID of the Lakehouse or Warehouse.
+        source_type : str, default="Lakehouse"
+            The source type (i.e. "Lakehouse" or "Warehouse").
+        source_workspace: str | uuid.UUID, default=None
+            The workspace name or ID of the source. This is the workspace in which the Lakehouse or Warehouse exists.
+            Defaults to None which resolves to the workspace of the attached lakehouse
+            or if no lakehouse attached, resolves to the workspace of the notebook.
+        """
+        import Microsoft.AnalysisServices.Tabular as TOM
+
+        p = next(p for p in self.model.Tables[table_name].Partitions)
+        if p.Mode != TOM.ModeType.DirectLake:
+            print(f"{icons.info} The '{table_name}' table is not in Direct Lake mode.")
+            return
+
+        partition_name = p.Name
+        partition_entity_name = entity_name or p.Source.EntityName
+        partition_schema = schema or p.Source.SchemaName
+
+        # Update name of the Direct Lake partition (will be removed later)
+        self.model.Tables[table_name].Partitions[
+            partition_name
+        ].Name = f"{partition_name}_remove"
+
+        source_workspace_id = resolve_workspace_id(workspace=source_workspace)
+        if source_type == "Lakehouse":
+            item_id = resolve_lakehouse_id(
+                lakehouse=source, workspace=source_workspace_id
+            )
+        else:
+            item_id = resolve_item_id(
+                item=source, type=source_type, workspace=source_workspace_id
+            )
+
+        def _generate_m_expression(
+            workspace_id, artifact_id, artifact_type, table_name, schema_name
+        ):
+            """
+            Generates the M expression for the import partition.
+            """
+
+            if artifact_type == "Lakehouse":
+                type_id = "lakehouseId"
+            elif artifact_type == "Warehouse":
+                type_id = "warehouseId"
+            else:
+                raise NotImplementedError
+
+            full_table_name = (
+                f"{schema_name}.{table_name}" if schema_name else table_name
+            )
+
+            return f"""let
+                Source = {artifact_type}.Contents(null),
+                #"Workspace" = Source{{[workspaceId="{workspace_id}"]}}[Data],
+                #"Artifact" = #"Workspace"{{[{type_id}="{artifact_id}"]}}[Data],
+                result = #"Artifact"{{[Id="{full_table_name}",ItemKind="Table"]}}[Data]
+            in
+                result
+            """
+
+        m_expression = _generate_m_expression(
+            source_workspace_id,
+            item_id,
+            source_type,
+            partition_entity_name,
+            partition_schema,
+        )
+
+        # Add the import partition
+        self.add_m_partition(
+            table_name=table_name,
+            partition_name=f"{partition_name}",
+            expression=m_expression,
+            mode="Import",
+        )
+        # Remove the Direct Lake partition
+        self.remove_object(object=p)
+
+        print(f"{icons.green_dot} The '{table_name}' table has been converted to Import mode.")
+
     def close(self):
 
         if not self._readonly and self.model is not None:
@@ -4956,18 +5080,25 @@ class TOMWrapper:
                         p.SourceType == TOM.PartitionSourceType.Entity
                         for p in t.Partitions
                     ):
-                        if t.LineageTag in list(self._table_map.keys()):
-                            if self._table_map.get(t.LineageTag) != t.Name:
-                                self.add_changed_property(object=t, property="Name")
+                        entity_name = next(p.Source.EntityName for p in t.Partitions)
+                        if t.Name != entity_name:
+                            self.add_changed_property(object=t, property="Name")
+                        # if t.LineageTag in list(self._table_map.keys()):
+                        #    if self._table_map.get(t.LineageTag) != t.Name:
+                        #        self.add_changed_property(object=t, property="Name")
 
                 for c in self.all_columns():
+                    # if c.LineageTag in list(self._column_map.keys()):
+                    if any(
+                        p.SourceType == TOM.PartitionSourceType.Entity
+                        for p in c.Parent.Partitions
+                    ):
+                        if c.Name != c.SourceColumn:
+                            self.add_changed_property(object=c, property="Name")
+                        # c.SourceLineageTag = c.SourceColumn
+                        # if self._column_map.get(c.LineageTag)[0] != c.Name:
+                        #    self.add_changed_property(object=c, property="Name")
                     if c.LineageTag in list(self._column_map.keys()):
-                        if any(
-                            p.SourceType == TOM.PartitionSourceType.Entity
-                            for p in c.Parent.Partitions
-                        ):
-                            if self._column_map.get(c.LineageTag)[0] != c.Name:
-                                self.add_changed_property(object=c, property="Name")
                         if self._column_map.get(c.LineageTag)[1] != c.DataType:
                             self.add_changed_property(object=c, property="DataType")