semantic-link-labs 0.12.3__py3-none-any.whl → 0.12.5__py3-none-any.whl

sempy_labs/lakehouse/_partitioning.py

@@ -0,0 +1,165 @@
+from typing import Optional, List
+from uuid import UUID
+from sempy_labs._helper_functions import (
+    _create_spark_session,
+    create_abfss_path,
+    resolve_workspace_id,
+    resolve_lakehouse_id,
+    _get_delta_table,
+)
+from sempy._utils._log import log
+
+
+@log
+def _get_partitions(
+    table_name: str,
+    schema_name: Optional[str] = None,
+    lakehouse: Optional[str | UUID] = None,
+    workspace: Optional[str | UUID] = None,
+):
+
+    workspace_id = resolve_workspace_id(workspace)
+    lakehouse_id = resolve_lakehouse_id(lakehouse, workspace)
+    path = create_abfss_path(lakehouse_id, workspace_id, table_name, schema_name)
+
+    delta_table = _get_delta_table(path)
+    details_df = delta_table.detail()
+
+    return details_df.collect()[0].asDict()
+
+
+@log
+def is_partitioned(
+    table: str,
+    schema: Optional[str] = None,
+    lakehouse: Optional[str | UUID] = None,
+    workspace: Optional[str | UUID] = None,
+) -> bool:
+    """
+    Checks if a delta table is partitioned.
+
+    Parameters
+    ----------
+    table : str
+        The name of the delta table.
+    schema : str, optional
+        The schema of the table to check. If not provided, the default schema is used.
+    lakehouse : str | uuid.UUID, default=None
+        The Fabric lakehouse name or ID.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID used by the lakehouse.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    bool
+        True if the table is partitioned, False otherwise.
+    """
+
+    details = _get_partitions(
+        table_name=table, schema_name=schema, lakehouse=lakehouse, workspace=workspace
+    )
+    return len(details["partitionColumns"]) > 0
+
+
+@log
+def list_partitioned_columns(
+    table: str,
+    schema: Optional[str] = None,
+    lakehouse: Optional[str | UUID] = None,
+    workspace: Optional[str | UUID] = None,
+) -> List[str]:
+    """
+    Lists the partitioned columns of a delta table.
+
+    Parameters
+    ----------
+    table : str
+        The name of the delta table.
+    schema : str, optional
+        The schema of the table to check. If not provided, the default schema is used.
+    lakehouse : str | uuid.UUID, default=None
+        The Fabric lakehouse name or ID.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID used by the lakehouse.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    List[str]
+        The list of partitioned columns.
+    """
+
+    details = _get_partitions(
+        table_name=table, schema_name=schema, lakehouse=lakehouse, workspace=workspace
+    )
+
+    return details["partitionColumns"]
+
+
+@log
+def is_over_partitioned(
+    table: str,
+    schema: Optional[str] = None,
+    lakehouse: Optional[str | UUID] = None,
+    workspace: Optional[str | UUID] = None,
+    total_table_size_gb: int = 1000,
+    average_partition_size_gb: int = 1,
+) -> bool:
+    """
+    Checks if a delta table is over-partitioned.
+
+    Parameters
+    ----------
+    table : str
+        The name of the delta table.
+    schema : str, optional
+        The schema of the table to check. If not provided, the default schema is used.
+    lakehouse : str | uuid.UUID, default=None
+        The Fabric lakehouse name or ID.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID used by the lakehouse.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    total_table_size_gb : int, default=1000
+        Threshold for total table size in GB (default 1TB).
+    average_partition_size_gb : int, default=1
+        Threshold for average partition size in GB.
+
+    Returns
+    -------
+    bool
+        True if the table is over-partitioned, False otherwise.
+    """
+
+    workspace_id = resolve_workspace_id(workspace)
+    lakehouse_id = resolve_lakehouse_id(lakehouse, workspace)
+    path = create_abfss_path(lakehouse_id, workspace_id, table, schema)
+    # Get DeltaTable details
+    spark = _create_spark_session()
+    details_df = spark.sql(f"DESCRIBE DETAIL delta.`{path}`")
+    details = details_df.collect()[0].asDict()
+
+    # Extract relevant fields
+    size_bytes = details["sizeInBytes"]
+    partition_cols = details["partitionColumns"]
+    num_files = details["numFiles"]
+
+    total_size_gb = size_bytes / (1024**3)
+
+    # Only check if the table is partitioned
+    if len(partition_cols) > 0 and num_files > 0:
+        avg_partition_size_gb = total_size_gb / num_files
+
+        if (
+            total_size_gb < total_table_size_gb
+            or avg_partition_size_gb < average_partition_size_gb
+        ):
+            return True
+
+    return False

sempy_labs/report/_export_report.py

@@ -12,10 +12,6 @@ from typing import Optional
 from sempy._utils._log import log
 import sempy_labs._icons as icons
 from uuid import UUID
-from sempy_labs.report._report_functions import (
-    list_report_visuals,
-    list_report_pages,
-)
 
 
 @log
@@ -187,15 +183,7 @@ def export_report(
             request_body = {"format": export_format, "powerBIReportConfiguration": {}}
 
             request_body["powerBIReportConfiguration"]["pages"] = []
-            dfPage = list_report_pages(report=report, workspace=workspace_id)
-
             for page in page_name:
-                dfPage_filt = dfPage[dfPage["Page ID"] == page]
-                if len(dfPage_filt) == 0:
-                    raise ValueError(
-                        f"{icons.red_dot} The '{page}' page does not exist in the '{report}' report within the '{workspace_name}' workspace."
-                    )
-
                 page_dict = {"pageName": page}
                 request_body["powerBIReportConfiguration"]["pages"].append(page_dict)
 
@@ -209,19 +197,9 @@ def export_report(
             request_body = {"format": export_format, "powerBIReportConfiguration": {}}
 
             request_body["powerBIReportConfiguration"]["pages"] = []
-            dfVisual = list_report_visuals(report=report, workspace=workspace_id)
             a = 0
             for page in page_name:
                 visual = visual_name[a]
-
-                dfVisual_filt = dfVisual[
-                    (dfVisual["Page ID"] == page) & (dfVisual["Visual ID"] == visual)
-                ]
-                if len(dfVisual_filt) == 0:
-                    raise ValueError(
-                        f"{icons.red_dot} The '{visual}' visual does not exist on the '{page}' in the '{report}' report within the '{workspace_name}' workspace."
-                    )
-
                 page_dict = {"pageName": page, "visualName": visual}
                 request_body["powerBIReportConfiguration"]["pages"].append(page_dict)
                 a += 1

sempy_labs/report/_report_rebind.py

@@ -1,4 +1,6 @@
 from sempy_labs._helper_functions import (
+    resolve_item_id,
+    resolve_workspace_id,
     resolve_workspace_name_and_id,
     resolve_item_name_and_id,
     _base_api,
@@ -8,6 +10,7 @@ from typing import Optional, List
 from sempy._utils._log import log
 import sempy_labs._icons as icons
 from uuid import UUID
+import sempy.fabric as fabric
 
 
 @log
@@ -108,57 +111,40 @@ def report_rebind_all(
         the new semantic model.
     """
 
-    from sempy_labs._list_functions import list_reports_using_semantic_model
-
-    (dataset_name, dataset_id) = resolve_dataset_name_and_id(
-        dataset=dataset, workspace=dataset_workspace
+    (dataset_name, dataset_id) = resolve_item_name_and_id(
+        item=dataset, type="SemanticModel", workspace=dataset_workspace
     )
-    (new_dataset_name, new_dataset_id) = resolve_dataset_name_and_id(
-        dataset=new_dataset, workspace=new_dataset_workspace
+    new_dataset_id = resolve_item_id(
+        item=new_dataset, type="SemanticModel", workspace=new_dataset_workspace
     )
 
     if dataset_id == new_dataset_id:
         raise ValueError(
-            f"{icons.red_dot} The 'dataset' and 'new_dataset' parameters are both set to '{dataset}'. These parameters must be set to different values."
+            f"{icons.red_dot} The 'dataset' and 'new_dataset' parameters are both set to the same semantic model within the same workspace. These parameters must be set to different values."
         )
-    (dataset_workspace_name, dataset_workspace_id) = resolve_workspace_name_and_id(
-        workspace=dataset_workspace
-    )
+    dataset_workspace_id = resolve_workspace_id(workspace=dataset_workspace)
 
-    if isinstance(report_workspace, str):
+    if isinstance(report_workspace, str) or report_workspace is None:
         report_workspace = [report_workspace]
 
-    dfR = list_reports_using_semantic_model(
-        dataset=dataset, workspace=dataset_workspace
-    )
-
-    if dfR.empty:
-        print(
-            f"{icons.info} The '{dataset_name}' semantic model within the '{dataset_workspace_name}' workspace has no dependent reports."
-        )
-        return
-
-    if report_workspace is None:
-        dfR_filt = dfR.copy()
-    else:
+    for w in report_workspace:
+        dfR = fabric.list_reports(workspace=w)
         dfR_filt = dfR[
-            (dfR["Report Workspace Name"].isin(report_workspace))
-            | (dfR["Report Workspace ID"].isin(report_workspace))
+            (dfR["Dataset ID"] == dataset_id)
+            & (dfR["Dataset Workspace Id"] == dataset_workspace_id)
         ]
-
-    if dfR_filt.empty:
-        print(
-            f"{icons.info} No reports found for the '{dataset_name}' semantic model within the '{dataset_workspace_name}' workspace."
-        )
-        return
-
-    for _, r in dfR_filt.iterrows():
-        rpt_name = r["Report Name"]
-        rpt_wksp = r["Report Workspace Name"]
-
-        report_rebind(
-            report=rpt_name,
-            dataset=new_dataset,
-            report_workspace=rpt_wksp,
-            dataset_workspace=new_dataset_workspace,
-        )
+        if dfR_filt.empty:
+            (wksp_name, _) = resolve_workspace_name_and_id(workspace=w)
+            print(
+                f"{icons.info} No reports found for the '{dataset_name}' semantic model within the '{wksp_name}' workspace."
+            )
+        else:
+            # Rebind reports to new dataset
+            for _, r in dfR_filt.iterrows():
+                rpt_name = r["Name"]
+                report_rebind(
+                    report=rpt_name,
+                    dataset=new_dataset,
+                    report_workspace=w,
+                    dataset_workspace=new_dataset_workspace,
+                )

sempy_labs/report/_reportwrapper.py

@@ -171,6 +171,7 @@ class ReportWrapper:
         self,
         file_path: str,
         json_path: Optional[str] = None,
+        verbose: bool = True,
     ) -> dict | List[Tuple[str, dict]]:
         """
         Get the json content of the specified report definition file.
@@ -181,6 +182,8 @@ class ReportWrapper:
             The path of the report definition file. For example: "definition/pages/pages.json". You may also use wildcards. For example: "definition/pages/*/page.json".
         json_path : str, default=None
             The json path to the specific part of the file to be retrieved. If None, the entire file content is returned.
+        verbose : bool, default=True
+            If True, prints messages about the retrieval process. If False, suppresses these messages.
 
         Returns
         -------
@@ -192,6 +195,7 @@ class ReportWrapper:
 
         # Find matching parts
         if "*" in file_path:
+            results = []
             matching_parts = [
                 (part.get("path"), part.get("payload"))
                 for part in parts
@@ -199,9 +203,11 @@ class ReportWrapper:
             ]
 
             if not matching_parts:
-                raise ValueError(
-                    f"{icons.red_dot} No files match the wildcard path '{file_path}'."
-                )
+                if verbose:
+                    print(
+                        f"{icons.red_dot} No files match the wildcard path '{file_path}'."
+                    )
+                return results
 
             results = []
             for path, payload in matching_parts:
@@ -220,8 +226,8 @@ class ReportWrapper:
                     #    raise ValueError(
                     #        f"{icons.red_dot} No match found for '{json_path}' in '{path}'."
                     #    )
-            if not results:
-                raise ValueError(
+            if not results and verbose:
+                print(
                     f"{icons.red_dot} No match found for '{json_path}' in any of the files matching the wildcard path '{file_path}'."
                 )
             return results
@@ -241,14 +247,11 @@ class ReportWrapper:
                     matches = jsonpath_expr.find(payload)
                     if matches:
                         return matches[0].value
-                    else:
-                        raise ValueError(
-                            f"{icons.red_dot} No match found for '{json_path}'."
-                        )
+                    elif verbose:
+                        print(f"{icons.red_dot} No match found for '{json_path}'.")
 
-        raise ValueError(
-            f"{icons.red_dot} File '{file_path}' not found in report definition."
-        )
+        if verbose:
+            print(f"{icons.red_dot} File '{file_path}' not found in report definition.")
 
     def add(self, file_path: str, payload: dict | bytes):
         """
@@ -674,33 +677,65 @@ class ReportWrapper:
         columns = {
             "Custom Visual Name": "str",
             "Custom Visual Display Name": "str",
+            "Is Public": "bool",
             "Used in Report": "bool",
         }
 
         df = _create_dataframe(columns=columns)
 
-        report_file = self.get(file_path=self._report_file_path)
+        visuals = []
+        rp = self.get(
+            file_path=self._report_file_path,
+            json_path="$.resourcePackages",
+            verbose=False,
+        )
 
-        df["Custom Visual Name"] = report_file.get("publicCustomVisuals")
-        df["Custom Visual Display Name"] = df["Custom Visual Name"].apply(
-            lambda x: helper.vis_type_mapping.get(x, x)
+        if rp:
+            visuals += [
+                {"Custom Visual Name": item.get("name"), "Is Public": False}
+                for item in rp
+                if item.get("type") == "CustomVisual"
+            ]
+
+        # Load public custom visuals
+        public_custom_visuals = (
+            self.get(
+                file_path=self._report_file_path,
+                json_path="$.publicCustomVisuals",
+                verbose=False,
+            )
+            or []
         )
 
-        visual_types = set()
-        for v in self.__all_visuals():
-            payload = v.get("payload", {})
-            visual = payload.get("visual", {})
-            visual_type = visual.get("visualType")
-            if visual_type:
-                visual_types.add(visual_type)
-
-        for _, r in df.iterrows():
-            if r["Custom Visual Name"] in visual_types:
-                df.at[_, "Used in Report"] = True
-            else:
-                df.at[_, "Used in Report"] = False
+        visuals += [
+            {
+                "Custom Visual Name": (
+                    item.get("name") if isinstance(item, dict) else item
+                ),
+                "Is Public": True,
+            }
+            for item in public_custom_visuals
+        ]
 
-        _update_dataframe_datatypes(dataframe=df, column_map=columns)
+        if visuals:
+            df = pd.DataFrame(visuals, columns=list(columns.keys()))
+
+            # df["Custom Visual Name"] = report_file.get("publicCustomVisuals")
+            df["Custom Visual Display Name"] = df["Custom Visual Name"].apply(
+                lambda x: helper.vis_type_mapping.get(x, x)
+            )
+
+            visual_types = set()
+            for v in self.__all_visuals():
+                payload = v.get("payload", {})
+                visual = payload.get("visual", {})
+                visual_type = visual.get("visualType")
+                if visual_type:
+                    visual_types.add(visual_type)
+
+            df["Used in Report"] = df["Custom Visual Name"].isin(visual_types)
+
+            _update_dataframe_datatypes(dataframe=df, column_map=columns)
 
         return df
 
@@ -1886,7 +1921,6 @@ class ReportWrapper:
             )
 
         self._ensure_pbir()
-        theme_version = "5.6.4"
 
         # Extract theme_json from theme_file_path
         if theme_file_path:
@@ -1912,14 +1946,25 @@ class ReportWrapper:
         theme_name_full = f"{theme_name}.json"
 
         # Add theme.json file
-        self.add(
-            file_path=f"StaticResources/RegisteredResources/{theme_name_full}",
-            payload=theme_json,
+        try:
+            self.add(
+                file_path=f"StaticResources/RegisteredResources/{theme_name_full}",
+                payload=theme_json,
+            )
+        except Exception:
+            self.update(
+                file_path=f"StaticResources/RegisteredResources/{theme_name_full}",
+                payload=theme_json,
+            )
+
+        rpt_version_at_import = self.get(
+            file_path=self._report_file_path,
+            json_path="$.themeCollection.baseTheme.reportVersionAtImport",
         )
 
         custom_theme = {
             "name": theme_name_full,
-            "reportVersionAtImport": theme_version,
+            "reportVersionAtImport": rpt_version_at_import,
             "type": "RegisteredResources",
         }
 

sempy_labs/tom/_model.py

@@ -151,7 +151,12 @@ class TOMWrapper:
 
         self._table_map = {}
         self._column_map = {}
-        self._compat_level = self.model.Model.Database.CompatibilityLevel
+        self._compat_level = self.model.Database.CompatibilityLevel
+
+        # Max compat level
+        s = self.model.Server.SupportedCompatibilityLevels
+        nums = [int(x) for x in s.split(",") if x.strip() != "1000000"]
+        self._max_compat_level = max(nums)
 
         # Minimum campat level for lineage tags is 1540 (https://learn.microsoft.com/dotnet/api/microsoft.analysisservices.tabular.table.lineagetag?view=analysisservices-dotnet#microsoft-analysisservices-tabular-table-lineagetag)
         if self._compat_level >= 1540:
@@ -240,6 +245,22 @@ class TOMWrapper:
             if t.CalculationGroup is not None:
                 yield t
 
+    def all_functions(self):
+        """
+        Outputs a list of all user-defined functions in the semantic model.
+
+        Parameters
+        ----------
+
+        Returns
+        -------
+        Iterator[Microsoft.AnalysisServices.Tabular.Function]
+            All user-defined functions within the semantic model.
+        """
+
+        for f in self.model.Functions:
+            yield f
+
     def all_measures(self):
         """
         Outputs a list of all measures in the semantic model.
@@ -759,6 +780,60 @@ class TOMWrapper:
             obj.Description = description
         self.model.Roles.Add(obj)
 
+    def set_compatibility_level(self, compatibility_level: int):
+        """
+        Sets compatibility level of the semantic model
+
+        Parameters
+        ----------
+        compatibility_level : int
+            The compatibility level to set the for the semantic model.
+        """
+        import Microsoft.AnalysisServices.Tabular as TOM
+
+        if compatibility_level < 1500 or compatibility_level > self._max_compat_level:
+            raise ValueError(
+                f"{icons.red_dot} Compatibility level must be between 1500 and {self._max_compat_level}."
+            )
+        if self._compat_level > compatibility_level:
+            print(
+                f"{icons.warning} Compatibility level can only be increased, not decreased."
+            )
+            return
+
+        self.model.Database.CompatibilityLevel = compatibility_level
+        bim = TOM.JsonScripter.ScriptCreateOrReplace(self.model.Database)
+        fabric.execute_tmsl(script=bim, workspace=self._workspace_id)
+
+    def set_user_defined_function(self, name: str, expression: str):
+        """
+        Sets the definition of a `user-defined <https://learn.microsoft.com/en-us/dax/best-practices/dax-user-defined-functions#using-model-explorer>_` function within the semantic model. This function requires that the compatibility level is at least 1702.
+
+        Parameters
+        ----------
+        name : str
+            Name of the user-defined function.
+        expression : str
+            The DAX expression for the user-defined function.
+        """
+        import Microsoft.AnalysisServices.Tabular as TOM
+
+        if self._compat_level < 1702:
+            raise ValueError(
+                f"{icons.warning} User-defined functions require a compatibility level of at least 1702. The current compatibility level is {self._compat_level}. Use the 'tom.set_compatibility_level' function to change the compatibility level."
+            )
+
+        existing = [f.Name for f in self.model.Functions]
+
+        if name in existing:
+            self.model.Functions[name].Expression = expression
+        else:
+            obj = TOM.Function()
+            obj.Name = name
+            obj.Expression = expression
+            obj.LineageTag = generate_guid()
+            self.model.Functions.Add(obj)
+
     def set_rls(self, role_name: str, table_name: str, filter_expression: str):
         """
         Sets the row level security permissions for a table within a role.
@@ -1908,6 +1983,8 @@ class TOMWrapper:
             object.Parent.CalculationItems.Remove(object.Name)
         elif objType == TOM.ObjectType.TablePermission:
             object.Parent.TablePermissions.Remove(object.Name)
+        elif objType == TOM.ObjectType.Function:
+            object.Parent.Functions.Remove(object.Name)
 
     def used_in_relationships(self, object: Union["TOM.Table", "TOM.Column"]):
         """
@@ -4749,8 +4826,8 @@ class TOMWrapper:
         value_filter_behavior = value_filter_behavior.capitalize()
         min_compat = 1606
 
-        if self.model.Model.Database.CompatibilityLevel < min_compat:
-            self.model.Model.Database.CompatibilityLevel = min_compat
+        if self.model.Database.CompatibilityLevel < min_compat:
+            self.model.Database.CompatibilityLevel = min_compat
 
         self.model.ValueFilterBehavior = System.Enum.Parse(
             TOM.ValueFilterBehaviorType, value_filter_behavior
@@ -5840,7 +5917,7 @@ class TOMWrapper:
             import Microsoft.AnalysisServices.Tabular as TOM
 
             # ChangedProperty logic (min compat level is 1567) https://learn.microsoft.com/dotnet/api/microsoft.analysisservices.tabular.changedproperty?view=analysisservices-dotnet
-            if self.model.Model.Database.CompatibilityLevel >= 1567:
+            if self.model.Database.CompatibilityLevel >= 1567:
                 for t in self.model.Tables:
                     if any(
                         p.SourceType == TOM.PartitionSourceType.Entity