semantic-link-labs 0.12.2__py3-none-any.whl → 0.12.4__py3-none-any.whl

sempy_labs/lakehouse/_partitioning.py

@@ -0,0 +1,165 @@
+from typing import Optional, List
+from uuid import UUID
+from sempy_labs._helper_functions import (
+    _create_spark_session,
+    create_abfss_path,
+    resolve_workspace_id,
+    resolve_lakehouse_id,
+    _get_delta_table,
+)
+from sempy._utils._log import log
+
+
+@log
+def _get_partitions(
+    table_name: str,
+    schema_name: Optional[str] = None,
+    lakehouse: Optional[str | UUID] = None,
+    workspace: Optional[str | UUID] = None,
+):
+
+    workspace_id = resolve_workspace_id(workspace)
+    lakehouse_id = resolve_lakehouse_id(lakehouse, workspace)
+    path = create_abfss_path(lakehouse_id, workspace_id, table_name, schema_name)
+
+    delta_table = _get_delta_table(path)
+    details_df = delta_table.detail()
+
+    return details_df.collect()[0].asDict()
+
+
+@log
+def is_partitioned(
+    table: str,
+    schema: Optional[str] = None,
+    lakehouse: Optional[str | UUID] = None,
+    workspace: Optional[str | UUID] = None,
+) -> bool:
+    """
+    Checks if a delta table is partitioned.
+
+    Parameters
+    ----------
+    table : str
+        The name of the delta table.
+    schema : str, optional
+        The schema of the table to check. If not provided, the default schema is used.
+    lakehouse : str | uuid.UUID, default=None
+        The Fabric lakehouse name or ID.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID used by the lakehouse.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    bool
+        True if the table is partitioned, False otherwise.
+    """
+
+    details = _get_partitions(
+        table_name=table, schema_name=schema, lakehouse=lakehouse, workspace=workspace
+    )
+    return len(details["partitionColumns"]) > 0
+
+
+@log
+def list_partitioned_columns(
+    table: str,
+    schema: Optional[str] = None,
+    lakehouse: Optional[str | UUID] = None,
+    workspace: Optional[str | UUID] = None,
+) -> List[str]:
+    """
+    Lists the partitioned columns of a delta table.
+
+    Parameters
+    ----------
+    table : str
+        The name of the delta table.
+    schema : str, optional
+        The schema of the table to check. If not provided, the default schema is used.
+    lakehouse : str | uuid.UUID, default=None
+        The Fabric lakehouse name or ID.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID used by the lakehouse.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    List[str]
+        The list of partitioned columns.
+    """
+
+    details = _get_partitions(
+        table_name=table, schema_name=schema, lakehouse=lakehouse, workspace=workspace
+    )
+
+    return details["partitionColumns"]
+
+
+@log
+def is_over_partitioned(
+    table: str,
+    schema: Optional[str] = None,
+    lakehouse: Optional[str | UUID] = None,
+    workspace: Optional[str | UUID] = None,
+    total_table_size_gb: int = 1000,
+    average_partition_size_gb: int = 1,
+) -> bool:
+    """
+    Checks if a delta table is over-partitioned.
+
+    Parameters
+    ----------
+    table : str
+        The name of the delta table.
+    schema : str, optional
+        The schema of the table to check. If not provided, the default schema is used.
+    lakehouse : str | uuid.UUID, default=None
+        The Fabric lakehouse name or ID.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID used by the lakehouse.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    total_table_size_gb : int, default=1000
+        Threshold for total table size in GB (default 1TB).
+    average_partition_size_gb : int, default=1
+        Threshold for average partition size in GB.
+
+    Returns
+    -------
+    bool
+        True if the table is over-partitioned, False otherwise.
+    """
+
+    workspace_id = resolve_workspace_id(workspace)
+    lakehouse_id = resolve_lakehouse_id(lakehouse, workspace)
+    path = create_abfss_path(lakehouse_id, workspace_id, table, schema)
+    # Get DeltaTable details
+    spark = _create_spark_session()
+    details_df = spark.sql(f"DESCRIBE DETAIL delta.`{path}`")
+    details = details_df.collect()[0].asDict()
+
+    # Extract relevant fields
+    size_bytes = details["sizeInBytes"]
+    partition_cols = details["partitionColumns"]
+    num_files = details["numFiles"]
+
+    total_size_gb = size_bytes / (1024**3)
+
+    # Only check if the table is partitioned
+    if len(partition_cols) > 0 and num_files > 0:
+        avg_partition_size_gb = total_size_gb / num_files
+
+        if (
+            total_size_gb < total_table_size_gb
+            or avg_partition_size_gb < average_partition_size_gb
+        ):
+            return True
+
+    return False

sempy_labs/report/_reportwrapper.py

@@ -1886,7 +1886,6 @@ class ReportWrapper:
             )
 
         self._ensure_pbir()
-        theme_version = "5.6.4"
 
         # Extract theme_json from theme_file_path
         if theme_file_path:
@@ -1912,14 +1911,25 @@ class ReportWrapper:
         theme_name_full = f"{theme_name}.json"
 
         # Add theme.json file
-        self.add(
-            file_path=f"StaticResources/RegisteredResources/{theme_name_full}",
-            payload=theme_json,
+        try:
+            self.add(
+                file_path=f"StaticResources/RegisteredResources/{theme_name_full}",
+                payload=theme_json,
+            )
+        except Exception:
+            self.update(
+                file_path=f"StaticResources/RegisteredResources/{theme_name_full}",
+                payload=theme_json,
+            )
+
+        rpt_version_at_import = self.get(
+            file_path=self._report_file_path,
+            json_path="$.themeCollection.baseTheme.reportVersionAtImport",
         )
 
         custom_theme = {
             "name": theme_name_full,
-            "reportVersionAtImport": theme_version,
+            "reportVersionAtImport": rpt_version_at_import,
             "type": "RegisteredResources",
         }
 

sempy_labs/tom/_model.py

@@ -12,7 +12,6 @@ from sempy_labs._helper_functions import (
     _make_list_unique,
     resolve_dataset_name_and_id,
     resolve_workspace_name_and_id,
-    _base_api,
     resolve_workspace_id,
     resolve_item_id,
     resolve_lakehouse_id,
@@ -152,7 +151,12 @@ class TOMWrapper:
 
         self._table_map = {}
         self._column_map = {}
-        self._compat_level = self.model.Model.Database.CompatibilityLevel
+        self._compat_level = self.model.Database.CompatibilityLevel
+
+        # Max compat level
+        s = self.model.Server.SupportedCompatibilityLevels
+        nums = [int(x) for x in s.split(",") if x.strip() != "1000000"]
+        self._max_compat_level = max(nums)
 
         # Minimum campat level for lineage tags is 1540 (https://learn.microsoft.com/dotnet/api/microsoft.analysisservices.tabular.table.lineagetag?view=analysisservices-dotnet#microsoft-analysisservices-tabular-table-lineagetag)
         if self._compat_level >= 1540:
@@ -241,6 +245,22 @@ class TOMWrapper:
             if t.CalculationGroup is not None:
                 yield t
 
+    def all_functions(self):
+        """
+        Outputs a list of all user-defined functions in the semantic model.
+
+        Parameters
+        ----------
+
+        Returns
+        -------
+        Iterator[Microsoft.AnalysisServices.Tabular.Function]
+            All user-defined functions within the semantic model.
+        """
+
+        for f in self.model.Functions:
+            yield f
+
     def all_measures(self):
         """
         Outputs a list of all measures in the semantic model.
@@ -760,6 +780,60 @@ class TOMWrapper:
             obj.Description = description
         self.model.Roles.Add(obj)
 
+    def set_compatibility_level(self, compatibility_level: int):
+        """
+        Sets compatibility level of the semantic model
+
+        Parameters
+        ----------
+        compatibility_level : int
+            The compatibility level to set the for the semantic model.
+        """
+        import Microsoft.AnalysisServices.Tabular as TOM
+
+        if compatibility_level < 1500 or compatibility_level > self._max_compat_level:
+            raise ValueError(
+                f"{icons.red_dot} Compatibility level must be between 1500 and {self._max_compat_level}."
+            )
+        if self._compat_level > compatibility_level:
+            print(
+                f"{icons.warning} Compatibility level can only be increased, not decreased."
+            )
+            return
+
+        self.model.Database.CompatibilityLevel = compatibility_level
+        bim = TOM.JsonScripter.ScriptCreateOrReplace(self.model.Database)
+        fabric.execute_tmsl(script=bim, workspace=self._workspace_id)
+
+    def set_user_defined_function(self, name: str, expression: str):
+        """
+        Sets the definition of a `user-defined <https://learn.microsoft.com/en-us/dax/best-practices/dax-user-defined-functions#using-model-explorer>_` function within the semantic model. This function requires that the compatibility level is at least 1702.
+
+        Parameters
+        ----------
+        name : str
+            Name of the user-defined function.
+        expression : str
+            The DAX expression for the user-defined function.
+        """
+        import Microsoft.AnalysisServices.Tabular as TOM
+
+        if self._compat_level < 1702:
+            raise ValueError(
+                f"{icons.warning} User-defined functions require a compatibility level of at least 1702. The current compatibility level is {self._compat_level}. Use the 'tom.set_compatibility_level' function to change the compatibility level."
+            )
+
+        existing = [f.Name for f in self.model.Functions]
+
+        if name in existing:
+            self.model.Functions[name].Expression = expression
+        else:
+            obj = TOM.Function()
+            obj.Name = name
+            obj.Expression = expression
+            obj.LineageTag = generate_guid()
+            self.model.Functions.Add(obj)
+
     def set_rls(self, role_name: str, table_name: str, filter_expression: str):
         """
         Sets the row level security permissions for a table within a role.
@@ -1909,6 +1983,8 @@ class TOMWrapper:
             object.Parent.CalculationItems.Remove(object.Name)
         elif objType == TOM.ObjectType.TablePermission:
             object.Parent.TablePermissions.Remove(object.Name)
+        elif objType == TOM.ObjectType.Function:
+            object.Parent.Functions.Remove(object.Name)
 
     def used_in_relationships(self, object: Union["TOM.Table", "TOM.Column"]):
         """
@@ -4670,9 +4746,7 @@ class TOMWrapper:
                 json=payload,
             )
             if response.status_code != 200:
-                raise FabricHTTPException(
-                    f"Failed to retrieve descriptions: {response.text}"
-                )
+                raise FabricHTTPException(response)
 
             for item in response.json().get("modelItems", []):
                 ms_name = item["urn"]
@@ -4752,8 +4826,8 @@ class TOMWrapper:
         value_filter_behavior = value_filter_behavior.capitalize()
         min_compat = 1606
 
-        if self.model.Model.Database.CompatibilityLevel < min_compat:
-            self.model.Model.Database.CompatibilityLevel = min_compat
+        if self.model.Database.CompatibilityLevel < min_compat:
+            self.model.Database.CompatibilityLevel = min_compat
 
         self.model.ValueFilterBehavior = System.Enum.Parse(
             TOM.ValueFilterBehaviorType, value_filter_behavior
@@ -5109,7 +5183,9 @@ class TOMWrapper:
         """
         import Microsoft.AnalysisServices.Tabular as TOM
 
-        p = next(p for p in self.model.Tables[table_name].Partitions)
+        t = self.model.Tables[table_name]
+
+        p = next(p for p in t.Partitions)
         if p.Mode != TOM.ModeType.DirectLake:
             print(f"{icons.info} The '{table_name}' table is not in Direct Lake mode.")
             return
@@ -5119,9 +5195,7 @@ class TOMWrapper:
         partition_schema = schema or p.Source.SchemaName
 
         # Update name of the Direct Lake partition (will be removed later)
-        self.model.Tables[table_name].Partitions[
-            partition_name
-        ].Name = f"{partition_name}_remove"
+        t.Partitions[partition_name].Name = f"{partition_name}_remove"
 
         source_workspace_id = resolve_workspace_id(workspace=source_workspace)
         if source_type == "Lakehouse":
@@ -5133,21 +5207,41 @@ class TOMWrapper:
                 item=source, type=source_type, workspace=source_workspace_id
             )
 
+        column_pairs = []
+        m_filter = None
+        for c in t.Columns:
+            if c.Type == TOM.ColumnType.Data:
+                if c.Name != c.SourceColumn:
+                    column_pairs.append((c.SourceColumn, c.Name))
+
+        if column_pairs:
+            m_filter = (
+                f'#"Renamed Columns" = Table.RenameColumns(ToDelta, {{'
+                + ", ".join([f'{{"{old}", "{new}"}}' for old, new in column_pairs])
+                + "})"
+            )
+
         def _generate_m_expression(
-            workspace_id, artifact_id, artifact_type, table_name, schema_name
+            workspace_id, artifact_id, artifact_type, table_name, schema_name, m_filter
         ):
             """
-            Generates the M expression for the import partition.
+            Generates the M expression for the import partition. Adds a rename step if any columns have been renamed in the model.
             """
 
             full_table_name = (
                 f"{schema_name}/{table_name}" if schema_name else table_name
             )
 
-            return f"""let\n\tSource = AzureStorage.DataLake("https://onelake.dfs.fabric.microsoft.com/{workspace_id}/{artifact_id}", [HierarchicalNavigation=true]),
+            code = f"""let\n\tSource = AzureStorage.DataLake("https://onelake.dfs.fabric.microsoft.com/{workspace_id}/{artifact_id}", [HierarchicalNavigation=true]),
         Tables = Source{{[Name = "Tables"]}}[Content],
         ExpressionTable = Tables{{[Name = "{full_table_name}"]}}[Content],
-        ToDelta = DeltaLake.Table(ExpressionTable)\nin\n\tToDelta"""
+        ToDelta = DeltaLake.Table(ExpressionTable)"""
+            if m_filter is None:
+                code += "\n in\n\tToDelta"
+            else:
+                code += f',\n\t {m_filter} \n in\n\t#"Renamed Columns"'
+
+            return code
 
         m_expression = _generate_m_expression(
             source_workspace_id,
@@ -5155,6 +5249,7 @@ class TOMWrapper:
             source_type,
             partition_entity_name,
             partition_schema,
+            m_filter,
         )
 
         # Add the import partition
@@ -5822,7 +5917,7 @@ class TOMWrapper:
             import Microsoft.AnalysisServices.Tabular as TOM
 
             # ChangedProperty logic (min compat level is 1567) https://learn.microsoft.com/dotnet/api/microsoft.analysisservices.tabular.changedproperty?view=analysisservices-dotnet
-            if self.model.Model.Database.CompatibilityLevel >= 1567:
+            if self.model.Database.CompatibilityLevel >= 1567:
                 for t in self.model.Tables:
                     if any(
                         p.SourceType == TOM.PartitionSourceType.Entity

sempy_labs/_eventstreams.py

@@ -1,123 +0,0 @@
-import pandas as pd
-from typing import Optional
-from sempy_labs._helper_functions import (
-    _base_api,
-    delete_item,
-    _create_dataframe,
-    create_item,
-    resolve_workspace_id,
-)
-from uuid import UUID
-import sempy_labs._icons as icons
-from sempy._utils._log import log
-
-
-@log
-def list_eventstreams(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
-    """
-    Shows the eventstreams within a workspace.
-
-    This is a wrapper function for the following API: `Items - List Eventstreams <https://learn.microsoft.com/rest/api/fabric/environment/items/list-eventstreams>`_.
-
-    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
-
-    Parameters
-    ----------
-    workspace : str | uuid.UUID, default=None
-        The Fabric workspace name or ID.
-        Defaults to None which resolves to the workspace of the attached lakehouse
-        or if no lakehouse attached, resolves to the workspace of the notebook.
-
-    Returns
-    -------
-    pandas.DataFrame
-        A pandas dataframe showing the eventstreams within a workspace.
-    """
-
-    columns = {
-        "Eventstream Name": "string",
-        "Eventstream Id": "string",
-        "Description": "string",
-    }
-    df = _create_dataframe(columns=columns)
-
-    workspace_id = resolve_workspace_id(workspace)
-    responses = _base_api(
-        request=f"/v1/workspaces/{workspace_id}/eventstreams",
-        uses_pagination=True,
-        client="fabric_sp",
-    )
-
-    rows = []
-    for r in responses:
-        for v in r.get("value", []):
-            rows.append(
-                {
-                    "Eventstream Name": v.get("displayName"),
-                    "Eventstream Id": v.get("id"),
-                    "Description": v.get("description"),
-                }
-            )
-
-    if rows:
-        df = pd.DataFrame(rows, columns=list(columns.keys()))
-
-    return df
-
-
-@log
-def create_eventstream(
-    name: str, description: Optional[str] = None, workspace: Optional[str | UUID] = None
-):
-    """
-    Creates a Fabric eventstream.
-
-    This is a wrapper function for the following API: `Items - Create Eventstream <https://learn.microsoft.com/rest/api/fabric/environment/items/create-eventstream>`_.
-
-    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
-
-    Parameters
-    ----------
-    name: str
-        Name of the eventstream.
-    description : str, default=None
-        A description of the environment.
-    workspace : str | uuid.UUID, default=None
-        The Fabric workspace name or ID.
-        Defaults to None which resolves to the workspace of the attached lakehouse
-        or if no lakehouse attached, resolves to the workspace of the notebook.
-    """
-
-    create_item(
-        name=name, description=description, type="Eventstream", workspace=workspace
-    )
-
-
-@log
-def delete_eventstream(
-    eventstream: str | UUID, workspace: Optional[str | UUID] = None, **kwargs
-):
-    """
-    Deletes a Fabric eventstream.
-
-    This is a wrapper function for the following API: `Items - Delete Eventstream <https://learn.microsoft.com/rest/api/fabric/environment/items/delete-eventstream>`_.
-
-    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
-
-    Parameters
-    ----------
-    eventstream: str | uuid.UUID
-        Name or ID of the eventstream.
-    workspace : str | uuid.UUID, default=None
-        The Fabric workspace name or ID.
-        Defaults to None which resolves to the workspace of the attached lakehouse
-        or if no lakehouse attached, resolves to the workspace of the notebook.
-    """
-
-    if "name" in kwargs:
-        eventstream = kwargs["name"]
-        print(
-            f"{icons.warning} The 'name' parameter is deprecated. Please use 'eventstream' instead."
-        )
-
-    delete_item(item=eventstream, type="Eventstream", workspace=workspace)