semantic-link-labs 0.9.0__py3-none-any.whl → 0.9.2__py3-none-any.whl

sempy_labs/lakehouse/__init__.py

@@ -4,12 +4,14 @@ from sempy_labs.lakehouse._lakehouse import (
     lakehouse_attached,
     optimize_lakehouse_tables,
     vacuum_lakehouse_tables,
+    run_table_maintenance,
 )
 
 from sempy_labs.lakehouse._shortcuts import (
     # create_shortcut,
     create_shortcut_onelake,
     delete_shortcut,
+    reset_shortcut_cache,
 )
 
 __all__ = [
@@ -21,4 +23,6 @@ __all__ = [
     "create_shortcut_onelake",
     "delete_shortcut",
     "vacuum_lakehouse_tables",
+    "reset_shortcut_cache",
+    "run_table_maintenance",
 ]

sempy_labs/lakehouse/_get_lakehouse_columns.py

@@ -4,6 +4,7 @@ from sempy_labs._helper_functions import (
     format_dax_object_name,
     resolve_workspace_name_and_id,
     resolve_lakehouse_name_and_id,
+    _create_dataframe,
 )
 from typing import Optional
 from sempy._utils._log import log
@@ -35,16 +36,15 @@ def get_lakehouse_columns(
     from sempy_labs.lakehouse._get_lakehouse_tables import get_lakehouse_tables
     from delta import DeltaTable
 
-    df = pd.DataFrame(
-        columns=[
-            "Workspace Name",
-            "Lakehouse Name",
-            "Table Name",
-            "Column Name",
-            "Full Column Name",
-            "Data Type",
-        ]
-    )
+    columns = {
+        "Workspace Name": "string",
+        "Lakehouse Name": "string",
+        "Table Name": "string",
+        "Column Name": "string",
+        "Full Column Name": "string",
+        "Data Type": "string",
+    }
+    df = _create_dataframe(columns=columns)
 
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
     (lakehouse_name, lakehouse_id) = resolve_lakehouse_name_and_id(

sempy_labs/lakehouse/_get_lakehouse_tables.py

@@ -7,8 +7,9 @@ from sempy_labs._helper_functions import (
     _get_column_aggregate,
     resolve_workspace_name_and_id,
     resolve_lakehouse_name_and_id,
-    pagination,
     save_as_delta_table,
+    _base_api,
+    _create_dataframe,
 )
 from sempy_labs.directlake._guardrails import (
     get_sku_size,
@@ -18,7 +19,6 @@ from sempy_labs.lakehouse._lakehouse import lakehouse_attached
 from typing import Optional
 import sempy_labs._icons as icons
 from sempy._utils._log import log
-from sempy.fabric.exceptions import FabricHTTPException
 from uuid import UUID
 
 
@@ -57,16 +57,15 @@ def get_lakehouse_tables(
         Shows the tables/columns within a lakehouse and their properties.
     """
 
-    df = pd.DataFrame(
-        columns=[
-            "Workspace Name",
-            "Lakehouse Name",
-            "Table Name",
-            "Format",
-            "Type",
-            "Location",
-        ]
-    )
+    columns = {
+        "Workspace Name": "string",
+        "Lakehouse Name": "string",
+        "Table Name": "string",
+        "Format": "string",
+        "Type": "string",
+        "Location": "string",
+    }
+    df = _create_dataframe(columns=columns)
 
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
     (lakehouse_name, lakehouse_id) = resolve_lakehouse_name_and_id(
@@ -86,16 +85,11 @@ def get_lakehouse_tables(
             "Count rows runs a spark query and cross-workspace spark queries are currently not supported."
         )
 
-    client = fabric.FabricRestClient()
-    response = client.get(
-        f"/v1/workspaces/{workspace_id}/lakehouses/{lakehouse_id}/tables"
+    responses = _base_api(
+        request=f"v1/workspaces/{workspace_id}/lakehouses/{lakehouse_id}/tables",
+        uses_pagination=True,
     )
 
-    if response.status_code != 200:
-        raise FabricHTTPException(response)
-
-    responses = pagination(client, response)
-
     if not responses[0].get("data"):
         return df
 

sempy_labs/lakehouse/_lakehouse.py

@@ -2,6 +2,13 @@ from tqdm.auto import tqdm
 from typing import List, Optional, Union
 from sempy._utils._log import log
 from uuid import UUID
+from sempy_labs._helper_functions import (
+    _base_api,
+    resolve_lakehouse_name_and_id,
+    resolve_workspace_name_and_id,
+)
+import sempy_labs._icons as icons
+import re
 
 
 def lakehouse_attached() -> bool:
@@ -13,12 +20,12 @@ def lakehouse_attached() -> bool:
     bool
         Returns True if a lakehouse is attached to the notebook.
     """
-    from pyspark.sql import SparkSession
 
-    spark = SparkSession.builder.getOrCreate()
-    lakeId = spark.conf.get("trident.lakehouse.id")
+    from sempy_labs._helper_functions import _get_fabric_context_setting
+
+    lake_id = _get_fabric_context_setting(name="trident.lakehouse.id")
 
-    if len(lakeId) > 0:
+    if len(lake_id) > 0:
         return True
     else:
         return False
@@ -128,3 +135,93 @@ def vacuum_lakehouse_tables(
             deltaTable.vacuum()
         else:
             deltaTable.vacuum(retain_n_hours)
+
+
+def run_table_maintenance(
+    table_name: str,
+    optimize: bool = False,
+    v_order: bool = False,
+    vacuum: bool = False,
+    retention_period: Optional[str] = None,
+    schema: Optional[str] = None,
+    lakehouse: Optional[str | UUID] = None,
+    workspace: Optional[str | UUID] = None,
+):
+    """
+    Runs table maintenance operations on the specified table within the lakehouse.
+
+    This is a wrapper function for the following API: `Background Jobs - Run On Demand Table Maintenance <https://learn.microsoft.com/rest/api/fabric/lakehouse/background-jobs/run-on-demand-table-maintenance>`_.
+
+    Parameters
+    ----------
+    table_name : str
+        Name of the delta table on which to run maintenance operations.
+    optimize : bool, default=False
+        If True, the `OPTIMIZE <https://docs.delta.io/latest/optimizations-oss.html>`_ function will be run on the table.
+    v_order : bool, default=False
+        If True, v-order will be enabled for the table.
+    vacuum : bool, default=False
+        If True, the `VACUUM <https://docs.delta.io/latest/delta-utility.html#remove-files-no-longer-referenced-by-a-delta-table>`_ function will be run on the table.
+    retention_period : str, default=None
+        If specified, the retention period for the vacuum operation. Must be in the 'd:hh:mm:ss' format.
+    schema : str, default=None
+        The schema of the tables within the lakehouse.
+    lakehouse : str | uuid.UUID, default=None
+        The Fabric lakehouse name or ID.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID used by the lakehouse.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    (lakehouse_name, lakehouse_id) = resolve_lakehouse_name_and_id(
+        lakehouse=lakehouse, workspace=workspace_id
+    )
+
+    if not optimize and not vacuum:
+        raise ValueError(
+            f"{icons.warning} At least one of 'optimize' or 'vacuum' must be set to True."
+        )
+    if not vacuum and retention_period is not None:
+        raise ValueError(
+            f"{icons.warning} The 'retention_period' parameter can only be set if 'vacuum' is set to True."
+        )
+    if retention_period is not None:
+
+        def is_valid_format(time_string):
+            pattern = r"^\d+:[0-2][0-9]:[0-5][0-9]:[0-5][0-9]$"
+            return bool(re.match(pattern, time_string))
+
+        if not is_valid_format(retention_period):
+            raise ValueError(
+                f"{icons.red_dot} The 'retention_period' parameter must be in the 'd:hh:mm:ss' format."
+            )
+
+    payload = {
+        "executionData": {
+            "tableName": table_name,
+        }
+    }
+    if schema is not None:
+        payload["executionData"]["schemaName"] = schema
+    if optimize:
+        payload["executionData"]["optimizeSettings"] = {}
+    if v_order:
+        payload["executionData"]["optimizeSettings"] = {"vorder": True}
+    if vacuum:
+        payload["executionData"]["vacuumSettings"] = {}
+    if vacuum and retention_period is not None:
+        payload["executionData"]["vacuumSettings"]["retentionPeriod"] = retention_period
+
+    _base_api(
+        request=f"/v1/workspaces/{workspace_id}/lakehouses/{lakehouse_id}/jobs/instances?jobType=TableMaintenance",
+        method="post",
+        payload=payload,
+        status_codes=202,
+    )
+
+    print(
+        f"{icons.green_dot} The table maintenance job for the '{table_name}' table in the '{lakehouse_name}' lakehouse within the '{workspace_name}' workspace has been initiated."
+    )

sempy_labs/lakehouse/_shortcuts.py

@@ -3,6 +3,7 @@ from sempy_labs._helper_functions import (
     resolve_lakehouse_name,
     resolve_lakehouse_id,
     resolve_workspace_name_and_id,
+    _base_api,
 )
 from typing import Optional
 import sempy_labs._icons as icons
@@ -71,37 +72,30 @@ def create_shortcut_onelake(
     if shortcut_name is None:
         shortcut_name = table_name
 
-    client = fabric.FabricRestClient()
-    tablePath = f"Tables/{table_name}"
+    table_path = f"Tables/{table_name}"
 
-    request_body = {
+    payload = {
         "path": "Tables",
         "name": shortcut_name.replace(" ", ""),
         "target": {
             "oneLake": {
                 "workspaceId": source_workspace_id,
                 "itemId": source_lakehouse_id,
-                "path": tablePath,
+                "path": table_path,
             }
         },
     }
 
-    try:
-        response = client.post(
-            f"/v1/workspaces/{destination_workspace_id}/items/{destination_lakehouse_id}/shortcuts",
-            json=request_body,
-        )
-        if response.status_code == 201:
-            print(
-                f"{icons.green_dot} The shortcut '{shortcut_name}' was created in the '{destination_lakehouse_name}' lakehouse within"
-                f" the '{destination_workspace_name} workspace. It is based on the '{table_name}' table in the '{source_lakehouse_name}' lakehouse within the '{source_workspace_name}' workspace."
-            )
-        else:
-            print(response.status_code)
-    except Exception as e:
-        raise ValueError(
-            f"{icons.red_dot} Failed to create a shortcut for the '{table_name}' table."
-        ) from e
+    _base_api(
+        request=f"/v1/workspaces/{destination_workspace_id}/items/{destination_lakehouse_id}/shortcuts",
+        payload=payload,
+        status_codes=201,
+        method="post",
+    )
+
+    print(
+        f"{icons.green_dot} The shortcut '{shortcut_name}' was created in the '{destination_lakehouse_name}' lakehouse within the '{destination_workspace_name} workspace. It is based on the '{table_name}' table in the '{source_lakehouse_name}' lakehouse within the '{source_workspace_name}' workspace."
+    )
 
 
 def create_shortcut(
@@ -223,3 +217,31 @@ def delete_shortcut(
     print(
         f"{icons.green_dot} The '{shortcut_name}' shortcut in the '{lakehouse}' within the '{workspace_name}' workspace has been deleted."
     )
+
+
+def reset_shortcut_cache(workspace: Optional[str | UUID]):
+    """
+    Deletes any cached files that were stored while reading from shortcuts.
+
+    This is a wrapper function for the following API: `OneLake Shortcuts - Reset Shortcut Cache <https://learn.microsoft.com/rest/api/fabric/core/onelake-shortcuts/reset-shortcut-cache>`_.
+
+    Parameters
+    ----------
+    workspace : str | uuid.UUID, default=None
+        The name or ID of the Fabric workspace.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+
+    _base_api(
+        request=f"/v1/workspaces/{workspace_id}/onelake/resetShortcutCache",
+        method="post",
+        lro_return_status_code=True,
+        status_codes=None,
+    )
+
+    print(
+        f"{icons.green_dot} The shortcut cache has been reset for the '{workspace_name}' workspace."
+    )

sempy_labs/migration/__init__.py

@@ -18,6 +18,9 @@ from sempy_labs.migration._migration_validation import (
 from sempy_labs.migration._refresh_calc_tables import (
     refresh_calc_tables,
 )
+from sempy_labs.migration._direct_lake_to_import import (
+    migrate_direct_lake_to_import,
+)
 
 __all__ = [
     "create_pqt_file",
@@ -28,4 +31,5 @@ __all__ = [
     "migrate_tables_columns_to_semantic_model",
     "migration_validation",
     "refresh_calc_tables",
+    "migrate_direct_lake_to_import",
 ]

sempy_labs/migration/_direct_lake_to_import.py

@@ -0,0 +1,66 @@
+import sempy
+from uuid import UUID
+import sempy_labs._icons as icons
+
+
+def migrate_direct_lake_to_import(dataset: str | UUID, workspace: str | UUID):
+    """
+    Migrates a semantic model from Direct Lake mode to import mode. After running this function, you must go to the semantic model settings and update the cloud connection. Not doing so will result in an inablity to refresh/use the semantic model.
+
+    Parameters
+    ----------
+    dataset : str | uuid.UUID
+        Name or ID of the semantic model.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    sempy.fabric._client._utils._init_analysis_services()
+    import Microsoft.AnalysisServices.Tabular as TOM
+    from sempy_labs.tom import connect_semantic_model
+
+    with connect_semantic_model(
+        dataset=dataset, workspace=workspace, readonly=False
+    ) as tom:
+
+        if not tom.is_direct_lake():
+            print(
+                f"{icons.warning} The '{dataset}' semantic model within the '{workspace}' workspace is not in Direct Lake mode."
+            )
+            return
+
+        for t in tom.model.Tables:
+            table_name = t.Name
+            if t.Partitions.Count == 1 and all(
+                p.Mode == TOM.ModeType.DirectLake for p in t.Partitions
+            ):
+                p = next(p for p in t.Partitions)
+                partition_name = p.Name
+                entity_name = p.Source.EntityName
+                schema_name = p.Source.SchemaName or "dbo"
+                # Rename Direct Lake partition
+                t.Partitions[partition_name].Name = f"{partition_name}_remove"
+
+                # Generate M expression for import partition
+                expression = f"""let\n\tSource = DatabaseQuery,\n\tData = Source{{[Schema="{schema_name}",Item="{entity_name}"]}}[Data]\nin\n\tData"""
+
+                # Generate M partition
+                tom.add_m_partition(
+                    table_name=table_name,
+                    partition_name=partition_name,
+                    expression=expression,
+                    mode="Import",
+                )
+                # Remove Direct Lake partition
+                tom.remove_object(object=p)
+
+        tom.model.Model.DefaultMode = TOM.ModeType.Import
+
+        # Check
+        # for t in tom.model.Tables:
+        #    if t.Partitions.Count == 1 and all(p.Mode == TOM.ModeType.Import for p in t.Partitions) and t.CalculationGroup is None:
+        #        p = next(p for p in t.Partitions)
+        #        print(p.Name)
+        #        print(p.Source.Expression)

sempy_labs/migration/_migrate_calctables_to_lakehouse.py

@@ -283,6 +283,7 @@ def migrate_field_parameters(
         new_dataset_workspace = workspace
 
     icons.sll_tags.append("DirectLakeMigration")
+    fabric.refresh_tom_cache(workspace=workspace)
 
     dfC = fabric.list_columns(dataset=dataset, workspace=workspace)
     dfC["Column Object"] = format_dax_object_name(dfC["Table Name"], dfC["Column Name"])

sempy_labs/migration/_migrate_calctables_to_semantic_model.py

@@ -54,6 +54,7 @@ def migrate_calc_tables_to_semantic_model(
         )
 
     workspace = fabric.resolve_workspace_name(workspace)
+    fabric.refresh_tom_cache(workspace=workspace)
 
     if new_dataset_workspace is None:
         new_dataset_workspace = workspace

sempy_labs/migration/_migrate_model_objects_to_semantic_model.py

@@ -48,6 +48,7 @@ def migrate_model_objects_to_semantic_model(
         )
 
     workspace = fabric.resolve_workspace_name(workspace)
+    fabric.refresh_tom_cache(workspace=workspace)
 
     if new_dataset_workspace is None:
         new_dataset_workspace = workspace

sempy_labs/migration/_migrate_tables_columns_to_semantic_model.py

@@ -75,6 +75,8 @@ def migrate_tables_columns_to_semantic_model(
         item_name=lakehouse, item_type="Lakehouse", workspace=lakehouse_workspace
     )
 
+    fabric.refresh_tom_cache(workspace=workspace)
+
     dfC = fabric.list_columns(dataset=dataset, workspace=workspace)
     dfT = fabric.list_tables(dataset=dataset, workspace=workspace)
     dfT.rename(columns={"Type": "Table Type"}, inplace=True)

sempy_labs/report/_download_report.py

@@ -4,14 +4,15 @@ from typing import Optional
 from sempy_labs._helper_functions import (
     resolve_workspace_name_and_id,
     resolve_lakehouse_name,
+    _base_api,
+    resolve_item_id,
 )
 from sempy_labs.lakehouse._lakehouse import lakehouse_attached
-from sempy.fabric.exceptions import FabricHTTPException
 from uuid import UUID
 
 
 def download_report(
-    report: str,
+    report: str | UUID,
     file_name: Optional[str] = None,
     download_type: str = "LiveConnect",
     workspace: Optional[str | UUID] = None,
@@ -23,8 +24,8 @@ def download_report(
 
     Parameters
     ----------
-    report: str
-        Name of the report.
+    report: str | uuid.UUID
+        Name or ID of the report.
     file_name : str, default=None
         Name of the .pbix file to be saved.
         Defaults to None which resolves to the name of the report.
@@ -55,18 +56,12 @@ def download_report(
         )
 
     file_name = file_name or report
-    report_id = fabric.resolve_item_id(
-        item_name=report, type="Report", workspace=workspace_id
-    )
+    report_id = resolve_item_id(item=report, type="Report", workspace=workspace)
 
-    client = fabric.PowerBIRestClient()
-    response = client.get(
-        f"/v1.0/myorg/groups/{workspace_id}/reports/{report_id}/Export?downloadType={download_type}"
+    response = _base_api(
+        request=f"v1.0/myorg/groups/{workspace_id}/reports/{report_id}/Export?downloadType={download_type}"
     )
 
-    if response.status_code != 200:
-        raise FabricHTTPException(response)
-
     # Save file to the attached lakehouse
     with open(f"/lakehouse/default/Files/{file_name}.pbix", "wb") as file:
         file.write(response.content)