semantic-link-labs 0.8.11__py3-none-any.whl → 0.9.0__py3-none-any.whl

sempy_labs/_capacity_migration.py

@@ -138,14 +138,11 @@ def migrate_workspaces(
 @log
 def migrate_capacities(
     azure_subscription_id: str,
-    key_vault_uri: str,
-    key_vault_tenant_id: str,
-    key_vault_client_id: str,
-    key_vault_client_secret: str,
     resource_group: str | dict,
     capacities: Optional[str | List[str]] = None,
     use_existing_rg_for_A_sku: bool = True,
     p_sku_only: bool = True,
+    **kwargs,
 ):
     """
     This function creates new Fabric capacities for given A or P sku capacities and reassigns their workspaces to the newly created capacity.
@@ -154,14 +151,6 @@ def migrate_capacities(
     ----------
     azure_subscription_id : str
         The Azure subscription ID.
-    key_vault_uri : str
-        The name of the `Azure key vault <https://azure.microsoft.com/products/key-vault>`_ URI. Example: "https://<Key Vault Name>.vault.azure.net/"
-    key_vault_tenant_id : str
-        The name of the Azure key vault secret storing the Tenant ID.
-    key_vault_client_id : str
-        The name of the Azure key vault secret storing the Client ID.
-    key_vault_client_secret : str
-        The name of the Azure key vault secret storing the Client Secret.
     resource_group : str | dict
         The name of the Azure resource group.
         For A skus, this parameter will be ignored and the resource group used for the F sku will be the same as the A sku's resource group.
@@ -233,10 +222,6 @@ def migrate_capacities(
                 create_fabric_capacity(
                     capacity_name=tgt_capacity,
                     azure_subscription_id=azure_subscription_id,
-                    key_vault_uri=key_vault_uri,
-                    key_vault_tenant_id=key_vault_tenant_id,
-                    key_vault_client_id=key_vault_client_id,
-                    key_vault_client_secret=key_vault_client_secret,
                     resource_group=rg,
                     region=region,
                     sku=icons.sku_mapping.get(sku_size),
@@ -248,7 +233,7 @@ def migrate_capacities(
             )
 
             # Migrate settings to new capacity
-            migrate_settings(source_capacity=cap_name, target_capacity=tgt_capacity)
+            # migrate_settings(source_capacity=cap_name, target_capacity=tgt_capacity)
 
 
 @log
@@ -624,15 +609,12 @@ def migrate_spark_settings(source_capacity: str, target_capacity: str):
 @log
 def migrate_fabric_trial_capacity(
     azure_subscription_id: str,
-    key_vault_uri: str,
-    key_vault_tenant_id: str,
-    key_vault_client_id: str,
-    key_vault_client_secret: str,
     resource_group: str,
     source_capacity: str,
     target_capacity: str,
     target_capacity_sku: str = "F64",
     target_capacity_admin_members: Optional[str | List[str]] = None,
+    **kwargs,
 ):
     """
     This function migrates a Fabric trial capacity to a Fabric capacity. If the 'target_capacity' does not exist, it is created with the relevant target capacity parameters (sku, region, admin members).
@@ -641,14 +623,6 @@ def migrate_fabric_trial_capacity(
     ----------
     azure_subscription_id : str
         The Azure subscription ID.
-    key_vault_uri : str
-        The name of the `Azure key vault <https://azure.microsoft.com/products/key-vault>`_ URI. Example: "https://<Key Vault Name>.vault.azure.net/"
-    key_vault_tenant_id : str
-        The name of the Azure key vault secret storing the Tenant ID.
-    key_vault_client_id : str
-        The name of the Azure key vault secret storing the Client ID.
-    key_vault_client_secret : str
-        The name of the Azure key vault secret storing the Client Secret.
     resource_group : str
         The name of the Azure resource group.
     source_capacity : str
@@ -701,10 +675,6 @@ def migrate_fabric_trial_capacity(
         create_fabric_capacity(
             capacity_name=target_capacity,
             azure_subscription_id=azure_subscription_id,
-            key_vault_uri=key_vault_uri,
-            key_vault_tenant_id=key_vault_tenant_id,
-            key_vault_client_id=key_vault_client_id,
-            key_vault_client_secret=key_vault_client_secret,
             resource_group=resource_group,
             region=target_capacity_region,
             admin_members=target_capacity_admin_members,
@@ -718,7 +688,7 @@ def migrate_fabric_trial_capacity(
     )
 
     # This migrates all the capacity settings
-    migrate_settings(
-        source_capacity=source_capacity,
-        target_capacity=target_capacity,
-    )
+    # migrate_settings(
+    #    source_capacity=source_capacity,
+    #    target_capacity=target_capacity,
+    # )

sempy_labs/_clear_cache.py

@@ -13,6 +13,7 @@ from sempy.fabric.exceptions import FabricHTTPException
 from uuid import UUID
 
 
+@log
 def clear_cache(dataset: str | UUID, workspace: Optional[str | UUID] = None):
     """
     Clears the cache of a semantic model.
@@ -105,7 +106,7 @@ def backup_semantic_model(
 
 @log
 def restore_semantic_model(
-    dataset: str | UUID,
+    dataset: str,
     file_path: str,
     allow_overwrite: bool = True,
     ignore_incompatibilities: bool = True,
@@ -118,8 +119,8 @@ def restore_semantic_model(
 
     Parameters
     ----------
-    dataset : str | uuid.UUID
-        Name or ID of the semantic model.
+    dataset : str
+        Name of the semantic model.
     file_path : str
         The location in which to backup the semantic model. Must end in '.abf'.
         Example 1: file_path = 'MyModel.abf'
@@ -142,11 +143,10 @@ def restore_semantic_model(
         )
 
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
-    (dataset_name, dataset_id) = resolve_dataset_name_and_id(dataset, workspace_id)
 
     tmsl = {
         "restore": {
-            "database": dataset_name,
+            "database": dataset,
             "file": file_path,
             "allowOverwrite": allow_overwrite,
             "security": "copyAll",
@@ -160,7 +160,7 @@ def restore_semantic_model(
     fabric.execute_tmsl(script=tmsl, workspace=workspace_id)
 
     print(
-        f"{icons.green_dot} The '{dataset_name}' semantic model has been restored to the '{workspace_name}' workspace based on the '{file_path}' backup file."
+        f"{icons.green_dot} The '{dataset}' semantic model has been restored to the '{workspace_name}' workspace based on the '{file_path}' backup file."
     )
 
 
@@ -318,8 +318,8 @@ def list_storage_account_files(
         ]
     )
 
-    onelake = _get_adls_client(storage_account)
-    fs = onelake.get_file_system_client(container)
+    client = _get_adls_client(storage_account)
+    fs = client.get_file_system_client(container)
 
     for x in list(fs.get_paths()):
         if not x.is_directory:

sempy_labs/_deployment_pipelines.py

@@ -148,7 +148,7 @@ def list_deployment_pipeline_stage_items(
         raise ValueError(
             f"{icons.red_dot} The '{stage_name}' stage does not exist within the '{deployment_pipeline}' deployment pipeline."
         )
-    stage_id = dfPS_filt["Deployment Pipeline Stage ID"].iloc[0]
+    stage_id = dfPS_filt["Deployment Pipeline Stage Id"].iloc[0]
 
     client = fabric.FabricRestClient()
     response = client.get(

sempy_labs/_gateways.py

@@ -1,4 +1,5 @@
 import sempy.fabric as fabric
+from sempy._utils._log import log
 import pandas as pd
 from typing import Optional
 from sempy.fabric.exceptions import FabricHTTPException
@@ -13,6 +14,7 @@ from uuid import UUID
 import sempy_labs._icons as icons
 
 
+@log
 def list_gateways() -> pd.DataFrame:
     """
     Returns a list of all gateways the user has permission for, including on-premises, on-premises (personal mode), and virtual network gateways.

sempy_labs/_generate_semantic_model.py

@@ -3,6 +3,7 @@ import pandas as pd
 import json
 import os
 from typing import Optional, List
+from sempy._utils._log import log
 from sempy_labs._helper_functions import (
     resolve_lakehouse_name,
     resolve_workspace_name_and_id,
@@ -17,6 +18,7 @@ from sempy_labs._refresh_semantic_model import refresh_semantic_model
 from uuid import UUID
 
 
+@log
 def create_blank_semantic_model(
     dataset: str,
     compatibility_level: int = 1605,
@@ -117,6 +119,7 @@ def create_blank_semantic_model(
     )
 
 
+@log
 def create_semantic_model_from_bim(
     dataset: str, bim_file: dict, workspace: Optional[str | UUID] = None
 ):
@@ -183,6 +186,7 @@ def create_semantic_model_from_bim(
     )
 
 
+@log
 def update_semantic_model_from_bim(
     dataset: str | UUID, bim_file: dict, workspace: Optional[str | UUID] = None
 ):
@@ -242,6 +246,7 @@ def update_semantic_model_from_bim(
     )
 
 
+@log
 def deploy_semantic_model(
     source_dataset: str,
     source_workspace: Optional[str | UUID] = None,
@@ -323,6 +328,7 @@ def deploy_semantic_model(
         refresh_semantic_model(dataset=target_dataset, workspace=target_workspace_id)
 
 
+@log
 def get_semantic_model_bim(
     dataset: str | UUID,
     workspace: Optional[str | UUID] = None,
@@ -386,6 +392,7 @@ def get_semantic_model_bim(
     return bimJson
 
 
+@log
 def get_semantic_model_definition(
     dataset: str | UUID,
     format: str = "TMSL",
@@ -454,6 +461,7 @@ def get_semantic_model_definition(
         return decoded_parts
 
 
+@log
 def get_semantic_model_size(
     dataset: str | UUID, workspace: Optional[str | UUID] = None
 ):

sempy_labs/_helper_functions.py

@@ -11,8 +11,8 @@ import datetime
 from typing import Optional, Tuple, List
 from uuid import UUID
 import sempy_labs._icons as icons
-import urllib.parse
 from azure.core.credentials import TokenCredential, AccessToken
+import urllib.parse
 import numpy as np
 from IPython.display import display, HTML
 
@@ -52,6 +52,24 @@ def create_abfss_path(
     return f"abfss://{lakehouse_workspace_id}@onelake.dfs.fabric.microsoft.com/{lakehouse_id}/Tables/{delta_table_name}"
 
 
+def _get_default_file_path() -> str:
+
+    default_file_storage = _get_fabric_context_setting(name="fs.defaultFS")
+
+    return default_file_storage.split("@")[-1][:-1]
+
+
+def _split_abfss_path(path: str) -> Tuple[UUID, UUID, str]:
+
+    parsed_url = urllib.parse.urlparse(path)
+
+    workspace_id = parsed_url.netloc.split("@")[0]
+    item_id = parsed_url.path.lstrip("/").split("/")[0]
+    delta_table_name = parsed_url.path.split("/")[-1]
+
+    return workspace_id, item_id, delta_table_name
+
+
 def format_dax_object_name(table: str, column: str) -> str:
     """
     Formats a table/column combination to the 'Table Name'[Column Name] format.
@@ -172,23 +190,40 @@ def resolve_item_name_and_id(
     return item_name, item_id
 
 
-def resolve_dataset_name_and_id(
-    dataset: str | UUID, workspace: Optional[str | UUID] = None
+def resolve_lakehouse_name_and_id(
+    lakehouse: Optional[str | UUID] = None, workspace: Optional[str | UUID] = None
 ) -> Tuple[str, UUID]:
 
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    type = "Lakehouse"
 
-    if _is_valid_uuid(dataset):
-        dataset_id = dataset
-        dataset_name = fabric.resolve_item_name(
-            item_id=dataset_id, type="SemanticModel", workspace=workspace_id
+    if lakehouse is None:
+        lakehouse_id = fabric.get_lakehouse_id()
+        lakehouse_name = fabric.resolve_item_name(
+            item_id=lakehouse_id, type=type, workspace=workspace_id
+        )
+    elif _is_valid_uuid(lakehouse):
+        lakehouse_id = lakehouse
+        lakehouse_name = fabric.resolve_item_name(
+            item_id=lakehouse_id, type=type, workspace=workspace_id
         )
     else:
-        dataset_name = dataset
-        dataset_id = fabric.resolve_item_id(
-            item_name=dataset, type="SemanticModel", workspace=workspace_id
+        lakehouse_name = lakehouse
+        lakehouse_id = fabric.resolve_item_id(
+            item_name=lakehouse, type=type, workspace=workspace_id
         )
 
+    return lakehouse_name, lakehouse_id
+
+
+def resolve_dataset_name_and_id(
+    dataset: str | UUID, workspace: Optional[str | UUID] = None
+) -> Tuple[str, UUID]:
+
+    (dataset_name, dataset_id) = resolve_item_name_and_id(
+        item=dataset, type="SemanticModel", workspace=workspace
+    )
+
     return dataset_name, dataset_id
 
 
@@ -280,15 +315,15 @@ def resolve_lakehouse_name(
 
 
 def resolve_lakehouse_id(
-    lakehouse: str, workspace: Optional[str | UUID] = None
+    lakehouse: Optional[str | UUID] = None, workspace: Optional[str | UUID] = None
 ) -> UUID:
     """
     Obtains the ID of the Fabric lakehouse.
 
     Parameters
     ----------
-    lakehouse : str
-        The name of the Fabric lakehouse.
+    lakehouse : str | uuid.UUID, default=None
+        The name or ID of the Fabric lakehouse.
     workspace : str | uuid.UUID, default=None
         The Fabric workspace name or ID.
         Defaults to None which resolves to the workspace of the attached lakehouse
@@ -300,9 +335,16 @@ def resolve_lakehouse_id(
         The ID of the Fabric lakehouse.
     """
 
-    return fabric.resolve_item_id(
-        item_name=lakehouse, type="Lakehouse", workspace=workspace
-    )
+    if lakehouse is None:
+        lakehouse_id = fabric.get_lakehouse_id()
+    elif _is_valid_uuid(lakehouse):
+        lakehouse_id = lakehouse
+    else:
+        lakehouse_id = fabric.resolve_item_id(
+            item_name=lakehouse, type="Lakehouse", workspace=workspace
+        )
+
+    return lakehouse_id
 
 
 def get_direct_lake_sql_endpoint(
@@ -328,9 +370,6 @@ def get_direct_lake_sql_endpoint(
 
     from sempy_labs.tom import connect_semantic_model
 
-    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
-    (dataset_name, dataset_id) = resolve_dataset_name_and_id(dataset, workspace_id)
-
     # dfP = fabric.list_partitions(dataset=dataset, workspace=workspace)
     # dfP_filt = dfP[dfP["Mode"] == "DirectLake"]
 
@@ -340,7 +379,7 @@ def get_direct_lake_sql_endpoint(
     #    )
 
     with connect_semantic_model(
-        dataset=dataset_id, readonly=True, workspace=workspace_id
+        dataset=dataset, readonly=True, workspace=workspace
     ) as tom:
         sqlEndpointId = None
         for e in tom.model.Expressions:
@@ -426,7 +465,7 @@ def save_as_delta_table(
     write_mode: str,
     merge_schema: bool = False,
     schema: Optional[dict] = None,
-    lakehouse: Optional[str] = None,
+    lakehouse: Optional[str | UUID] = None,
     workspace: Optional[str | UUID] = None,
 ):
     """
@@ -444,8 +483,8 @@ def save_as_delta_table(
         Merges the schemas of the dataframe to the delta table.
     schema : dict, default=None
         A dictionary showing the schema of the columns and their data types.
-    lakehouse : str, default=None
-        The Fabric lakehouse used by the Direct Lake semantic model.
+    lakehouse : str | uuid.UUID, default=None
+        The Fabric lakehouse name or ID.
         Defaults to None which resolves to the lakehouse attached to the notebook.
     workspace : str | uuid.UUID, default=None
         The Fabric workspace name or ID.
@@ -468,21 +507,16 @@ def save_as_delta_table(
     )
 
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    (lakehouse_name, lakehouse_id) = resolve_lakehouse_name_and_id(
+        lakehouse=lakehouse, workspace=workspace_id
+    )
 
-    if lakehouse is None:
-        lakehouse_id = fabric.get_lakehouse_id()
-        lakehouse = resolve_lakehouse_name(
-            lakehouse_id=lakehouse_id, workspace=workspace_id
-        )
-    else:
-        lakehouse_id = resolve_lakehouse_id(lakehouse, workspace_id)
-
-    writeModes = ["append", "overwrite"]
+    write_modes = ["append", "overwrite"]
     write_mode = write_mode.lower()
 
-    if write_mode not in writeModes:
+    if write_mode not in write_modes:
         raise ValueError(
-            f"{icons.red_dot} Invalid 'write_type' parameter. Choose from one of the following values: {writeModes}."
+            f"{icons.red_dot} Invalid 'write_type' parameter. Choose from one of the following values: {write_modes}."
         )
 
     if " " in delta_table_name:
@@ -507,16 +541,19 @@ def save_as_delta_table(
         "timestamp": TimestampType(),
     }
 
-    if schema is None:
-        spark_df = spark.createDataFrame(dataframe)
+    if isinstance(dataframe, pd.DataFrame):
+        if schema is None:
+            spark_df = spark.createDataFrame(dataframe)
+        else:
+            schema_map = StructType(
+                [
+                    StructField(column_name, type_mapping[data_type], True)
+                    for column_name, data_type in schema.items()
+                ]
+            )
+            spark_df = spark.createDataFrame(dataframe, schema_map)
     else:
-        schema_map = StructType(
-            [
-                StructField(column_name, type_mapping[data_type], True)
-                for column_name, data_type in schema.items()
-            ]
-        )
-        spark_df = spark.createDataFrame(dataframe, schema_map)
+        spark_df = dataframe
 
     filePath = create_abfss_path(
         lakehouse_id=lakehouse_id,
@@ -531,7 +568,7 @@ def save_as_delta_table(
     else:
         spark_df.write.mode(write_mode).format("delta").save(filePath)
     print(
-        f"{icons.green_dot} The dataframe has been saved as the '{delta_table_name}' table in the '{lakehouse}' lakehouse within the '{workspace_name}' workspace."
+        f"{icons.green_dot} The dataframe has been saved as the '{delta_table_name}' table in the '{lakehouse_name}' lakehouse within the '{workspace_name}' workspace."
     )
 
 
@@ -621,9 +658,7 @@ def _conv_b64(file):
 
 def _decode_b64(file, format: Optional[str] = "utf-8"):
 
-    result = base64.b64decode(file).decode(format)
-
-    return result
+    return base64.b64decode(file).decode(format)
 
 
 def is_default_semantic_model(
@@ -690,15 +725,15 @@ def resolve_item_type(item_id: UUID, workspace: Optional[str | UUID] = None) ->
 
 
 def resolve_dataset_from_report(
-    report: str, workspace: Optional[str | UUID] = None
+    report: str | UUID, workspace: Optional[str | UUID] = None
 ) -> Tuple[UUID, str, UUID, str]:
     """
     Obtains the basic semantic model properties from which the report's data is sourced.
 
     Parameters
     ----------
-    report : str
-        The name of the Power BI report.
+    report : str | uuid.UUID
+        The name or ID of the Power BI report.
     workspace : str | uuid.UUID, default=None
         The Fabric workspace name or ID.
         Defaults to None which resolves to the workspace of the attached lakehouse
@@ -706,20 +741,15 @@ def resolve_dataset_from_report(
 
     Returns
     -------
-    Tuple[UUID, str, UUID, str]
+    Tuple[uuid.UUID, str, uuid.UUID, str]
         The semantic model UUID, semantic model name, semantic model workspace UUID, semantic model workspace name
     """
 
-    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    from sempy_labs.report._generate_report import _get_report
 
-    dfR = fabric.list_reports(workspace=workspace_id)
-    dfR_filt = dfR[dfR["Name"] == report]
-    if len(dfR_filt) == 0:
-        raise ValueError(
-            f"{icons.red_dot} The '{report}' report does not exist within the '{workspace_name}' workspace."
-        )
-    dataset_id = dfR_filt["Dataset Id"].iloc[0]
-    dataset_workspace_id = dfR_filt["Dataset Workspace Id"].iloc[0]
+    dfR = _get_report(report=report, workspace=workspace)
+    dataset_id = dfR["Dataset Id"].iloc[0]
+    dataset_workspace_id = dfR["Dataset Workspace Id"].iloc[0]
     dataset_workspace = fabric.resolve_workspace_name(dataset_workspace_id)
     dataset_name = resolve_dataset_name(
         dataset_id=dataset_id, workspace=dataset_workspace
@@ -975,7 +1005,7 @@ def resolve_deployment_pipeline_id(deployment_pipeline: str) -> UUID:
 
     Returns
     -------
-    UUID
+    uuid.UUID
         The deployment pipeline Id.
     """
 
@@ -1024,14 +1054,16 @@ def _get_adls_client(account_name):
     return service_client
 
 
-def resolve_warehouse_id(warehouse: str, workspace: Optional[str | UUID]) -> UUID:
+def resolve_warehouse_id(
+    warehouse: str | UUID, workspace: Optional[str | UUID]
+) -> UUID:
     """
     Obtains the Id for a given warehouse.
 
     Parameters
     ----------
-    warehouse : str
-        The warehouse name
+    warehouse : str | uuid.UUID
+        The warehouse name or ID.
     workspace : str | uuid.UUID, default=None
         The Fabric workspace name or ID in which the semantic model resides.
         Defaults to None which resolves to the workspace of the attached lakehouse
@@ -1039,13 +1071,16 @@ def resolve_warehouse_id(warehouse: str, workspace: Optional[str | UUID]) -> UUI
 
     Returns
     -------
-    UUID
+    uuid.UUID
         The warehouse Id.
     """
 
-    return fabric.resolve_item_id(
-        item_name=warehouse, type="Warehouse", workspace=workspace
-    )
+    if _is_valid_uuid(warehouse):
+        return warehouse
+    else:
+        return fabric.resolve_item_id(
+            item_name=warehouse, type="Warehouse", workspace=workspace
+        )
 
 
 def get_language_codes(languages: str | List[str]):
@@ -1188,16 +1223,24 @@ def generate_guid():
     return str(uuid.uuid4())
 
 
-def _get_max_run_id(lakehouse: str, table_name: str) -> int:
+def _get_column_aggregate(
+    lakehouse: str,
+    table_name: str,
+    column_name: str = "RunId",
+    function: str = "max",
+    default_value: int = 0,
+) -> int:
 
     from pyspark.sql import SparkSession
 
     spark = SparkSession.builder.getOrCreate()
-    query = f"SELECT MAX(RunId) FROM {lakehouse}.{table_name}"
+    function = function.upper()
+    query = f"SELECT {function}({column_name}) FROM {lakehouse}.{table_name}"
+    if "COUNT" in function and "DISTINCT" in function:
+        query = f"SELECT COUNT(DISTINCT({column_name})) FROM {lakehouse}.{table_name}"
     dfSpark = spark.sql(query)
-    max_run_id = dfSpark.collect()[0][0] or 0
 
-    return max_run_id
+    return dfSpark.collect()[0][0] or default_value
 
 
 def _make_list_unique(my_list):
@@ -1209,20 +1252,17 @@ def _get_partition_map(
     dataset: str, workspace: Optional[str | UUID] = None
 ) -> pd.DataFrame:
 
-    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
-    (dataset_name, dataset_id) = resolve_dataset_name_and_id(dataset, workspace_id)
-
     partitions = fabric.evaluate_dax(
-        dataset=dataset_id,
-        workspace=workspace_id,
+        dataset=dataset,
+        workspace=workspace,
         dax_string="""
     select [ID] AS [PartitionID], [TableID], [Name] AS [PartitionName] from $system.tmschema_partitions
     """,
     )
 
     tables = fabric.evaluate_dax(
-        dataset=dataset_id,
-        workspace=workspace_id,
+        dataset=dataset,
+        workspace=workspace,
         dax_string="""
     select [ID] AS [TableID], [Name] AS [TableName] from $system.tmschema_tables
     """,