semantic-link-labs 0.9.9__py3-none-any.whl → 0.9.10__py3-none-any.whl

sempy_labs/lakehouse/_lakehouse.py

@@ -7,6 +7,7 @@ from sempy_labs._helper_functions import (
     resolve_lakehouse_name_and_id,
     resolve_workspace_name_and_id,
     _create_spark_session,
+    _pure_python_notebook,
 )
 import sempy_labs._icons as icons
 import re
@@ -32,6 +33,33 @@ def lakehouse_attached() -> bool:
         return False
 
 
+def _optimize_table(path):
+
+    if _pure_python_notebook():
+        from deltalake import DeltaTable
+
+        DeltaTable(path).optimize.compact()
+    else:
+        from delta import DeltaTable
+
+        spark = _create_spark_session()
+        DeltaTable.forPath(spark, path).optimize().executeCompaction()
+
+
+def _vacuum_table(path, retain_n_hours):
+
+    if _pure_python_notebook():
+        from deltalake import DeltaTable
+
+        DeltaTable(path).vacuum(retention_hours=retain_n_hours)
+    else:
+        from delta import DeltaTable
+
+        spark = _create_spark_session()
+        spark.conf.set("spark.databricks.delta.vacuum.parallelDelete.enabled", "true")
+        DeltaTable.forPath(spark, path).vacuum(retain_n_hours)
+
+
 @log
 def optimize_lakehouse_tables(
     tables: Optional[Union[str, List[str]]] = None,
@@ -56,27 +84,20 @@ def optimize_lakehouse_tables(
     """
 
     from sempy_labs.lakehouse._get_lakehouse_tables import get_lakehouse_tables
-    from delta import DeltaTable
 
-    lakeTables = get_lakehouse_tables(lakehouse=lakehouse, workspace=workspace)
-    lakeTablesDelta = lakeTables[lakeTables["Format"] == "delta"]
+    df = get_lakehouse_tables(lakehouse=lakehouse, workspace=workspace)
+    df_delta = df[df["Format"] == "delta"]
 
     if isinstance(tables, str):
         tables = [tables]
 
-    if tables is not None:
-        tables_filt = lakeTablesDelta[lakeTablesDelta["Table Name"].isin(tables)]
-    else:
-        tables_filt = lakeTablesDelta.copy()
-
-    spark = _create_spark_session()
+    df_tables = df_delta[df_delta["Table Name"].isin(tables)] if tables else df_delta
 
-    for _, r in (bar := tqdm(tables_filt.iterrows())):
-        tableName = r["Table Name"]
-        tablePath = r["Location"]
-        bar.set_description(f"Optimizing the '{tableName}' table...")
-        deltaTable = DeltaTable.forPath(spark, tablePath)
-        deltaTable.optimize().executeCompaction()
+    for _, r in (bar := tqdm(df_tables.iterrows())):
+        table_name = r["Table Name"]
+        path = r["Location"]
+        bar.set_description(f"Optimizing the '{table_name}' table...")
+        _optimize_table(path=path)
 
 
 @log
@@ -108,32 +129,20 @@ def vacuum_lakehouse_tables(
     """
 
     from sempy_labs.lakehouse._get_lakehouse_tables import get_lakehouse_tables
-    from delta import DeltaTable
 
-    lakeTables = get_lakehouse_tables(lakehouse=lakehouse, workspace=workspace)
-    lakeTablesDelta = lakeTables[lakeTables["Format"] == "delta"]
+    df = get_lakehouse_tables(lakehouse=lakehouse, workspace=workspace)
+    df_delta = df[df["Format"] == "delta"]
 
     if isinstance(tables, str):
         tables = [tables]
 
-    if tables is not None:
-        tables_filt = lakeTablesDelta[lakeTablesDelta["Table Name"].isin(tables)]
-    else:
-        tables_filt = lakeTablesDelta.copy()
-
-    spark = _create_spark_session()
-    spark.conf.set("spark.databricks.delta.vacuum.parallelDelete.enabled", "true")
-
-    for _, r in (bar := tqdm(tables_filt.iterrows())):
-        tableName = r["Table Name"]
-        tablePath = r["Location"]
-        bar.set_description(f"Vacuuming the '{tableName}' table...")
-        deltaTable = DeltaTable.forPath(spark, tablePath)
+    df_tables = df_delta[df_delta["Table Name"].isin(tables)] if tables else df_delta
 
-        if retain_n_hours is None:
-            deltaTable.vacuum()
-        else:
-            deltaTable.vacuum(retain_n_hours)
+    for _, r in (bar := tqdm(df_tables.iterrows())):
+        table_name = r["Table Name"]
+        path = r["Location"]
+        bar.set_description(f"Vacuuming the '{table_name}' table...")
+        _vacuum_table(path=path, retain_n_hours=retain_n_hours)
 
 
 def run_table_maintenance(

sempy_labs/migration/_migrate_calctables_to_lakehouse.py

@@ -6,10 +6,9 @@ from sempy_labs.lakehouse._get_lakehouse_tables import get_lakehouse_tables
 from sempy_labs._helper_functions import (
     resolve_lakehouse_name,
     resolve_lakehouse_id,
-    create_abfss_path,
     retry,
     generate_guid,
-    _create_spark_session,
+    save_as_delta_table,
 )
 from sempy_labs.tom import connect_semantic_model
 from typing import Optional
@@ -98,8 +97,6 @@ def migrate_calc_tables_to_lakehouse(
     if killFunction:
         return
 
-    spark = _create_spark_session()
-
     if len(dfP_filt) == 0:
         print(
             f"{icons.yellow_dot} The '{dataset}' semantic model in the '{workspace}' workspace has no calculated tables."
@@ -198,14 +195,12 @@ def migrate_calc_tables_to_lakehouse(
 
                                 delta_table_name = t.Name.replace(" ", "_").lower()
 
-                                spark_df = spark.createDataFrame(df)
-                                filePath = create_abfss_path(
-                                    lakehouse_id=lakehouse_id,
-                                    lakehouse_workspace_id=lakehouse_workspace_id,
-                                    delta_table_name=delta_table_name,
-                                )
-                                spark_df.write.mode("overwrite").format("delta").save(
-                                    filePath
+                                save_as_delta_table(
+                                    dataframe=df,
+                                    table_name=delta_table_name,
+                                    lakehouse=lakehouse,
+                                    workspace=lakehouse_workspace,
+                                    write_mode="overwrite",
                                 )
 
                                 @retry(

sempy_labs/migration/_refresh_calc_tables.py

@@ -1,7 +1,6 @@
 import sempy.fabric as fabric
 import pandas as pd
 import re
-from sempy_labs._helper_functions import retry
 from sempy_labs.tom import connect_semantic_model
 from typing import Optional
 from sempy._utils._log import log
@@ -10,7 +9,8 @@ from uuid import UUID
 from sempy_labs._helper_functions import (
     resolve_workspace_name_and_id,
     resolve_dataset_name_and_id,
-    _create_spark_session,
+    save_as_delta_table,
+    retry,
 )
 
 
@@ -29,7 +29,6 @@ def refresh_calc_tables(dataset: str | UUID, workspace: Optional[str | UUID] = N
         or if no lakehouse attached, resolves to the workspace of the notebook.
     """
 
-    spark = _create_spark_session()
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
     (dataset_name, dataset_id) = resolve_dataset_name_and_id(dataset, workspace_id)
     icons.sll_tags.append("DirectLakeMigration")
@@ -117,10 +116,12 @@ def refresh_calc_tables(dataset: str | UUID, workspace: Optional[str | UUID] = N
                         f"{icons.in_progress} Refresh of the '{delta_table_name}' table within the lakehouse is in progress..."
                     )
 
-                    spark_df = spark.createDataFrame(df)
-                    spark_df.write.mode("overwrite").format("delta").saveAsTable(
-                        delta_table_name
+                    save_as_delta_table(
+                        dataframe=df,
+                        table_name=delta_table_name,
+                        write_mode="overwrite",
                     )
+
                     print(
                         f"{icons.green_dot} Calculated table '{tName}' has been refreshed as the '{delta_table_name.lower()}' table in the lakehouse."
                     )

sempy_labs/tom/_model.py

@@ -938,19 +938,23 @@ class TOMWrapper:
         import Microsoft.AnalysisServices.Tabular as TOM
         import System
 
-        if cross_filtering_behavior is None:
+        if not cross_filtering_behavior:
             cross_filtering_behavior = "Automatic"
-        if security_filtering_behavior is None:
+        if not security_filtering_behavior:
             security_filtering_behavior = "OneDirection"
 
-        from_cardinality = from_cardinality.capitalize()
-        to_cardinality = to_cardinality.capitalize()
-        cross_filtering_behavior = cross_filtering_behavior.capitalize()
-        security_filtering_behavior = security_filtering_behavior.capitalize()
+        for var_name in [
+            "from_cardinality",
+            "to_cardinality",
+            "cross_filtering_behavior",
+            "security_filtering_behavior",
+        ]:
+            locals()[var_name] = locals()[var_name].capitalize()
+
+        cross_filtering_behavior = cross_filtering_behavior.replace("direct", "Direct")
         security_filtering_behavior = security_filtering_behavior.replace(
             "direct", "Direct"
         )
-        cross_filtering_behavior = cross_filtering_behavior.replace("direct", "Direct")
 
         rel = TOM.SingleColumnRelationship()
         rel.FromColumn = self.model.Tables[from_table].Columns[from_column]
@@ -962,13 +966,16 @@ class TOMWrapper:
             TOM.RelationshipEndCardinality, to_cardinality
         )
         rel.IsActive = is_active
-        rel.CrossFilteringBehavior = System.Enum.Parse(
-            TOM.CrossFilteringBehavior, cross_filtering_behavior
-        )
-        rel.SecurityFilteringBehavior = System.Enum.Parse(
-            TOM.SecurityFilteringBehavior, security_filtering_behavior
-        )
-        rel.RelyOnReferentialIntegrity = rely_on_referential_integrity
+        if cross_filtering_behavior != "Automatic":
+            rel.CrossFilteringBehavior = System.Enum.Parse(
+                TOM.CrossFilteringBehavior, cross_filtering_behavior
+            )
+        if security_filtering_behavior != "OneDirection":
+            rel.SecurityFilteringBehavior = System.Enum.Parse(
+                TOM.SecurityFilteringBehavior, security_filtering_behavior
+            )
+        if rely_on_referential_integrity:
+            rel.RelyOnReferentialIntegrity = True
 
         self.model.Relationships.Add(rel)