semantic-link-labs 0.5.0__py3-none-any.whl → 0.7.0__py3-none-any.whl

sempy_labs/migration/_migrate_tables_columns_to_semantic_model.py

@@ -1,13 +1,13 @@
-import sempy
 import sempy.fabric as fabric
 import pandas as pd
-import datetime, time
+import datetime
+import time
 from sempy_labs._list_functions import list_tables
 from sempy_labs.directlake._get_shared_expression import get_shared_expression
-from sempy_labs._helper_functions import resolve_lakehouse_name
+from sempy_labs._helper_functions import resolve_lakehouse_name, retry
 from sempy_labs.lakehouse._lakehouse import lakehouse_attached
 from sempy_labs.tom import connect_semantic_model
-from typing import List, Optional, Union
+from typing import Optional
 from sempy._utils._log import log
 import sempy_labs._icons as icons
 
@@ -61,105 +61,99 @@ def migrate_tables_columns_to_semantic_model(
         lakehouse = resolve_lakehouse_name(lakehouse_id, lakehouse_workspace)
 
     # Check that lakehouse is attached to the notebook
-    lakeAttach = lakehouse_attached()
-
-    # Run if lakehouse is attached to the notebook or a lakehouse & lakehouse workspace are specified
-    if lakeAttach or (lakehouse is not None and lakehouse_workspace is not None):
-        shEx = get_shared_expression(lakehouse, lakehouse_workspace)
-
-        dfC = fabric.list_columns(dataset=dataset, workspace=workspace)
-        dfT = list_tables(dataset, workspace)
-        dfT.rename(columns={"Type": "Table Type"}, inplace=True)
-        dfC = pd.merge(
-            dfC,
-            dfT[["Name", "Table Type"]],
-            left_on="Table Name",
-            right_on="Name",
-            how="left",
-        )
-        dfT_filt = dfT[dfT["Table Type"] == "Table"]
-        dfC_filt = dfC[
-            (dfC["Table Type"] == "Table")
-            & ~(dfC["Column Name"].str.startswith("RowNumber-"))
-            & (dfC["Type"] != "Calculated")
-        ]
-
-        print(f"{icons.in_progress} Updating '{new_dataset}' based on '{dataset}'...")
-        start_time = datetime.datetime.now()
-        timeout = datetime.timedelta(minutes=1)
-        success = False
-
-        while not success:
-            try:
-                with connect_semantic_model(
-                    dataset=new_dataset, readonly=False, workspace=new_dataset_workspace
-                ) as tom:
-                    success = True
-                    try:
-                        tom.model.Expressions["DatabaseQuery"]
-                    except:
-                        tom.add_expression("DatabaseQuery", expression=shEx)
-                        print(
-                            f"{icons.green_dot} The 'DatabaseQuery' expression has been added."
-                        )
-
-                    for i, r in dfT_filt.iterrows():
-                        tName = r["Name"]
-                        tDC = r["Data Category"]
-                        tHid = bool(r["Hidden"])
-                        tDesc = r["Description"]
-
-                        try:
-                            tom.model.Tables[tName]
-                        except:
-                            tom.add_table(
-                                name=tName,
-                                description=tDesc,
-                                data_category=tDC,
-                                hidden=tHid,
-                            )
-                            tom.add_entity_partition(
-                                table_name=tName, entity_name=tName.replace(" ", "_")
-                            )
-                            print(
-                                f"{icons.green_dot} The '{tName}' table has been added."
-                            )
-
-                    for i, r in dfC_filt.iterrows():
-                        tName = r["Table Name"]
-                        cName = r["Column Name"]
-                        scName = r["Source"].replace(" ", "_")
-                        cHid = bool(r["Hidden"])
-                        cDataType = r["Data Type"]
-
-                        try:
-                            tom.model.Tables[tName].Columns[cName]
-                        except:
-                            tom.add_data_column(
-                                table_name=tName,
-                                column_name=cName,
-                                source_column=scName,
-                                hidden=cHid,
-                                data_type=cDataType,
-                            )
-                            print(
-                                f"{icons.green_dot} The '{tName}'[{cName}] column has been added."
-                            )
-
-                    print(
-                        f"\n{icons.green_dot} All regular tables and columns have been added to the '{new_dataset}' semantic model."
-                    )
-            except Exception as e:
-                if datetime.datetime.now() - start_time > timeout:
-                    break
-                time.sleep(1)
-    else:
-        print(
-            f"{icons.red_dot} Lakehouse not attached to notebook and lakehouse/lakehouse_workspace are not specified. Please add your lakehouse to this notebook or specify the lakehouse/lakehouse_workspace parameters."
-        )
-        print(
+    if not lakehouse_attached() and (lakehouse is None and lakehouse_workspace is None):
+        raise ValueError(
+            f"{icons.red_dot} Lakehouse not attached to notebook and lakehouse/lakehouse_workspace are not specified. Please add your lakehouse to this notebook"
+            f" or specify the lakehouse/lakehouse_workspace parameters."
             "To attach a lakehouse to a notebook, go to the the 'Explorer' window to the left, click 'Lakehouses' to add your lakehouse to this notebook"
+            "\nLearn more here: https://learn.microsoft.com/fabric/data-engineering/lakehouse-notebook-explore#add-or-remove-a-lakehouse"
         )
+    shEx = get_shared_expression(lakehouse, lakehouse_workspace)
+
+    dfC = fabric.list_columns(dataset=dataset, workspace=workspace)
+    dfT = list_tables(dataset, workspace)
+    dfT.rename(columns={"Type": "Table Type"}, inplace=True)
+    dfC = pd.merge(
+        dfC,
+        dfT[["Name", "Table Type"]],
+        left_on="Table Name",
+        right_on="Name",
+        how="left",
+    )
+    dfT_filt = dfT[dfT["Table Type"] == "Table"]
+    dfC_filt = dfC[
+        (dfC["Table Type"] == "Table")
+        & ~(dfC["Column Name"].str.startswith("RowNumber-"))
+        & (dfC["Type"] != "Calculated")
+    ]
+
+    print(f"{icons.in_progress} Updating '{new_dataset}' based on '{dataset}'...")
+
+    @retry(
+        sleep_time=1,
+        timeout_error_message=f"{icons.red_dot} Function timed out after 1 minute",
+    )
+    def dyn_connect():
+        with connect_semantic_model(
+            dataset=new_dataset, readonly=True, workspace=new_dataset_workspace
+        ) as tom:
+
+            tom.model
+
+    dyn_connect()
+
+    with connect_semantic_model(
+        dataset=new_dataset, readonly=False, workspace=new_dataset_workspace
+    ) as tom:
+        if not any(e.Name == "DatabaseQuery" for e in tom.model.Expressions):
+            tom.add_expression("DatabaseQuery", expression=shEx)
+            print(f"{icons.green_dot} The 'DatabaseQuery' expression has been added.")
+
+        for i, r in dfT_filt.iterrows():
+            tName = r["Name"]
+            tDC = r["Data Category"]
+            tHid = bool(r["Hidden"])
+            tDesc = r["Description"]
+            ent_name = tName.replace(" ", "_")
+            for char in icons.special_characters:
+                ent_name = ent_name.replace(char, "")
+
+            if not any(t.Name == tName for t in tom.model.Tables):
+                tom.add_table(
+                    name=tName,
+                    description=tDesc,
+                    data_category=tDC,
+                    hidden=tHid,
+                )
+                tom.add_entity_partition(table_name=tName, entity_name=ent_name)
+                print(f"{icons.green_dot} The '{tName}' table has been added.")
+
+        for i, r in dfC_filt.iterrows():
+            tName = r["Table Name"]
+            cName = r["Column Name"]
+            scName = r["Source"].replace(" ", "_")
+            cHid = bool(r["Hidden"])
+            cDataType = r["Data Type"]
+            for char in icons.special_characters:
+                scName = scName.replace(char, "")
+
+            if scName.endswith("_"):
+                scName = scName[:-1]
+
+            if not any(
+                c.Name == cName and c.Parent.Name == tName for c in tom.all_columns()
+            ):
+                tom.add_data_column(
+                    table_name=tName,
+                    column_name=cName,
+                    source_column=scName,
+                    hidden=cHid,
+                    data_type=cDataType,
+                )
+                print(
+                    f"{icons.green_dot} The '{tName}'[{cName}] column has been added."
+                )
+
         print(
-            "\nLearn more here: https://learn.microsoft.com/fabric/data-engineering/lakehouse-notebook-explore#add-or-remove-a-lakehouse"
+            f"\n{icons.green_dot} All regular tables and columns have been added to the '{new_dataset}' semantic model."
         )

sempy_labs/migration/_migration_validation.py

@@ -1,10 +1,10 @@
-import sempy
 import sempy.fabric as fabric
 import pandas as pd
-from typing import List, Optional, Union
+from typing import Optional
 from sempy_labs._list_functions import list_semantic_model_objects
 from sempy._utils._log import log
 
+
 @log
 def migration_validation(
     dataset: str,

sempy_labs/migration/_refresh_calc_tables.py

@@ -1,10 +1,10 @@
-import sempy
 import sempy.fabric as fabric
 import pandas as pd
-import re, datetime, time
+import re
+from sempy_labs._helper_functions import retry
 from pyspark.sql import SparkSession
 from sempy_labs.tom import connect_semantic_model
-from typing import List, Optional, Union
+from typing import Optional
 from sempy._utils._log import log
 import sempy_labs._icons as icons
 
@@ -25,105 +25,99 @@ def refresh_calc_tables(dataset: str, workspace: Optional[str] = None):
     """
 
     spark = SparkSession.builder.getOrCreate()
-
-    start_time = datetime.datetime.now()
-    timeout = datetime.timedelta(minutes=1)
-    success = False
-
     workspace = fabric.resolve_workspace_name(workspace)
 
-    while not success:
-        try:
-            with connect_semantic_model(
-                dataset=dataset, readonly=True, workspace=workspace
-            ) as tom:
-                success = True
-                for a in tom.model.Annotations:
-                    if any(a.Name == t.Name for t in tom.model.Tables):
-                        tName = a.Name
-                        query = a.Value
-
-                        if not query.startswith("EVALUATE"):
-                            daxquery = "EVALUATE \n" + query
-                        else:
-                            daxquery = query
-
-                        try:
-                            df = fabric.evaluate_dax(
-                                dataset=dataset,
-                                dax_string=daxquery,
-                                workspace=workspace,
+    @retry(
+        sleep_time=1,
+        timeout_error_message=f"{icons.red_dot} Function timed out after 1 minute",
+    )
+    def dyn_connect():
+        with connect_semantic_model(
+            dataset=dataset, readonly=True, workspace=workspace
+        ) as tom:
+
+            tom.model
+
+    dyn_connect()
+
+    with connect_semantic_model(
+        dataset=dataset, readonly=True, workspace=workspace
+    ) as tom:
+        for a in tom.model.Annotations:
+            if any(a.Name == t.Name for t in tom.model.Tables):
+                tName = a.Name
+                query = a.Value
+
+                if not query.startswith("EVALUATE"):
+                    daxquery = "EVALUATE \n" + query
+                else:
+                    daxquery = query
+
+                try:
+                    df = fabric.evaluate_dax(
+                        dataset=dataset,
+                        dax_string=daxquery,
+                        workspace=workspace,
+                    )
+
+                    # Update column names for non-field parameters
+                    if query.find("NAMEOF") == -1:
+                        for old_column_name in df.columns:
+                            pattern = r"\[([^\]]+)\]"
+
+                            matches = re.findall(pattern, old_column_name)
+                            new_column_name = matches[0]
+                            new_column_name = new_column_name.replace(" ", "")
+
+                            df.rename(
+                                columns={old_column_name: new_column_name},
+                                inplace=True,
                             )
 
-                            # Update column names for non-field parameters
-                            if query.find("NAMEOF") == -1:
-                                for old_column_name in df.columns:
-                                    pattern = r"\[([^\]]+)\]"
-
-                                    matches = re.findall(pattern, old_column_name)
-                                    new_column_name = matches[0]
-                                    new_column_name = new_column_name.replace(" ", "")
-
-                                    df.rename(
-                                        columns={old_column_name: new_column_name},
-                                        inplace=True,
-                                    )
-
-                                    # Update data types for lakehouse columns
-                                    dataType = next(
-                                        str(c.DataType)
-                                        for c in tom.all_columns()
-                                        if c.Parent.Name == tName
-                                        and c.SourceColumn == new_column_name
-                                    )
-                                    # dfC_type = dfC[(dfC['Table Name'] == tName) & (dfC['Source'] == new_column_name)]
-                                    # dataType = dfC_type['Data Type'].iloc[0]
-
-                                    if dataType == "Int64":
-                                        df[new_column_name] = df[
-                                            new_column_name
-                                        ].astype(int)
-                                    elif dataType in ["Decimal", "Double"]:
-                                        df[new_column_name] = df[
-                                            new_column_name
-                                        ].astype(float)
-                                    elif dataType == "Boolean":
-                                        df[new_column_name] = df[
-                                            new_column_name
-                                        ].astype(bool)
-                                    elif dataType == "DateTime":
-                                        df[new_column_name] = pd.to_datetime(
-                                            df[new_column_name]
-                                        )
-                                    else:
-                                        df[new_column_name] = df[
-                                            new_column_name
-                                        ].astype(str)
-                            # else:
-                            #    second_column_name = df.columns[1]
-                            #    third_column_name = df.columns[2]
-                            #    df[third_column_name] = df[third_column_name].astype(int)
-
-                            # Remove calc columns from field parameters
-                            #    mask = df[second_column_name].isin(dfC_filt['Full Column Name'])
-                            #    df = df[~mask]
-
-                            delta_table_name = tName.replace(" ", "_")
-                            print(
-                                f"{icons.in_progress} Refresh of the '{delta_table_name}' table within the lakehouse is in progress..."
+                            # Update data types for lakehouse columns
+                            dataType = next(
+                                str(c.DataType)
+                                for c in tom.all_columns()
+                                if c.Parent.Name == tName
+                                and c.SourceColumn == new_column_name
                             )
-
-                            spark_df = spark.createDataFrame(df)
-                            spark_df.write.mode("overwrite").format(
-                                "delta"
-                            ).saveAsTable(delta_table_name)
-                            print(
-                                f"{icons.green_dot} Calculated table '{tName}' has been refreshed as the '{delta_table_name.lower()}' table in the lakehouse."
-                            )
-                        except Exception as e:
-                            raise ValueError(f"{icons.red_dot} Failed to create calculated table '{tName}' as a delta table in the lakehouse.") from e
-
-        except Exception as e:
-            if datetime.datetime.now() - start_time > timeout:
-                break
-            time.sleep(1)
+                            # dfC_type = dfC[(dfC['Table Name'] == tName) & (dfC['Source'] == new_column_name)]
+                            # dataType = dfC_type['Data Type'].iloc[0]
+
+                            if dataType == "Int64":
+                                df[new_column_name] = df[new_column_name].astype(int)
+                            elif dataType in ["Decimal", "Double"]:
+                                df[new_column_name] = df[new_column_name].astype(float)
+                            elif dataType == "Boolean":
+                                df[new_column_name] = df[new_column_name].astype(bool)
+                            elif dataType == "DateTime":
+                                df[new_column_name] = pd.to_datetime(
+                                    df[new_column_name]
+                                )
+                            else:
+                                df[new_column_name] = df[new_column_name].astype(str)
+                    # else:
+                    #    second_column_name = df.columns[1]
+                    #    third_column_name = df.columns[2]
+                    #    df[third_column_name] = df[third_column_name].astype(int)
+
+                    # Remove calc columns from field parameters
+                    #    mask = df[second_column_name].isin(dfC_filt['Full Column Name'])
+                    #    df = df[~mask]
+
+                    delta_table_name = tName.replace(" ", "_")
+                    print(
+                        f"{icons.in_progress} Refresh of the '{delta_table_name}' table within the lakehouse is in progress..."
+                    )
+
+                    spark_df = spark.createDataFrame(df)
+                    spark_df.write.mode("overwrite").format("delta").saveAsTable(
+                        delta_table_name
+                    )
+                    print(
+                        f"{icons.green_dot} Calculated table '{tName}' has been refreshed as the '{delta_table_name.lower()}' table in the lakehouse."
+                    )
+                except Exception as e:
+                    raise ValueError(
+                        f"{icons.red_dot} Failed to create calculated table '{tName}' as a delta table in the lakehouse."
+                    ) from e