semantic-link-labs 0.4.1__py3-none-any.whl

sempy_labs/migration/_migrate_calctables_to_semantic_model.py

@@ -0,0 +1,150 @@
+import sempy
+import sempy.fabric as fabric
+import re, datetime, time
+from sempy_labs.lakehouse._get_lakehouse_tables import get_lakehouse_tables
+from sempy_labs._helper_functions import resolve_lakehouse_name
+from sempy_labs._tom import connect_semantic_model
+from typing import Optional
+from sempy._utils._log import log
+import sempy_labs._icons as icons
+
+
+@log
+def migrate_calc_tables_to_semantic_model(
+    dataset: str,
+    new_dataset: str,
+    workspace: Optional[str] = None,
+    new_dataset_workspace: Optional[str] = None,
+    lakehouse: Optional[str] = None,
+    lakehouse_workspace: Optional[str] = None,
+):
+    """
+    Creates new tables in the Direct Lake semantic model based on the lakehouse tables created using the 'migrate_calc_tables_to_lakehouse' function.
+
+    Parameters
+    ----------
+    dataset : str
+        Name of the import/DirectQuery semantic model.
+    new_dataset : str
+        Name of the Direct Lake semantic model.
+    workspace : str, default=None
+        The Fabric workspace name in which the import/DirectQuery semantic model exists.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    new_dataset_workspace : str
+        The Fabric workspace name in which the Direct Lake semantic model will be created.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    lakehouse : str, default=None
+        The Fabric lakehouse used by the Direct Lake semantic model.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    lakehouse_workspace : str, default=None
+        The Fabric workspace used by the lakehouse.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    workspace = fabric.resolve_workspace_name(workspace)
+
+    if new_dataset_workspace == None:
+        new_dataset_workspace = workspace
+
+    if lakehouse_workspace == None:
+        lakehouse_workspace = new_dataset_workspace
+    if lakehouse == None:
+        lakehouse_id = fabric.get_lakehouse_id()
+        lakehouse = resolve_lakehouse_name(lakehouse_id, lakehouse_workspace)
+
+    # Get calc tables but not field parameters
+    dfP = fabric.list_partitions(dataset=dataset, workspace=workspace)
+    dfP_filt = dfP[(dfP["Source Type"] == "Calculated")]
+    dfP_filt = dfP_filt[~dfP_filt["Query"].str.contains("NAMEOF")]
+
+    dfC = fabric.list_columns(dataset=dataset, workspace=workspace)
+    lc = get_lakehouse_tables(lakehouse=lakehouse, workspace=lakehouse_workspace)
+    # Get all calc table columns of calc tables not including field parameters
+    dfC_filt = dfC[
+        (dfC["Table Name"].isin(dfP_filt["Table Name"]))
+    ]  # & (dfC['Type'] == 'CalculatedTableColumn')]
+    # dfA = list_annotations(new_dataset, new_dataset_workspace)
+    # dfA_filt = dfA[(dfA['Object Type'] == 'Model') & ~ (dfA['Annotation Value'].str.contains('NAMEOF'))]
+
+    if len(dfP_filt) == 0:
+        print(
+            f"{icons.green_dot} The '{dataset}' semantic model has no calculated tables."
+        )
+        return
+
+    start_time = datetime.datetime.now()
+    timeout = datetime.timedelta(minutes=1)
+    success = False
+
+    while not success:
+        try:
+            with connect_semantic_model(
+                dataset=new_dataset, readonly=False, workspace=new_dataset_workspace
+            ) as tom:
+                success = True
+                for tName in dfC_filt["Table Name"].unique():
+                    if tName.lower() in lc["Table Name"].values:
+
+                        try:
+                            tom.model.Tables[tName]
+                        except:
+                            tom.add_table(name=tName)
+                            tom.add_entity_partition(
+                                table_name=tName,
+                                entity_name=tName.replace(" ", "_").lower(),
+                            )
+
+                    columns_in_table = dfC_filt.loc[
+                        dfC_filt["Table Name"] == tName, "Column Name"
+                    ].unique()
+
+                    for cName in columns_in_table:
+                        scName = dfC.loc[
+                            (dfC["Table Name"] == tName)
+                            & (dfC["Column Name"] == cName),
+                            "Source",
+                        ].iloc[0]
+                        cDataType = dfC.loc[
+                            (dfC["Table Name"] == tName)
+                            & (dfC["Column Name"] == cName),
+                            "Data Type",
+                        ].iloc[0]
+                        cType = dfC.loc[
+                            (dfC["Table Name"] == tName)
+                            & (dfC["Column Name"] == cName),
+                            "Type",
+                        ].iloc[0]
+
+                        # av = tom.get_annotation_value(object = tom.model, name = tName)
+
+                        # if cType == 'CalculatedTableColumn':
+                        # lakeColumn = scName.replace(' ','_')
+                        # elif cType == 'Calculated':
+                        pattern = r"\[([^]]+)\]"
+
+                        matches = re.findall(pattern, scName)
+                        lakeColumn = matches[0].replace(" ", "")
+                        try:
+                            tom.model.Tables[tName].Columns[cName]
+                        except:
+                            tom.add_data_column(
+                                table_name=tName,
+                                column_name=cName,
+                                source_column=lakeColumn,
+                                data_type=cDataType,
+                            )
+                            print(
+                                f"{icons.green_dot} The '{tName}'[{cName}] column has been added."
+                            )
+
+                print(
+                    f"\n{icons.green_dot} All viable calculated tables have been added to the model."
+                )
+
+        except Exception as e:
+            if datetime.datetime.now() - start_time > timeout:
+                break
+            time.sleep(1)

sempy_labs/migration/_migrate_model_objects_to_semantic_model.py

@@ -0,0 +1,524 @@
+import sempy
+import sempy.fabric as fabric
+import re, datetime, time
+from sempy_labs._list_functions import list_tables
+from sempy_labs._helper_functions import create_relationship_name
+from sempy_labs._tom import connect_semantic_model
+from typing import Optional
+from sempy._utils._log import log
+import sempy_labs._icons as icons
+
+
+@log
+def migrate_model_objects_to_semantic_model(
+    dataset: str,
+    new_dataset: str,
+    workspace: Optional[str] = None,
+    new_dataset_workspace: Optional[str] = None,
+):
+    """
+    Adds the rest of the model objects (besides tables/columns) and their properties to a Direct Lake semantic model based on an import/DirectQuery semantic model.
+
+    Parameters
+    ----------
+    dataset : str
+        Name of the import/DirectQuery semantic model.
+    new_dataset : str
+        Name of the Direct Lake semantic model.
+    workspace : str, default=None
+        The Fabric workspace name in which the import/DirectQuery semantic model exists.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    new_dataset_workspace : str
+        The Fabric workspace name in which the Direct Lake semantic model will be created.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    sempy.fabric._client._utils._init_analysis_services()
+    import Microsoft.AnalysisServices.Tabular as TOM
+    import System
+
+    if workspace == None:
+        workspace_id = fabric.get_workspace_id()
+        workspace = fabric.resolve_workspace_name(workspace_id)
+    else:
+        workspaceId = fabric.resolve_workspace_id(workspace)
+
+    if new_dataset_workspace == None:
+        new_dataset_workspace = workspace
+
+    dfT = list_tables(dataset, workspace)
+    dfC = fabric.list_columns(dataset=dataset, workspace=workspace)
+    dfM = fabric.list_measures(dataset=dataset, workspace=workspace)
+    dfR = fabric.list_relationships(dataset=dataset, workspace=workspace)
+    dfRole = fabric.get_roles(dataset=dataset, workspace=workspace)
+    dfRLS = fabric.get_row_level_security_permissions(
+        dataset=dataset, workspace=workspace
+    )
+    dfCI = fabric.list_calculation_items(dataset=dataset, workspace=workspace)
+    dfP = fabric.list_perspectives(dataset=dataset, workspace=workspace)
+    dfTranslation = fabric.list_translations(dataset=dataset, workspace=workspace)
+    dfH = fabric.list_hierarchies(dataset=dataset, workspace=workspace)
+    dfPar = fabric.list_partitions(dataset=dataset, workspace=workspace)
+
+    dfP_cc = dfPar[(dfPar["Source Type"] == "Calculated")]
+    dfP_fp = dfP_cc[dfP_cc["Query"].str.contains("NAMEOF")]
+    dfC_fp = dfC[dfC["Table Name"].isin(dfP_fp["Table Name"].values)]
+
+    print(f"{icons.in_progress} Updating '{new_dataset}' based on '{dataset}'...")
+    start_time = datetime.datetime.now()
+    timeout = datetime.timedelta(minutes=1)
+    success = False
+
+    while not success:
+        try:
+            with connect_semantic_model(
+                dataset=new_dataset, readonly=False, workspace=new_dataset_workspace
+            ) as tom:
+                success = True
+
+                isDirectLake = any(
+                    str(p.Mode) == "DirectLake"
+                    for t in tom.model.Tables
+                    for p in t.Partitions
+                )
+
+                print(f"\n{icons.in_progress} Updating table properties...")
+                for t in tom.model.Tables:
+                    t.IsHidden = bool(dfT.loc[dfT["Name"] == t.Name, "Hidden"].iloc[0])
+                    t.Description = dfT.loc[dfT["Name"] == t.Name, "Description"].iloc[
+                        0
+                    ]
+                    t.DataCategory = dfT.loc[
+                        dfT["Name"] == t.Name, "Data Category"
+                    ].iloc[0]
+
+                    print(
+                        f"{icons.green_dot} The '{t.Name}' table's properties have been updated."
+                    )
+
+                print(f"\n{icons.in_progress} Updating column properties...")
+                for t in tom.model.Tables:
+                    if (
+                        t.Name not in dfP_fp["Table Name"].values
+                    ):  # do not include field parameters
+                        dfT_filtered = dfT[dfT["Name"] == t.Name]
+                        tType = dfT_filtered["Type"].iloc[0]
+                        for c in t.Columns:
+                            if not c.Name.startswith("RowNumber-"):
+                                dfC_filt = dfC[
+                                    (dfC["Table Name"] == t.Name)
+                                    & (dfC["Column Name"] == c.Name)
+                                ]
+                                cName = dfC_filt["Column Name"].iloc[0]
+                                c.Name = cName
+                                if tType == "Table":
+                                    c.SourceColumn = cName.replace(" ", "_")
+                                c.IsHidden = bool(dfC_filt["Hidden"].iloc[0])
+                                c.DataType = System.Enum.Parse(
+                                    TOM.DataType, dfC_filt["Data Type"].iloc[0]
+                                )
+                                c.DisplayFolder = dfC_filt["Display Folder"].iloc[0]
+                                c.FormatString = dfC_filt["Format String"].iloc[0]
+                                c.SummarizeBy = System.Enum.Parse(
+                                    TOM.AggregateFunction,
+                                    dfC_filt["Summarize By"].iloc[0],
+                                )
+                                c.DataCategory = dfC_filt["Data Category"].iloc[0]
+                                c.IsKey = bool(dfC_filt["Key"].iloc[0])
+                                sbc = dfC_filt["Sort By Column"].iloc[0]
+
+                                if sbc != None:
+                                    try:
+                                        c.SortByColumn = tom.model.Tables[
+                                            t.Name
+                                        ].Columns[sbc]
+                                    except:
+                                        print(
+                                            f"{icons.red_dot} Failed to create '{sbc}' as a Sort By Column for the '{c.Name}' in the '{t.Name}' table."
+                                        )
+                                print(
+                                    f"{icons.green_dot} The '{t.Name}'[{c.Name}] column's properties have been updated."
+                                )
+
+                print(f"\n{icons.in_progress} Creating hierarchies...")
+                dfH_grouped = (
+                    dfH.groupby(
+                        [
+                            "Table Name",
+                            "Hierarchy Name",
+                            "Hierarchy Hidden",
+                            "Hierarchy Description",
+                        ]
+                    )
+                    .agg({"Level Name": list, "Column Name": list})
+                    .reset_index()
+                )
+
+                for i, r in dfH_grouped.iterrows():
+                    tName = r["Table Name"]
+                    hName = r["Hierarchy Name"]
+                    hDesc = r["Hierarchy Description"]
+                    hHid = bool(r["Hierarchy Hidden"])
+                    cols = r["Column Name"]
+                    lvls = r["Level Name"]
+
+                    try:
+                        tom.model.Tables[tName].Hierarchies[hName]
+                    except:
+                        tom.add_hierarchy(
+                            table_name=tName,
+                            hierarchy_name=hName,
+                            hierarchy_description=hDesc,
+                            hierarchy_hidden=hHid,
+                            columns=cols,
+                            levels=lvls,
+                        )
+                        print(
+                            f"{icons.green_dot} The '{hName}' hierarchy has been added."
+                        )
+
+                print(f"\n{icons.in_progress} Creating measures...")
+                for i, r in dfM.iterrows():
+                    tName = r["Table Name"]
+                    mName = r["Measure Name"]
+                    mExpr = r["Measure Expression"]
+                    mHidden = bool(r["Measure Hidden"])
+                    mDF = r["Measure Display Folder"]
+                    mDesc = r["Measure Description"]
+                    mFS = r["Format String"]
+
+                    try:
+                        tom.model.Tables[tName].Measures[mName]
+                    except:
+                        tom.add_measure(
+                            table_name=tName,
+                            measure_name=mName,
+                            expression=mExpr,
+                            hidden=mHidden,
+                            display_folder=mDF,
+                            description=mDesc,
+                            format_string=mFS,
+                        )
+                        print(
+                            f"{icons.green_dot} The '{mName}' measure has been added."
+                        )
+
+                for cgName in dfCI["Calculation Group Name"].unique():
+
+                    isHidden = bool(
+                        dfCI.loc[
+                            (dfCI["Calculation Group Name"] == cgName), "Hidden"
+                        ].iloc[0]
+                    )
+                    prec = int(
+                        dfCI.loc[
+                            (dfCI["Calculation Group Name"] == cgName), "Precedence"
+                        ].iloc[0]
+                    )
+                    desc = dfCI.loc[
+                        (dfCI["Calculation Group Name"] == cgName), "Description"
+                    ].iloc[0]
+
+                    try:
+                        tom.model.Tables[cgName]
+                    except:
+                        tom.add_calculation_group(
+                            name=cgName,
+                            description=desc,
+                            precedence=prec,
+                            hidden=isHidden,
+                        )
+                        print(
+                            f"{icons.green_dot} The '{cgName}' calculation group has been added."
+                        )
+                        tom.model.DiscourageImplicitMeasures = True
+
+                        print(
+                            f"\n{icons.in_progress} Updating calculation group column name..."
+                        )
+                        dfC_filt = dfC[
+                            (dfC["Table Name"] == cgName) & (dfC["Hidden"] == False)
+                        ]
+                        colName = dfC_filt["Column Name"].iloc[0]
+                        tom.model.Tables[cgName].Columns["Name"].Name = colName
+
+                    calcItems = dfCI.loc[
+                        dfCI["Calculation Group Name"] == cgName,
+                        "Calculation Item Name",
+                    ].unique()
+
+                    print(f"\n{icons.in_progress} Creating calculation items...")
+                    for calcItem in calcItems:
+                        ordinal = int(
+                            dfCI.loc[
+                                (dfCI["Calculation Group Name"] == cgName)
+                                & (dfCI["Calculation Item Name"] == calcItem),
+                                "Ordinal",
+                            ].iloc[0]
+                        )
+                        expr = dfCI.loc[
+                            (dfCI["Calculation Group Name"] == cgName)
+                            & (dfCI["Calculation Item Name"] == calcItem),
+                            "Expression",
+                        ].iloc[0]
+                        fse = dfCI.loc[
+                            (dfCI["Calculation Group Name"] == cgName)
+                            & (dfCI["Calculation Item Name"] == calcItem),
+                            "Format String Expression",
+                        ].iloc[0]
+                        try:
+                            tom.model.Tables[cgName].CalculationGroup.CalculationItems[
+                                calcItem
+                            ]
+                        except:
+                            tom.add_calculation_item(
+                                table_name=cgName,
+                                calculation_item_name=calcItem,
+                                expression=expr,
+                                format_string_expression=fse,
+                                ordinal=ordinal,
+                            )
+                            print(
+                                f"{icons.green_dot} The '{calcItem}' has been added to the '{cgName}' calculation group."
+                            )
+
+                print(f"\n{icons.in_progress} Creating relationships...")
+                for index, row in dfR.iterrows():
+                    fromTable = row["From Table"]
+                    fromColumn = row["From Column"]
+                    toTable = row["To Table"]
+                    toColumn = row["To Column"]
+                    isActive = row["Active"]
+                    cfb = row["Cross Filtering Behavior"]
+                    sfb = row["Security Filtering Behavior"]
+                    rori = row["Rely On Referential Integrity"]
+                    mult = row["Multiplicity"]
+
+                    card_mapping = {"m": "Many", "1": "One", "0": "None"}
+
+                    fromCard = card_mapping.get(mult[0])
+                    toCard = card_mapping.get(mult[-1])
+
+                    relName = create_relationship_name(
+                        fromTable, fromColumn, toTable, toColumn
+                    )
+
+                    if any(
+                        r.FromTable.Name == fromTable
+                        and r.FromColumn.Name == fromColumn
+                        and r.ToTable.Name == toTable
+                        and r.ToColumn.Name == toColumn
+                        for r in tom.model.Relationships
+                    ):
+                        print(
+                            f"{icons.yellow_dot} {relName} already exists as a relationship in the semantic model."
+                        )
+                    elif isDirectLake and any(
+                        r.FromTable.Name == fromTable
+                        and r.FromColumn.Name == fromColumn
+                        and r.ToTable.Name == toTable
+                        and r.ToColumn.Name == toColumn
+                        and (
+                            r.FromColumn.DataType == "DateTime"
+                            or r.ToColumn.DataType == "DateTime"
+                        )
+                        for r in tom.model.Relationships
+                    ):
+                        print(
+                            f"{icons.yellow_dot} {relName} was not created since relationships based on DateTime columns are not supported."
+                        )
+                    elif isDirectLake and any(
+                        r.FromTable.Name == fromTable
+                        and r.FromColumn.Name == fromColumn
+                        and r.ToTable.Name == toTable
+                        and r.ToColumn.Name == toColumn
+                        and (r.FromColumn.DataType != r.ToColumn.DataType)
+                        for r in tom.model.Relationships
+                    ):
+                        print(
+                            f"{icons.yellow_dot} {relName} was not created since columns used in a relationship must have the same data type."
+                        )
+                    else:
+                        try:
+                            tom.add_relationship(
+                                from_table=fromTable,
+                                from_column=fromColumn,
+                                to_table=toTable,
+                                to_column=toColumn,
+                                from_cardinality=fromCard,
+                                to_cardinality=toCard,
+                                cross_filtering_behavior=cfb,
+                                security_filtering_behavior=sfb,
+                                rely_on_referential_integrity=rori,
+                                is_active=isActive,
+                            )
+
+                            print(
+                                f"{icons.green_dot} The {relName} relationship has been added."
+                            )
+                        except:
+                            print(
+                                f"{icons.red_dot} The {relName} relationship was not added."
+                            )
+
+                print(f"\n{icons.in_progress} Creating roles...")
+                for index, row in dfRole.iterrows():
+                    roleName = row["Role"]
+                    roleDesc = row["Description"]
+                    modPerm = row["Model Permission"]
+
+                    try:
+                        tom.model.Roles[roleName]
+                    except:
+                        tom.add_role(
+                            role_name=roleName,
+                            model_permission=modPerm,
+                            description=roleDesc,
+                        )
+                        print(
+                            f"{icons.green_dot} The '{roleName}' role has been added."
+                        )
+
+                print(f"\n{icons.in_progress} Creating row level security...")
+                for index, row in dfRLS.iterrows():
+                    roleName = row["Role"]
+                    tName = row["Table"]
+                    expr = row["Filter Expression"]
+
+                    try:
+                        tom.set_rls(
+                            role_name=roleName, table_name=tName, filter_expression=expr
+                        )
+                        print(
+                            f"{icons.green_dot} Row level security for the '{tName}' table within the '{roleName}' role has been set."
+                        )
+                    except:
+                        print(
+                            f"{icons.red_dot} Row level security for the '{tName}' table within the '{roleName}' role was not set."
+                        )
+
+                print(f"\n{icons.in_progress} Creating perspectives...")
+                for pName in dfP["Perspective Name"].unique():
+
+                    try:
+                        tom.model.Perspectives[pName]
+                    except:
+                        tom.add_perspective(perspective_name=pName)
+                        print(
+                            f"{icons.green_dot} The '{pName}' perspective has been added."
+                        )
+
+                print(f"\n{icons.in_progress} Adding objects to perspectives...")
+                for index, row in dfP.iterrows():
+                    pName = row["Perspective Name"]
+                    tName = row["Table Name"]
+                    oName = row["Object Name"]
+                    oType = row["Object Type"]
+                    tType = dfT.loc[(dfT["Name"] == tName), "Type"].iloc[0]
+
+                    try:
+                        if oType == "Table":
+                            tom.add_to_perspective(
+                                object=tom.model.Tables[tName], perspective_name=pName
+                            )
+                        elif oType == "Column":
+                            tom.add_to_perspective(
+                                object=tom.model.Tables[tName].Columns[oName],
+                                perspective_name=pName,
+                            )
+                        elif oType == "Measure":
+                            tom.add_to_perspective(
+                                object=tom.model.Tables[tName].Measures[oName],
+                                perspective_name=pName,
+                            )
+                        elif oType == "Hierarchy":
+                            tom.add_to_perspective(
+                                object=tom.model.Tables[tName].Hierarchies[oName],
+                                perspective_name=pName,
+                            )
+                    except:
+                        pass
+
+                print(f"\n{icons.in_progress} Creating translation languages...")
+                for trName in dfTranslation["Culture Name"].unique():
+                    try:
+                        tom.model.Cultures[trName]
+                    except:
+                        tom.add_translation(trName)
+                        print(
+                            f"{icons.green_dot} The '{trName}' translation language has been added."
+                        )
+
+                print(f"\n{icons.in_progress} Creating translation values...")
+                for index, row in dfTranslation.iterrows():
+                    trName = row["Culture Name"]
+                    tName = row["Table Name"]
+                    oName = row["Object Name"]
+                    oType = row["Object Type"]
+                    translation = row["Translation"]
+                    prop = row["Property"]
+
+                    if prop == "Caption":
+                        prop = "Name"
+                    elif prop == "DisplayFolder":
+                        prop = "Display Folder"
+
+                    try:
+                        if oType == "Table":
+                            tom.set_translation(
+                                object=tom.model.Tables[tName],
+                                language=trName,
+                                property=prop,
+                                value=translation,
+                            )
+                        elif oType == "Column":
+                            tom.set_translation(
+                                object=tom.model.Tables[tName].Columns[oName],
+                                language=trName,
+                                property=prop,
+                                value=translation,
+                            )
+                        elif oType == "Measure":
+                            tom.set_translation(
+                                object=tom.model.Tables[tName].Measures[oName],
+                                language=trName,
+                                property=prop,
+                                value=translation,
+                            )
+                        elif oType == "Hierarchy":
+                            tom.set_translation(
+                                object=tom.model.Tables[tName].Hierarchies[oName],
+                                language=trName,
+                                property=prop,
+                                value=translation,
+                            )
+                        elif oType == "Level":
+
+                            pattern = r"\[([^]]+)\]"
+                            matches = re.findall(pattern, oName)
+                            lName = matches[0]
+
+                            pattern = r"'([^']+)'"
+                            matches = re.findall(pattern, oName)
+                            hName = matches[0]
+                            tom.set_translation(
+                                object=tom.model.Tables[tName]
+                                .Hierarchies[hName]
+                                .Levels[lName],
+                                language=trName,
+                                property=prop,
+                                value=translation,
+                            )
+                    except:
+                        pass
+
+                print(
+                    f"\n{icons.green_dot} Migration of objects from '{dataset}' -> '{new_dataset}' is complete."
+                )
+
+        except Exception as e:
+            if datetime.datetime.now() - start_time > timeout:
+                break
+            time.sleep(1)