semantic-link-labs 0.12.8__py3-none-any.whl

sempy_labs/migration/_migrate_tables_columns_to_semantic_model.py

@@ -0,0 +1,172 @@
+import sempy.fabric as fabric
+import pandas as pd
+from sempy_labs.directlake._generate_shared_expression import generate_shared_expression
+from sempy_labs._helper_functions import retry
+from sempy_labs.lakehouse._lakehouse import lakehouse_attached
+from sempy_labs.tom import connect_semantic_model
+from typing import Optional
+from sempy._utils._log import log
+import sempy_labs._icons as icons
+from uuid import UUID
+
+
+@log
+def migrate_tables_columns_to_semantic_model(
+    dataset: str,
+    new_dataset: str,
+    workspace: Optional[str | UUID] = None,
+    new_dataset_workspace: Optional[str | UUID] = None,
+    lakehouse: Optional[str | UUID] = None,
+    lakehouse_workspace: Optional[str | UUID] = None,
+):
+    """
+    Adds tables/columns to the new Direct Lake semantic model based on an import/DirectQuery semantic model.
+
+    Parameters
+    ----------
+    dataset : str
+        Name of the import/DirectQuery semantic model.
+    new_dataset : str
+        Name of the Direct Lake semantic model.
+    workspace : str, default=None
+        The Fabric workspace name in which the import/DirectQuery semantic model exists.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    new_dataset_workspace : str
+        The Fabric workspace name in which the Direct Lake semantic model will be created.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    lakehouse : str, default=None
+        The Fabric lakehouse used by the Direct Lake semantic model.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    lakehouse_workspace : str, default=None
+        The Fabric workspace used by the lakehouse.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    if dataset == new_dataset:
+        raise ValueError(
+            f"{icons.red_dot} The 'dataset' and 'new_dataset' parameters are both set to '{dataset}'. These parameters must be set to different values."
+        )
+
+    icons.sll_tags.append("DirectLakeMigration")
+
+    # Check that lakehouse is attached to the notebook
+    if not lakehouse_attached() and (lakehouse is None and lakehouse_workspace is None):
+        raise ValueError(
+            f"{icons.red_dot} Lakehouse not attached to notebook and lakehouse/lakehouse_workspace are not specified. Please add your lakehouse to this notebook"
+            f" or specify the lakehouse/lakehouse_workspace parameters."
+            "To attach a lakehouse to a notebook, go to the the 'Explorer' window to the left, click 'Lakehouses' to add your lakehouse to this notebook"
+            "\nLearn more here: https://learn.microsoft.com/fabric/data-engineering/lakehouse-notebook-explore#add-or-remove-a-lakehouse"
+        )
+    shEx = generate_shared_expression(
+        item_name=lakehouse,
+        item_type="Lakehouse",
+        workspace=lakehouse_workspace,
+        use_sql_endpoint=False,
+    )
+
+    fabric.refresh_tom_cache(workspace=workspace)
+    dfC = fabric.list_columns(dataset=dataset, workspace=workspace)
+    dfT = fabric.list_tables(dataset=dataset, workspace=workspace)
+    dfT.rename(columns={"Type": "Table Type"}, inplace=True)
+    dfC = pd.merge(
+        dfC,
+        dfT[["Name", "Table Type"]],
+        left_on="Table Name",
+        right_on="Name",
+        how="left",
+    )
+    dfT_filt = dfT[dfT["Table Type"] == "Table"]
+    dfC_filt = dfC[
+        (dfC["Table Type"] == "Table")
+        & ~(dfC["Column Name"].str.startswith("RowNumber-"))
+        & (dfC["Type"] != "Calculated")
+    ]
+
+    print(f"{icons.in_progress} Updating '{new_dataset}' based on '{dataset}'...")
+
+    @retry(
+        sleep_time=1,
+        timeout_error_message=f"{icons.red_dot} Function timed out after 1 minute",
+    )
+    def dyn_connect():
+        with connect_semantic_model(
+            dataset=new_dataset, readonly=True, workspace=new_dataset_workspace
+        ) as tom:
+
+            tom.model
+
+    dyn_connect()
+
+    with connect_semantic_model(
+        dataset=new_dataset, readonly=False, workspace=new_dataset_workspace
+    ) as tom:
+
+        # Additional updates
+        tom.set_annotation(
+            object=tom.model, name="__PBI_TimeIntelligenceEnabled", value="0"
+        )
+        tom.set_annotation(
+            object=tom.model, name="PBI_QueryOrder", value='["DatabaseQuery"]'
+        )
+
+        # Begin migration
+        if not any(e.Name == "DatabaseQuery" for e in tom.model.Expressions):
+            tom.add_expression("DatabaseQuery", expression=shEx)
+            print(f"{icons.green_dot} The 'DatabaseQuery' expression has been added.")
+            tom.set_annotation(
+                object=tom.model.Expressions["DatabaseQuery"],
+                name="PBI_IncludeFutureArtifacts",
+                value="False",
+            )
+
+        for i, r in dfT_filt.iterrows():
+            tName = r["Name"]
+            tDC = r["Data Category"]
+            tHid = bool(r["Hidden"])
+            tDesc = r["Description"]
+            ent_name = tName  # .replace(" ", "_")
+            for char in icons.special_characters:
+                ent_name = ent_name.replace(char, "")
+
+            if not any(t.Name == tName for t in tom.model.Tables):
+                tom.add_table(
+                    name=tName,
+                    description=tDesc,
+                    data_category=tDC,
+                    hidden=tHid,
+                )
+                tom.add_entity_partition(table_name=tName, entity_name=ent_name)
+                print(f"{icons.green_dot} The '{tName}' table has been added.")
+
+        for i, r in dfC_filt.iterrows():
+            tName = r["Table Name"]
+            cName = r["Column Name"]
+            scName = r["Source"].replace(" ", "_")
+            cHid = bool(r["Hidden"])
+            cDataType = r["Data Type"]
+            for char in icons.special_characters:
+                scName = scName.replace(char, "")
+
+            if scName.endswith("_"):
+                scName = scName[:-1]
+
+            if not any(
+                c.Name == cName and c.Parent.Name == tName for c in tom.all_columns()
+            ):
+                tom.add_data_column(
+                    table_name=tName,
+                    column_name=cName,
+                    source_column=scName,
+                    hidden=cHid,
+                    data_type=cDataType,
+                )
+                print(
+                    f"{icons.green_dot} The '{tName}'[{cName}] column has been added."
+                )
+
+        print(
+            f"\n{icons.green_dot} All regular tables and columns have been added to the '{new_dataset}' semantic model."
+        )

sempy_labs/migration/_migration_validation.py

@@ -0,0 +1,71 @@
+import pandas as pd
+from typing import Optional
+from .._list_functions import list_semantic_model_objects
+from sempy._utils._log import log
+import sempy_labs._icons as icons
+
+
+@log
+def migration_validation(
+    dataset: str,
+    new_dataset: str,
+    workspace: Optional[str] = None,
+    new_dataset_workspace: Optional[str] = None,
+) -> pd.DataFrame:
+    """
+    Shows the objects in the original semantic model and whether then were migrated successfully or not.
+
+    Parameters
+    ----------
+    dataset : str
+        Name of the import/DirectQuery semantic model.
+    new_dataset : str
+        Name of the Direct Lake semantic model.
+    workspace : str, default=None
+        The Fabric workspace name in which the import/DirectQuery semantic model exists.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    new_dataset_workspace : str
+        The Fabric workspace name in which the Direct Lake semantic model will be created.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    pandas.DataFrame
+       A pandas dataframe showing a list of objects and whether they were successfully migrated. Also shows the % of objects which were migrated successfully.
+    """
+
+    if dataset == new_dataset:
+        raise ValueError(
+            f"{icons.red_dot} The 'dataset' and 'new_dataset' parameters are both set to '{dataset}'. These parameters must be set to different values."
+        )
+
+    icons.sll_tags.append("DirectLakeMigration")
+
+    dfA = list_semantic_model_objects(dataset=dataset, workspace=workspace)
+    dfB = list_semantic_model_objects(
+        dataset=new_dataset, workspace=new_dataset_workspace
+    )
+
+    def is_migrated(row):
+        if row["Object Type"] == "Calculated Table":
+            return (
+                (dfB["Parent Name"] == row["Parent Name"])
+                & (dfB["Object Name"] == row["Object Name"])
+                & (dfB["Object Type"].isin(["Calculated Table", "Table"]))
+            ).any()
+        else:
+            return (
+                (dfB["Parent Name"] == row["Parent Name"])
+                & (dfB["Object Name"] == row["Object Name"])
+                & (dfB["Object Type"] == row["Object Type"])
+            ).any()
+
+    dfA["Migrated"] = dfA.apply(is_migrated, axis=1)
+
+    denom = len(dfA)
+    num = len(dfA[dfA["Migrated"]])
+    print(f"{100 * round(num / denom,2)}% migrated")
+
+    return dfA

sempy_labs/migration/_refresh_calc_tables.py

@@ -0,0 +1,131 @@
+import sempy.fabric as fabric
+import pandas as pd
+import re
+from ..tom import connect_semantic_model
+from typing import Optional
+from sempy._utils._log import log
+import sempy_labs._icons as icons
+from uuid import UUID
+from .._helper_functions import (
+    resolve_workspace_name_and_id,
+    resolve_dataset_name_and_id,
+    save_as_delta_table,
+    retry,
+)
+
+
+@log
+def refresh_calc_tables(dataset: str | UUID, workspace: Optional[str | UUID] = None):
+    """
+    Recreates the delta tables in the lakehouse based on the DAX expressions stored as model annotations in the Direct Lake semantic model.
+
+    Parameters
+    ----------
+    dataset : str | UUID
+        Name or ID of the semantic model.
+    workspace : str | UUID, default=None
+        The Fabric workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    (dataset_name, dataset_id) = resolve_dataset_name_and_id(dataset, workspace_id)
+    icons.sll_tags.append("DirectLakeMigration")
+
+    @retry(
+        sleep_time=1,
+        timeout_error_message=f"{icons.red_dot} Function timed out after 1 minute",
+    )
+    def dyn_connect():
+        with connect_semantic_model(
+            dataset=dataset_id, readonly=True, workspace=workspace_id
+        ) as tom:
+
+            tom.model
+
+    dyn_connect()
+
+    with connect_semantic_model(
+        dataset=dataset_id, readonly=True, workspace=workspace_id
+    ) as tom:
+        for a in tom.model.Annotations:
+            if any(a.Name == t.Name for t in tom.model.Tables):
+                tName = a.Name
+                query = a.Value
+
+                if not query.startswith("EVALUATE"):
+                    daxquery = "EVALUATE \n" + query
+                else:
+                    daxquery = query
+
+                try:
+                    df = fabric.evaluate_dax(
+                        dataset=dataset_id,
+                        dax_string=daxquery,
+                        workspace=workspace_id,
+                    )
+
+                    # Update column names for non-field parameters
+                    if query.find("NAMEOF") == -1:
+                        for old_column_name in df.columns:
+                            pattern = r"\[([^\]]+)\]"
+
+                            matches = re.findall(pattern, old_column_name)
+                            new_column_name = matches[0]
+                            new_column_name = new_column_name.replace(" ", "")
+
+                            df.rename(
+                                columns={old_column_name: new_column_name},
+                                inplace=True,
+                            )
+
+                            # Update data types for lakehouse columns
+                            dataType = next(
+                                str(c.DataType)
+                                for c in tom.all_columns()
+                                if c.Parent.Name == tName
+                                and c.SourceColumn == new_column_name
+                            )
+                            # dfC_type = dfC[(dfC['Table Name'] == tName) & (dfC['Source'] == new_column_name)]
+                            # dataType = dfC_type['Data Type'].iloc[0]
+
+                            if dataType == "Int64":
+                                df[new_column_name] = df[new_column_name].astype(int)
+                            elif dataType in ["Decimal", "Double"]:
+                                df[new_column_name] = df[new_column_name].astype(float)
+                            elif dataType == "Boolean":
+                                df[new_column_name] = df[new_column_name].astype(bool)
+                            elif dataType == "DateTime":
+                                df[new_column_name] = pd.to_datetime(
+                                    df[new_column_name]
+                                )
+                            else:
+                                df[new_column_name] = df[new_column_name].astype(str)
+                    # else:
+                    #    second_column_name = df.columns[1]
+                    #    third_column_name = df.columns[2]
+                    #    df[third_column_name] = df[third_column_name].astype(int)
+
+                    # Remove calc columns from field parameters
+                    #    mask = df[second_column_name].isin(dfC_filt['Full Column Name'])
+                    #    df = df[~mask]
+
+                    delta_table_name = tName.replace(" ", "_")
+                    print(
+                        f"{icons.in_progress} Refresh of the '{delta_table_name}' table within the lakehouse is in progress..."
+                    )
+
+                    save_as_delta_table(
+                        dataframe=df,
+                        table_name=delta_table_name,
+                        write_mode="overwrite",
+                    )
+
+                    print(
+                        f"{icons.green_dot} Calculated table '{tName}' has been refreshed as the '{delta_table_name.lower()}' table in the lakehouse."
+                    )
+                except Exception as e:
+                    raise ValueError(
+                        f"{icons.red_dot} Failed to create calculated table '{tName}' as a delta table in the lakehouse."
+                    ) from e

sempy_labs/mirrored_azure_databricks_catalog/__init__.py

@@ -0,0 +1,15 @@
+from ._refresh_catalog_metadata import (
+    refresh_catalog_metadata,
+)
+from ._discover import (
+    discover_catalogs,
+    discover_schemas,
+    discover_tables,
+)
+
+__all__ = [
+    "refresh_catalog_metadata",
+    "discover_catalogs",
+    "discover_schemas",
+    "discover_tables",
+]

sempy_labs/mirrored_azure_databricks_catalog/_discover.py

@@ -0,0 +1,213 @@
+from uuid import UUID
+from typing import Optional
+from sempy_labs._helper_functions import (
+    resolve_workspace_id,
+    _base_api,
+    _create_dataframe,
+)
+import pandas as pd
+from sempy._utils._log import log
+
+
+@log
+def discover_catalogs(
+    databricks_workspace_connection_id: UUID,
+    workspace: Optional[str | UUID] = None,
+    max_results: Optional[int] = None,
+) -> pd.DataFrame:
+    """
+    Returns a list of catalogs from Unity Catalog.
+
+    This is a wrapper function for the following API: `Databricks Metadata Discovery - Discover Catalogs <https://learn.microsoft.comrest/api/fabric/mirroredazuredatabrickscatalog/databricks-metadata-discovery/discover-catalogs>`_.
+
+    Parameters
+    ----------
+    databricks_workspace_connection_id : uuid.UUID
+        The ID of the Databricks workspace connection.
+    workspace : str | uuid.UUID, default=None
+        The workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    max_results : int, default=None
+        The maximum number of results to return. If not specified, all results are returned.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing a list of catalogs from Unity Catalog.
+    """
+
+    workspace_id = resolve_workspace_id(workspace)
+
+    url = f"/v1/workspaces/{workspace_id}/azuredatabricks/catalogs?databricksWorkspaceConnectionId={databricks_workspace_connection_id}"
+    if max_results:
+        url += f"&maxResults={max_results}"
+
+    responses = _base_api(request=url, uses_pagination=True)
+
+    columns = {
+        "Catalog Name": "str",
+        "Catalog Full Name": "str",
+        "Catalog Type": "str",
+        "Storage Location": "str",
+    }
+
+    df = _create_dataframe(columns=columns)
+
+    rows = []
+    for r in responses:
+        for i in r.get("value", []):
+            rows.append(
+                {
+                    "Catalog Name": i.get("name"),
+                    "Catalog Full Name": i.get("fullName"),
+                    "Catalog Type": i.get("catalogType"),
+                    "Storage Location": i.get("storageLocation"),
+                }
+            )
+
+    if rows:
+        df = pd.DataFrame(rows, columns=list(columns.keys()))
+
+    return df
+
+
+@log
+def discover_schemas(
+    catalog: str,
+    databricks_workspace_connection_id: UUID,
+    workspace: Optional[str | UUID] = None,
+    max_results: Optional[int] = None,
+) -> pd.DataFrame:
+    """
+    Returns a list of schemas in the given catalog from Unity Catalog.
+
+    This is a wrapper function for the following API: `Databricks Metadata Discovery - Discover Schemas <https://learn.microsoft.comrest/api/fabric/mirroredazuredatabrickscatalog/databricks-metadata-discovery/discover-schemas>`_.
+
+    Parameters
+    ----------
+    catalog : str
+        The name of the catalog.
+    databricks_workspace_connection_id : uuid.UUID
+        The ID of the Databricks workspace connection.
+    workspace : str | uuid.UUID, default=None
+        The workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    max_results : int, default=None
+        The maximum number of results to return. If not specified, all results are returned.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing a list of schemas in the given catalog from Unity Catalog.
+    """
+
+    workspace_id = resolve_workspace_id(workspace)
+
+    url = f"/v1/workspaces/{workspace_id}/azuredatabricks/catalogs/{catalog}/schemas?databricksWorkspaceConnectionId={databricks_workspace_connection_id}"
+    if max_results:
+        url += f"&maxResults={max_results}"
+
+    responses = _base_api(request=url, uses_pagination=True)
+
+    columns = {
+        "Catalog Name": "str",
+        "Schema Name": "str",
+        "Schema Full Name": "str",
+        "Storage Location": "str",
+    }
+
+    df = _create_dataframe(columns=columns)
+
+    rows = []
+    for r in responses:
+        for i in r.get("value", []):
+            rows.append(
+                {
+                    "Catalog Name": catalog,
+                    "Schema Name": i.get("name"),
+                    "Schema Full Name": i.get("fullName"),
+                    "Storage Location": i.get("storageLocation"),
+                }
+            )
+
+    if rows:
+        df = pd.DataFrame(rows, columns=list(columns.keys()))
+
+    return df
+
+
+@log
+def discover_tables(
+    catalog: str,
+    schema: str,
+    databricks_workspace_connection_id: UUID,
+    workspace: Optional[str | UUID] = None,
+    max_results: Optional[int] = None,
+) -> pd.DataFrame:
+    """
+    Returns a list of schemas in the given catalog from Unity Catalog.
+
+    This is a wrapper function for the following API: `Databricks Metadata Discovery - Discover Tables <https://learn.microsoft.comrest/api/fabric/mirroredazuredatabrickscatalog/databricks-metadata-discovery/discover-tables>`_.
+
+    Parameters
+    ----------
+    catalog : str
+        The name of the catalog.
+    schema : str
+        The name of the schema.
+    databricks_workspace_connection_id : uuid.UUID
+        The ID of the Databricks workspace connection.
+    workspace : str | uuid.UUID, default=None
+        The workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    max_results : int, default=None
+        The maximum number of results to return. If not specified, all results are returned.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing a list of schemas in the given catalog from Unity Catalog.
+    """
+
+    workspace_id = resolve_workspace_id(workspace)
+
+    url = f"/v1/workspaces/{workspace_id}/azuredatabricks/catalogs/{catalog}/schemas/{schema}/tables?databricksWorkspaceConnectionId={databricks_workspace_connection_id}"
+    if max_results:
+        url += f"&maxResults={max_results}"
+
+    responses = _base_api(request=url, uses_pagination=True)
+
+    columns = {
+        "Catalog Name": "str",
+        "Schema Name": "str",
+        "Table Name": "str",
+        "Table Full Name": "str",
+        "Storage Location": "str",
+        "Table Type": "str",
+        "Data Source Format": "str",
+    }
+
+    df = _create_dataframe(columns=columns)
+
+    rows = []
+    for r in responses:
+        for i in r.get("value", []):
+            rows.append(
+                {
+                    "Catalog Name": catalog,
+                    "Schema Name": schema,
+                    "Table Name": i.get("name"),
+                    "Table Full Name": i.get("fullName"),
+                    "Storage Location": i.get("storageLocation"),
+                    "Table Type": i.get("tableType"),
+                    "Data Source Format": i.get("dataSourceFormat"),
+                }
+            )
+
+    if rows:
+        df = pd.DataFrame(rows, columns=list(columns.keys()))
+
+    return df

sempy_labs/mirrored_azure_databricks_catalog/_refresh_catalog_metadata.py

@@ -0,0 +1,45 @@
+from uuid import UUID
+from typing import Optional
+from sempy_labs._helper_functions import (
+    resolve_workspace_name_and_id,
+    resolve_item_name_and_id,
+    _base_api,
+)
+import sempy_labs._icons as icons
+from sempy._utils._log import log
+
+
+@log
+def refresh_catalog_metadata(
+    mirrored_azure_databricks_catalog: str | UUID,
+    workspace: Optional[str | UUID] = None,
+):
+    """
+    Refresh Databricks catalog metadata in mirroredAzureDatabricksCatalogs Item.
+
+    This is a wrapper function for the following API: `Refresh Metadata - Items RefreshCatalogMetadata <https://learn.microsoft.com/rest/api/fabric/mirroredazuredatabrickscatalog/refresh-metadata/items-refresh-catalog-metadata>`_.
+
+    Parameters
+    ----------
+    mirrored_azure_databricks_catalog : str | uuid.UUID
+        The name or ID of the mirrored Azure Databricks catalog.
+    workspace : str | uuie.UUID, default=None
+        The workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook
+    """
+
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    (catalog_name, catalog_id) = resolve_item_name_and_id(
+        mirrored_azure_databricks_catalog
+    )
+
+    _base_api(
+        request=f"/v1/workspaces/{workspace_id}/mirroredAzureDatabricksCatalogs/{catalog_id}/refreshCatalogMetadata",
+        method="post",
+        lro_return_status_code=True,
+    )
+
+    print(
+        f"{icons.green_dot} The '{catalog_name}' Databricks Catalog metadata within the '{workspace_name}' workspace has been refreshed."
+    )

sempy_labs/ml_model/__init__.py

@@ -0,0 +1,23 @@
+from ._functions import (
+    list_ml_models,
+    create_ml_model,
+    delete_ml_model,
+    activate_ml_model_endpoint_version,
+    deactivate_all_ml_model_endpoint_versions,
+    deactivate_ml_model_endpoint_version,
+    list_ml_model_endpoint_versions,
+    score_ml_model_endpoint,
+    score_ml_model_endpoint_version,
+)
+
+__all__ = [
+    "list_ml_models",
+    "create_ml_model",
+    "delete_ml_model",
+    "activate_ml_model_endpoint_version",
+    "deactivate_all_ml_model_endpoint_versions",
+    "deactivate_ml_model_endpoint_version",
+    "list_ml_model_endpoint_versions",
+    "score_ml_model_endpoint",
+    "score_ml_model_endpoint_version",
+]