semantic-link-labs 0.4.1__py3-none-any.whl

sempy_labs/lakehouse/_get_lakehouse_tables.py

@@ -0,0 +1,250 @@
+import sempy.fabric as fabric
+import pandas as pd
+from pyspark.sql import SparkSession
+import pyarrow.parquet as pq
+import datetime
+from sempy_labs._helper_functions import (
+    resolve_lakehouse_id,
+    resolve_lakehouse_name,
+    resolve_workspace_name_and_id,
+)
+from sempy_labs.directlake._guardrails import (
+    get_sku_size,
+    get_directlake_guardrails_for_sku,
+)
+from sempy_labs.lakehouse._lakehouse import lakehouse_attached
+from typing import Optional
+
+
+def get_lakehouse_tables(
+    lakehouse: Optional[str] = None,
+    workspace: Optional[str] = None,
+    extended: Optional[bool] = False,
+    count_rows: Optional[bool] = False,
+    export: Optional[bool] = False,
+):
+    """
+    Shows the tables of a lakehouse and their respective properties. Option to include additional properties relevant to Direct Lake guardrails.
+
+    Parameters
+    ----------
+    lakehouse : str, default=None
+        The Fabric lakehouse.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    lakehouse_workspace : str, default=None
+        The Fabric workspace used by the lakehouse.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    extended : bool, default=False
+        Obtains additional columns relevant to the size of each table.
+    count_rows : bool, default=False
+        Obtains a row count for each lakehouse table.
+    export : bool, default=False
+        Exports the resulting dataframe to a delta table in the lakehouse.
+
+    Returns
+    -------
+    pandas.DataFrame
+        Shows the tables/columns within a lakehouse and their properties.
+    """
+
+    df = pd.DataFrame(
+        columns=[
+            "Workspace Name",
+            "Lakehouse Name",
+            "Table Name",
+            "Format",
+            "Type",
+            "Location",
+        ]
+    )
+
+    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
+
+    if lakehouse == None:
+        lakehouse_id = fabric.get_lakehouse_id()
+        lakehouse = resolve_lakehouse_name(lakehouse_id, workspace)
+    else:
+        lakehouse_id = resolve_lakehouse_id(lakehouse, workspace)
+
+    if count_rows:  # Setting countrows defaults to extended=True
+        extended = True
+
+    client = fabric.FabricRestClient()
+    response = client.get(
+        f"/v1/workspaces/{workspace_id}/lakehouses/{lakehouse_id}/tables"
+    )
+
+    for i in response.json()["data"]:
+        tName = i["name"]
+        tType = i["type"]
+        tFormat = i["format"]
+        tLocation = i["location"]
+        if extended == False:
+            new_data = {
+                "Workspace Name": workspace,
+                "Lakehouse Name": lakehouse,
+                "Table Name": tName,
+                "Format": tFormat,
+                "Type": tType,
+                "Location": tLocation,
+            }
+            df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
+        else:
+            sku_value = get_sku_size(workspace)
+            guardrail = get_directlake_guardrails_for_sku(sku_value)
+
+            spark = SparkSession.builder.getOrCreate()
+
+            intColumns = ["Files", "Row Groups", "Table Size"]
+            if tType == "Managed" and tFormat == "delta":
+                detail_df = spark.sql(f"DESCRIBE DETAIL `{tName}`").collect()[0]
+                num_files = detail_df.numFiles
+                size_in_bytes = detail_df.sizeInBytes
+
+                delta_table_path = f"Tables/{tName}"
+                latest_files = (
+                    spark.read.format("delta").load(delta_table_path).inputFiles()
+                )
+                file_paths = [f.split("/")[-1] for f in latest_files]
+
+                # Handle FileNotFoundError
+                num_rowgroups = 0
+                for filename in file_paths:
+                    try:
+                        num_rowgroups += pq.ParquetFile(
+                            f"/lakehouse/default/{delta_table_path}/{filename}"
+                        ).num_row_groups
+                    except FileNotFoundError:
+                        continue
+
+                if count_rows:
+                    num_rows = spark.table(tName).count()
+                    intColumns.append("Row Count")
+                    new_data = {
+                        "Workspace Name": workspace,
+                        "Lakehouse Name": lakehouse,
+                        "Table Name": tName,
+                        "Format": tFormat,
+                        "Type": tType,
+                        "Location": tLocation,
+                        "Files": num_files,
+                        "Row Groups": num_rowgroups,
+                        "Row Count": num_rows,
+                        "Table Size": size_in_bytes,
+                    }
+                else:
+                    new_data = {
+                        "Workspace Name": workspace,
+                        "Lakehouse Name": lakehouse,
+                        "Table Name": tName,
+                        "Format": tFormat,
+                        "Type": tType,
+                        "Location": tLocation,
+                        "Files": num_files,
+                        "Row Groups": num_rowgroups,
+                        "Table Size": size_in_bytes,
+                    }
+
+                df = pd.concat(
+                    [df, pd.DataFrame(new_data, index=[0])], ignore_index=True
+                )
+                df[intColumns] = df[intColumns].astype(int)
+
+            df["SKU"] = guardrail["Fabric SKUs"].iloc[0]
+            df["Parquet File Guardrail"] = guardrail["Parquet files per table"].iloc[0]
+            df["Row Group Guardrail"] = guardrail["Row groups per table"].iloc[0]
+            df["Row Count Guardrail"] = (
+                guardrail["Rows per table (millions)"].iloc[0] * 1000000
+            )
+
+            df["Parquet File Guardrail Hit"] = (
+                df["Files"] > df["Parquet File Guardrail"]
+            )
+            df["Row Group Guardrail Hit"] = df["Row Groups"] > df["Row Group Guardrail"]
+
+            if count_rows:
+                df["Row Count Guardrail Hit"] = (
+                    df["Row Count"] > df["Row Count Guardrail"]
+                )
+
+    if export:
+        lakeAttach = lakehouse_attached()
+        if lakeAttach == False:
+            print(
+                f"In order to save the report.json file, a lakehouse must be attached to the notebook. Please attach a lakehouse to this notebook."
+            )
+            return
+        spark = SparkSession.builder.getOrCreate()
+
+        lakehouse_id = fabric.get_lakehouse_id()
+        lakehouse = resolve_lakehouse_name(
+            lakehouse_id=lakehouse_id, workspace=workspace
+        )
+        lakeTName = "lakehouse_table_details"
+        lakeT_filt = df[df["Table Name"] == lakeTName]
+
+        query = f"SELECT MAX(RunId) FROM {lakehouse}.{lakeTName}"
+
+        if len(lakeT_filt) == 0:
+            runId = 1
+        else:
+            dfSpark = spark.sql(query)
+            maxRunId = dfSpark.collect()[0][0]
+            runId = maxRunId + 1
+
+        export_df = df.copy()
+
+        cols = [
+            "Files",
+            "Row Groups",
+            "Row Count",
+            "Table Size",
+            "SKU",
+            "Parquet File Guardrail",
+            "Row Group Guardrail",
+            "Row Count Guardrail",
+            "Parquet File Guardrail Hit",
+            "Row Group Guardrail Hit",
+            "Row Count Guardrail Hit",
+        ]
+
+        for c in cols:
+            if c not in export_df:
+                if c in [
+                    "Files",
+                    "Row Groups",
+                    "Row Count",
+                    "Table Size",
+                    "Parquet File Guardrail",
+                    "Row Group Guardrail",
+                    "Row Count Guardrail",
+                ]:
+                    export_df[c] = 0
+                    export_df[c] = export_df[c].astype(int)
+                elif c in ["SKU"]:
+                    export_df[c] = None
+                    export_df[c] = export_df[c].astype(str)
+                elif c in [
+                    "Parquet File Guardrail Hit",
+                    "Row Group Guardrail Hit",
+                    "Row Count Guardrail Hit",
+                ]:
+                    export_df[c] = False
+                    export_df[c] = export_df[c].astype(bool)
+
+        print(
+            f"Saving Lakehouse table properties to the '{lakeTName}' table in the lakehouse...\n"
+        )
+        now = datetime.datetime.now()
+        export_df["Timestamp"] = now
+        export_df["RunId"] = runId
+
+        export_df.columns = export_df.columns.str.replace(" ", "_")
+        spark_df = spark.createDataFrame(export_df)
+        spark_df.write.mode("append").format("delta").saveAsTable(lakeTName)
+        print(
+            f"\u2022 Lakehouse table properties have been saved to the '{lakeTName}' delta table."
+        )
+
+    return df

sempy_labs/lakehouse/_lakehouse.py

@@ -0,0 +1,85 @@
+import sempy
+import sempy.fabric as fabric
+from tqdm.auto import tqdm
+from pyspark.sql import SparkSession
+from sempy_labs._helper_functions import resolve_lakehouse_name
+from typing import List, Optional, Union
+
+
+def lakehouse_attached() -> bool:
+    """
+    Identifies if a lakehouse is attached to the notebook.
+
+    Returns
+    -------
+    bool
+        Returns True if a lakehouse is attached to the notebook.
+    """
+
+    spark = SparkSession.builder.getOrCreate()
+    lakeId = spark.conf.get("trident.lakehouse.id")
+
+    if len(lakeId) > 0:
+        return True
+    else:
+        return False
+
+
+def optimize_lakehouse_tables(
+    tables: Optional[Union[str, List[str]]] = None,
+    lakehouse: Optional[str] = None,
+    workspace: Optional[str] = None,
+):
+    """
+    Runs the `OPTIMIZE <https://docs.delta.io/latest/optimizations-oss.html>`_ function over the specified lakehouse tables.
+
+    Parameters
+    ----------
+    tables : str | List[str] | None
+        The table(s) to optimize. If no tables are specified, all tables in the lakehouse will be optimized.
+    lakehouse : str, default=None
+        The Fabric lakehouse.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    workspace : str, default=None
+        The Fabric workspace used by the lakehouse.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    from sempy_labs.lakehouse._get_lakehouse_tables import get_lakehouse_tables
+    from delta import DeltaTable
+
+    if workspace == None:
+        workspace_id = fabric.get_workspace_id()
+        workspace = fabric.resolve_workspace_name(workspace_id)
+
+    if lakehouse == None:
+        lakehouse_id = fabric.get_lakehouse_id()
+        lakehouse = resolve_lakehouse_name(lakehouse_id, workspace)
+
+    lakeTables = get_lakehouse_tables(lakehouse=lakehouse, workspace=workspace)
+    lakeTablesDelta = lakeTables[lakeTables["Format"] == "delta"]
+
+    if isinstance(tables, str):
+        tables = [tables]
+
+    if tables is not None:
+        tables_filt = lakeTablesDelta[lakeTablesDelta["Table Name"].isin(tables)]
+    else:
+        tables_filt = lakeTablesDelta.copy()
+
+    tableCount = len(tables_filt)
+
+    spark = SparkSession.builder.getOrCreate()
+
+    i = 1
+    for _, r in (bar := tqdm(tables_filt.iterrows())):
+        tableName = r["Table Name"]
+        tablePath = r["Location"]
+        bar.set_description(f"Optimizing the '{tableName}' table...")
+        deltaTable = DeltaTable.forPath(spark, tablePath)
+        deltaTable.optimize().executeCompaction()
+        print(
+            f"The '{tableName}' table has been optimized. ({str(i)}/{str(tableCount)})"
+        )
+        i += 1

sempy_labs/lakehouse/_shortcuts.py

@@ -0,0 +1,296 @@
+import sempy
+import sempy.fabric as fabric
+import pandas as pd
+from sempy_labs._helper_functions import (
+    resolve_lakehouse_name,
+    resolve_lakehouse_id,
+    resolve_workspace_name_and_id,
+)
+from typing import List, Optional, Union
+import sempy_labs._icons as icons
+
+
+def create_shortcut_onelake(
+    table_name: str,
+    source_lakehouse: str,
+    source_workspace: str,
+    destination_lakehouse: str,
+    destination_workspace: Optional[str] = None,
+    shortcut_name: Optional[str] = None,
+):
+    """
+    Creates a `shortcut <https://learn.microsoft.com/fabric/onelake/onelake-shortcuts>`_ to a delta table in OneLake.
+
+    Parameters
+    ----------
+    table_name : str
+        The table name for which a shortcut will be created.
+    source_lakehouse : str
+        The Fabric lakehouse in which the table resides.
+    source_workspace : str
+        The name of the Fabric workspace in which the source lakehouse exists.
+    destination_lakehouse : str
+        The Fabric lakehouse in which the shortcut will be created.
+    destination_workspace : str, default=None
+        The name of the Fabric workspace in which the shortcut will be created.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    shortcut_name : str, default=None
+        The name of the shortcut 'table' to be created. This defaults to the 'table_name' parameter value.
+    """
+
+    sourceWorkspaceId = fabric.resolve_workspace_id(source_workspace)
+    sourceLakehouseId = resolve_lakehouse_id(source_lakehouse, source_workspace)
+
+    if destination_workspace == None:
+        destination_workspace = source_workspace
+
+    destinationWorkspaceId = fabric.resolve_workspace_id(destination_workspace)
+    destinationLakehouseId = resolve_lakehouse_id(
+        destination_lakehouse, destination_workspace
+    )
+
+    if shortcut_name == None:
+        shortcut_name = table_name
+
+    client = fabric.FabricRestClient()
+    tablePath = "Tables/" + table_name
+
+    request_body = {
+        "path": "Tables",
+        "name": shortcut_name.replace(" ", ""),
+        "target": {
+            "oneLake": {
+                "workspaceId": sourceWorkspaceId,
+                "itemId": sourceLakehouseId,
+                "path": tablePath,
+            }
+        },
+    }
+
+    try:
+        response = client.post(
+            f"/v1/workspaces/{destinationWorkspaceId}/items/{destinationLakehouseId}/shortcuts",
+            json=request_body,
+        )
+        if response.status_code == 201:
+            print(
+                f"{icons.green_dot} The shortcut '{shortcut_name}' was created in the '{destination_lakehouse}' lakehouse within the '{destination_workspace} workspace. It is based on the '{table_name}' table in the '{source_lakehouse}' lakehouse within the '{source_workspace}' workspace."
+            )
+        else:
+            print(response.status_code)
+    except Exception as e:
+        print(
+            f"{icons.red_dot} Failed to create a shortcut for the '{table_name}' table: {e}"
+        )
+
+
+def create_shortcut(
+    shortcut_name: str,
+    location: str,
+    subpath: str,
+    source: str,
+    connection_id: str,
+    lakehouse: Optional[str] = None,
+    workspace: Optional[str] = None,
+):
+    """
+    Creates a `shortcut <https://learn.microsoft.com/fabric/onelake/onelake-shortcuts>`_ to an ADLS Gen2 or Amazon S3 source.
+
+    Parameters
+    ----------
+    shortcut_name : str
+    location : str
+    subpath : str
+    source : str
+    connection_id: str
+    lakehouse : str
+        The Fabric lakehouse in which the shortcut will be created.
+    workspace : str, default=None
+        The name of the Fabric workspace in which the shortcut will be created.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    source_titles = {"adlsGen2": "ADLS Gen2", "amazonS3": "Amazon S3"}
+
+    sourceValues = list(source_titles.keys())
+
+    if source not in sourceValues:
+        print(
+            f"{icons.red_dot} The 'source' parameter must be one of these values: {sourceValues}."
+        )
+        return
+
+    sourceTitle = source_titles[source]
+
+    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
+
+    if lakehouse == None:
+        lakehouse_id = fabric.get_lakehouse_id()
+    else:
+        lakehouse_id = resolve_lakehouse_id(lakehouse, workspace)
+
+    client = fabric.FabricRestClient()
+    shortcutActualName = shortcut_name.replace(" ", "")
+
+    request_body = {
+        "path": "Tables",
+        "name": shortcutActualName,
+        "target": {
+            source: {
+                "location": location,
+                "subpath": subpath,
+                "connectionId": connection_id,
+            }
+        },
+    }
+
+    try:
+        response = client.post(
+            f"/v1/workspaces/{workspace_id}/items/{lakehouse_id}/shortcuts",
+            json=request_body,
+        )
+        if response.status_code == 201:
+            print(
+                f"{icons.green_dot} The shortcut '{shortcutActualName}' was created in the '{lakehouse}' lakehouse within the '{workspace} workspace. It is based on the '{subpath}' table in '{sourceTitle}'."
+            )
+        else:
+            print(response.status_code)
+    except:
+        print(
+            f"{icons.red_dot} Failed to create a shortcut for the '{shortcut_name}' table."
+        )
+
+
+def list_shortcuts(
+    lakehouse: Optional[str] = None, workspace: Optional[str] = None
+) -> pd.DataFrame:
+    """
+    Shows all shortcuts which exist in a Fabric lakehouse.
+
+    Parameters
+    ----------
+    lakehouse : str, default=None
+        The Fabric lakehouse name.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    workspace : str, default=None
+        The name of the Fabric workspace in which lakehouse resides.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing all the shortcuts which exist in the specified lakehouse.
+    """
+
+    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
+
+    if lakehouse == None:
+        lakehouse_id = fabric.get_lakehouse_id()
+        lakehouse = resolve_lakehouse_name(lakehouse_id, workspace)
+    else:
+        lakehouse_id = resolve_lakehouse_id(lakehouse, workspace)
+
+    df = pd.DataFrame(
+        columns=[
+            "Shortcut Name",
+            "Shortcut Path",
+            "Source",
+            "Source Lakehouse Name",
+            "Source Workspace Name",
+            "Source Path",
+            "Source Connection ID",
+            "Source Location",
+            "Source SubPath",
+        ]
+    )
+
+    client = fabric.FabricRestClient()
+    response = client.get(
+        f"/v1/workspaces/{workspace_id}/items/{lakehouse_id}/shortcuts"
+    )
+    if response.status_code == 200:
+        for s in response.json()["value"]:
+            shortcutName = s["name"]
+            shortcutPath = s["path"]
+            source = list(s["target"].keys())[0]
+            (
+                sourceLakehouseName,
+                sourceWorkspaceName,
+                sourcePath,
+                connectionId,
+                location,
+                subpath,
+            ) = (None, None, None, None, None, None)
+            if source == "oneLake":
+                sourceLakehouseId = s["target"][source]["itemId"]
+                sourcePath = s["target"][source]["path"]
+                sourceWorkspaceId = s["target"][source]["workspaceId"]
+                sourceWorkspaceName = fabric.resolve_workspace_name(sourceWorkspaceId)
+                sourceLakehouseName = resolve_lakehouse_name(
+                    sourceLakehouseId, sourceWorkspaceName
+                )
+            else:
+                connectionId = s["target"][source]["connectionId"]
+                location = s["target"][source]["location"]
+                subpath = s["target"][source]["subpath"]
+
+            new_data = {
+                "Shortcut Name": shortcutName,
+                "Shortcut Path": shortcutPath,
+                "Source": source,
+                "Source Lakehouse Name": sourceLakehouseName,
+                "Source Workspace Name": sourceWorkspaceName,
+                "Source Path": sourcePath,
+                "Source Connection ID": connectionId,
+                "Source Location": location,
+                "Source SubPath": subpath,
+            }
+            df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
+
+    print(
+        f"This function relies on an API which is not yet official as of May 21, 2024. Once the API becomes official this function will work as expected."
+    )
+    return df
+
+
+def delete_shortcut(
+    shortcut_name: str, lakehouse: Optional[str] = None, workspace: Optional[str] = None
+):
+    """
+    Deletes a shortcut.
+
+    Parameters
+    ----------
+    shortcut_name : str
+        The name of the shortcut.
+    lakehouse : str, default=None
+        The Fabric lakehouse name in which the shortcut resides.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    workspace : str, default=None
+        The name of the Fabric workspace in which lakehouse resides.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
+
+    if lakehouse == None:
+        lakehouse_id = fabric.get_lakehouse_id()
+        lakehouse = resolve_lakehouse_name(lakehouse_id, workspace)
+    else:
+        lakehouse_id = resolve_lakehouse_id(lakehouse, workspace)
+
+    client = fabric.FabricRestClient()
+    response = client.delete(
+        f"/v1/workspaces/{workspace_id}/items/{lakehouse_id}/shortcuts/Tables/{shortcut_name}"
+    )
+
+    if response.status_code == 200:
+        print(
+            f"{icons.green_dot} The '{shortcut_name}' shortcut in the '{lakehouse}' within the '{workspace}' workspace has been deleted."
+        )
+    else:
+        print(f"{icons.red_dot} The '{shortcut_name}' has not been deleted.")

sempy_labs/migration/__init__.py

@@ -0,0 +1,29 @@
+from sempy_labs.migration._create_pqt_file import create_pqt_file
+from sempy_labs.migration._migrate_calctables_to_lakehouse import (
+    migrate_calc_tables_to_lakehouse,
+    migrate_field_parameters,
+)
+from sempy_labs.migration._migrate_calctables_to_semantic_model import (
+    migrate_calc_tables_to_semantic_model,
+)
+from sempy_labs.migration._migrate_model_objects_to_semantic_model import (
+    migrate_model_objects_to_semantic_model,
+)
+from sempy_labs.migration._migrate_tables_columns_to_semantic_model import (
+    migrate_tables_columns_to_semantic_model,
+)
+from sempy_labs.migration._migration_validation import (
+    migration_validation,
+    # list_semantic_model_objects
+)
+
+__all__ = [
+    "create_pqt_file",
+    "migrate_calc_tables_to_lakehouse",
+    "migrate_field_parameters",
+    "migrate_calc_tables_to_semantic_model",
+    "migrate_model_objects_to_semantic_model",
+    "migrate_tables_columns_to_semantic_model",
+    "migration_validation",
+    # list_semantic_model_objects
+]