semantic-link-labs 0.9.9__py3-none-any.whl → 0.9.11__py3-none-any.whl

sempy_labs/lakehouse/_get_lakehouse_tables.py

@@ -1,7 +1,7 @@
-import sempy.fabric as fabric
+import os
 import pandas as pd
 import pyarrow.parquet as pq
-import datetime
+from datetime import datetime
 from sempy_labs._helper_functions import (
     _get_column_aggregate,
     resolve_workspace_name_and_id,
@@ -9,7 +9,11 @@ from sempy_labs._helper_functions import (
     save_as_delta_table,
     _base_api,
     _create_dataframe,
-    _create_spark_session,
+    _read_delta_table,
+    _get_delta_table,
+    _mount,
+    create_abfss_path,
+    _pure_python_notebook,
 )
 from sempy_labs.directlake._guardrails import (
     get_sku_size,
@@ -33,8 +37,12 @@ def get_lakehouse_tables(
     """
     Shows the tables of a lakehouse and their respective properties. Option to include additional properties relevant to Direct Lake guardrails.
 
+    This function can be executed in either a PySpark or pure Python notebook.
+
     This is a wrapper function for the following API: `Tables - List Tables <https://learn.microsoft.com/rest/api/fabric/lakehouse/tables/list-tables>`_ plus extended capabilities.
 
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
     Parameters
     ----------
     lakehouse : str | uuid.UUID, default=None
@@ -75,19 +83,10 @@ def get_lakehouse_tables(
     if count_rows:  # Setting countrows defaults to extended=True
         extended = True
 
-    if (
-        workspace_id != fabric.get_workspace_id()
-        and lakehouse_id != fabric.get_lakehouse_id()
-        and count_rows
-    ):
-        raise ValueError(
-            f"{icons.red_dot} If 'count_rows' is set to True, you must run this function against the default lakehouse attached to the notebook. "
-            "Count rows runs a spark query and cross-workspace spark queries are currently not supported."
-        )
-
     responses = _base_api(
         request=f"v1/workspaces/{workspace_id}/lakehouses/{lakehouse_id}/tables",
         uses_pagination=True,
+        client="fabric_sp",
     )
 
     if not responses[0].get("data"):
@@ -112,40 +111,59 @@ def get_lakehouse_tables(
     if extended:
         sku_value = get_sku_size(workspace_id)
         guardrail = get_directlake_guardrails_for_sku(sku_value)
-        spark = _create_spark_session()
-        df["Files"] = None
-        df["Row Groups"] = None
-        df["Table Size"] = None
+        local_path = _mount(lakehouse=lakehouse_id, workspace=workspace_id)
+
+        df["Files"], df["Row Groups"], df["Table Size"] = None, None, None
         if count_rows:
             df["Row Count"] = None
+
         for i, r in df.iterrows():
-            tName = r["Table Name"]
+            table_name = r["Table Name"]
             if r["Type"] == "Managed" and r["Format"] == "delta":
-                detail_df = spark.sql(f"DESCRIBE DETAIL `{tName}`").collect()[0]
-                num_files = detail_df.numFiles
-                size_in_bytes = detail_df.sizeInBytes
-
-                delta_table_path = f"Tables/{tName}"
-                latest_files = (
-                    spark.read.format("delta").load(delta_table_path).inputFiles()
+                delta_table_path = create_abfss_path(
+                    lakehouse_id, workspace_id, table_name
                 )
-                file_paths = [f.split("/")[-1] for f in latest_files]
 
-                # Handle FileNotFoundError
+                if _pure_python_notebook():
+                    from deltalake import DeltaTable
+
+                    delta_table = DeltaTable(delta_table_path)
+                    latest_files = [
+                        file["path"]
+                        for file in delta_table.get_add_actions().to_pylist()
+                    ]
+                    size_in_bytes = 0
+                    for f in latest_files:
+                        local_file_path = os.path.join(
+                            local_path, "Tables", table_name, os.path.basename(f)
+                        )
+                        if os.path.exists(local_file_path):
+                            size_in_bytes += os.path.getsize(local_file_path)
+                    num_latest_files = len(latest_files)
+                else:
+                    delta_table = _get_delta_table(delta_table_path)
+                    latest_files = _read_delta_table(delta_table_path).inputFiles()
+                    table_df = delta_table.toDF()
+                    table_details = delta_table.detail().collect()[0].asDict()
+                    num_latest_files = table_details.get("numFiles", 0)
+                    size_in_bytes = table_details.get("sizeInBytes", 0)
+
+                table_path = os.path.join(local_path, "Tables", table_name)
+                file_paths = [os.path.basename(f) for f in latest_files]
+
                 num_rowgroups = 0
                 for filename in file_paths:
-                    try:
-                        num_rowgroups += pq.ParquetFile(
-                            f"/lakehouse/default/{delta_table_path}/{filename}"
-                        ).num_row_groups
-                    except FileNotFoundError:
-                        continue
-                df.at[i, "Files"] = num_files
+                    parquet_file = pq.ParquetFile(f"{table_path}/{filename}")
+                    num_rowgroups += parquet_file.num_row_groups
+                df.at[i, "Files"] = num_latest_files
                 df.at[i, "Row Groups"] = num_rowgroups
                 df.at[i, "Table Size"] = size_in_bytes
             if count_rows:
-                num_rows = spark.table(tName).count()
-                df.at[i, "Row Count"] = num_rows
+                if _pure_python_notebook():
+                    row_count = delta_table.to_pyarrow_table().num_rows
+                else:
+                    row_count = table_df.count()
+                df.at[i, "Row Count"] = row_count
 
     if extended:
         intColumns = ["Files", "Row Groups", "Table Size"]
@@ -168,19 +186,16 @@ def get_lakehouse_tables(
     if export:
         if not lakehouse_attached():
             raise ValueError(
-                f"{icons.red_dot} In order to save the report.json file, a lakehouse must be attached to the notebook. Please attach a lakehouse to this notebook."
+                f"{icons.red_dot} In order to save the dataframe, a lakehouse must be attached to the notebook. Please attach a lakehouse to this notebook."
             )
 
-        (current_lakehouse_name, current_lakehouse_id) = resolve_lakehouse_name_and_id()
-        lakeTName = "lakehouse_table_details"
-        lakeT_filt = df[df["Table Name"] == lakeTName]
+        lake_table_name = "lakehouse_table_details"
+        df_filt = df[df["Table Name"] == lake_table_name]
 
-        if len(lakeT_filt) == 0:
+        if df_filt.empty:
             run_id = 1
         else:
-            max_run_id = _get_column_aggregate(
-                lakehouse=current_lakehouse_name, table_name=lakeTName
-            )
+            max_run_id = _get_column_aggregate(table_name=lake_table_name)
             run_id = max_run_id + 1
 
         export_df = df.copy()
@@ -224,13 +239,13 @@ def get_lakehouse_tables(
                     export_df[c] = export_df[c].astype(bool)
 
         print(
-            f"{icons.in_progress} Saving Lakehouse table properties to the '{lakeTName}' table in the lakehouse...\n"
+            f"{icons.in_progress} Saving Lakehouse table properties to the '{lake_table_name}' table in the lakehouse...\n"
         )
-        export_df["Timestamp"] = datetime.datetime.now()
+        export_df["Timestamp"] = datetime.now()
         export_df["RunId"] = run_id
 
         save_as_delta_table(
-            dataframe=export_df, delta_table_name=lakeTName, write_mode="append"
+            dataframe=export_df, delta_table_name=lake_table_name, write_mode="append"
         )
 
     return df

sempy_labs/lakehouse/_helper.py

@@ -0,0 +1,211 @@
+from uuid import UUID
+from typing import Optional, Literal
+import pyarrow.dataset as ds
+from sempy_labs._helper_functions import (
+    _mount,
+    delete_item,
+    _base_api,
+    resolve_workspace_name_and_id,
+    resolve_lakehouse_name_and_id,
+)
+from sempy._utils._log import log
+import sempy_labs._icons as icons
+import os
+
+
+@log
+def is_v_ordered(
+    table_name: str,
+    lakehouse: Optional[str | UUID] = None,
+    workspace: Optional[str | UUID] = None,
+    schema: Optional[str] = None,
+) -> bool:
+    """
+    Checks if a delta table is v-ordered.
+
+    Parameters
+    ----------
+    table_name : str
+        The name of the table to check.
+    lakehouse : str | uuid.UUID, default=None
+        The Fabric lakehouse name or ID.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID used by the lakehouse.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    schema : str, optional
+        The schema of the table to check. If not provided, the default schema is used.
+
+    Returns
+    -------
+    bool
+        True if the table is v-ordered, False otherwise.
+    """
+
+    local_path = _mount(lakehouse=lakehouse, workspace=workspace)
+    table_path = (
+        f"{local_path}/Tables/{schema}/{table_name}"
+        if schema
+        else f"{local_path}/Tables/{table_name}"
+    )
+    ds_schema = ds.dataset(table_path).schema.metadata
+
+    return any(b"vorder" in key for key in ds_schema.keys())
+
+
+def delete_lakehouse(
+    lakehouse: str | UUID, workspace: Optional[str | UUID] = None
+) -> None:
+    """
+    Deletes a lakehouse.
+
+    This is a wrapper function for the following API: `Items - Delete Lakehouse <https://learn.microsoft.com/rest/api/fabric/lakehouse/items/delete-lakehouse>`_.
+
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
+    Parameters
+    ----------
+    lakehouse : str | uuid.UUID
+        The name or ID of the lakehouse to delete.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID used by the lakehouse.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    delete_item(item=lakehouse, item_type="lakehouse", workspace=workspace)
+
+
+def update_lakehouse(
+    name: Optional[str] = None,
+    description: Optional[str] = None,
+    lakehouse: Optional[str | UUID] = None,
+    workspace: Optional[str | UUID] = None,
+):
+    """
+    Updates a lakehouse.
+
+    This is a wrapper function for the following API: `Items - Update Lakehouse <https://learn.microsoft.com/rest/api/fabric/lakehouse/items/update-lakehouse>`_.
+
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
+    Parameters
+    ----------
+    name: str, default=None
+        The new name of the lakehouse.
+        Defaults to None which does not update the name.
+    description: str, default=None
+        The new description of the lakehouse.
+        Defaults to None which does not update the description.
+    lakehouse : str | uuid.UUID, default=None
+        The name or ID of the lakehouse to update.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID used by the lakehouse.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    if not name and not description:
+        raise ValueError(
+            f"{icons.red_dot} Either name or description must be provided."
+        )
+
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    (lakehouse_name, lakehouse_id) = resolve_lakehouse_name_and_id(
+        lakehouse, workspace_id
+    )
+
+    payload = {}
+    if name:
+        payload["displayName"] = name
+    if description:
+        payload["description"] = description
+
+    _base_api(
+        request=f"/v1/workspaces/{workspace_id}/lakehouses/{lakehouse_id}",
+        method="patch",
+        client="fabric_sp",
+        payload=payload,
+    )
+
+    print(
+        f"{icons.green_dot} The '{lakehouse_name}' lakehouse within the '{workspace_name}' workspace has been updated accordingly."
+    )
+
+
+@log
+def load_table(
+    table_name: str,
+    file_path: str,
+    mode: Literal["Overwrite", "Append"],
+    lakehouse: Optional[str | UUID] = None,
+    workspace: Optional[str | UUID] = None,
+):
+    """
+    Loads a table into a lakehouse. Currently only files are supported, not folders.
+
+    This is a wrapper function for the following API: `Tables - Load Table <https://learn.microsoft.com/rest/api/fabric/lakehouse/tables/load-table>`_.
+
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
+    Parameters
+    ----------
+    table_name : str
+        The name of the table to load.
+    file_path : str
+        The path to the data to load.
+    mode : Literal["Overwrite", "Append"]
+        The mode to use when loading the data.
+        "Overwrite" will overwrite the existing data.
+        "Append" will append the data to the existing data.
+    lakehouse : str | uuid.UUID, default=None
+        The name or ID of the lakehouse to load the table into.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID used by the lakehouse.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    (lakehouse_name, lakehouse_id) = resolve_lakehouse_name_and_id(
+        lakehouse, workspace_id
+    )
+
+    file_extension = os.path.splitext(file_path)[1]
+
+    payload = {
+        "relativePath": file_path,
+        "pathType": "File",
+        "mode": mode,
+        "formatOptions": {},
+    }
+
+    if file_extension == ".csv":
+        payload["formatOptions"] = {"format": "Csv", "header": True, "delimiter": ","}
+    elif file_extension == ".parquet":
+        payload["formatOptions"] = {
+            "format": "Parquet",
+            "header": True,
+        }
+    # Solve for loading folders
+    # elif file_extension == '':
+    #    payload['pathType'] = "Folder"
+    #    payload["recursive"] = recursive
+    #    payload['formatOptions']
+    else:
+        raise NotImplementedError()
+
+    _base_api(
+        request=f"/v1/workspaces/{workspace_id}/lakehouses/{lakehouse_id}/tables/{table_name}/load",
+        client="fabric_sp",
+        method="post",
+        status_codes=202,
+        lro_return_status_code=True,
+    )
+
+    print(
+        f"{icons.green_dot} The '{table_name}' table has been loaded into the '{lakehouse_name}' lakehouse within the '{workspace_name}' workspace."
+    )

sempy_labs/lakehouse/_lakehouse.py

@@ -7,6 +7,7 @@ from sempy_labs._helper_functions import (
     resolve_lakehouse_name_and_id,
     resolve_workspace_name_and_id,
     _create_spark_session,
+    _pure_python_notebook,
 )
 import sempy_labs._icons as icons
 import re
@@ -32,6 +33,33 @@ def lakehouse_attached() -> bool:
         return False
 
 
+def _optimize_table(path):
+
+    if _pure_python_notebook():
+        from deltalake import DeltaTable
+
+        DeltaTable(path).optimize.compact()
+    else:
+        from delta import DeltaTable
+
+        spark = _create_spark_session()
+        DeltaTable.forPath(spark, path).optimize().executeCompaction()
+
+
+def _vacuum_table(path, retain_n_hours):
+
+    if _pure_python_notebook():
+        from deltalake import DeltaTable
+
+        DeltaTable(path).vacuum(retention_hours=retain_n_hours)
+    else:
+        from delta import DeltaTable
+
+        spark = _create_spark_session()
+        spark.conf.set("spark.databricks.delta.vacuum.parallelDelete.enabled", "true")
+        DeltaTable.forPath(spark, path).vacuum(retain_n_hours)
+
+
 @log
 def optimize_lakehouse_tables(
     tables: Optional[Union[str, List[str]]] = None,
@@ -56,27 +84,20 @@ def optimize_lakehouse_tables(
     """
 
     from sempy_labs.lakehouse._get_lakehouse_tables import get_lakehouse_tables
-    from delta import DeltaTable
 
-    lakeTables = get_lakehouse_tables(lakehouse=lakehouse, workspace=workspace)
-    lakeTablesDelta = lakeTables[lakeTables["Format"] == "delta"]
+    df = get_lakehouse_tables(lakehouse=lakehouse, workspace=workspace)
+    df_delta = df[df["Format"] == "delta"]
 
     if isinstance(tables, str):
         tables = [tables]
 
-    if tables is not None:
-        tables_filt = lakeTablesDelta[lakeTablesDelta["Table Name"].isin(tables)]
-    else:
-        tables_filt = lakeTablesDelta.copy()
-
-    spark = _create_spark_session()
+    df_tables = df_delta[df_delta["Table Name"].isin(tables)] if tables else df_delta
 
-    for _, r in (bar := tqdm(tables_filt.iterrows())):
-        tableName = r["Table Name"]
-        tablePath = r["Location"]
-        bar.set_description(f"Optimizing the '{tableName}' table...")
-        deltaTable = DeltaTable.forPath(spark, tablePath)
-        deltaTable.optimize().executeCompaction()
+    for _, r in (bar := tqdm(df_tables.iterrows())):
+        table_name = r["Table Name"]
+        path = r["Location"]
+        bar.set_description(f"Optimizing the '{table_name}' table...")
+        _optimize_table(path=path)
 
 
 @log
@@ -92,7 +113,7 @@ def vacuum_lakehouse_tables(
     Parameters
     ----------
     tables : str | List[str] | None
-        The table(s) to vacuum. If no tables are specified, all tables in the lakehouse will be optimized.
+        The table(s) to vacuum. If no tables are specified, all tables in the lakehouse will be vacuumed.
     lakehouse : str | uuid.UUID, default=None
         The Fabric lakehouse name or ID.
         Defaults to None which resolves to the lakehouse attached to the notebook.
@@ -108,32 +129,20 @@ def vacuum_lakehouse_tables(
     """
 
     from sempy_labs.lakehouse._get_lakehouse_tables import get_lakehouse_tables
-    from delta import DeltaTable
 
-    lakeTables = get_lakehouse_tables(lakehouse=lakehouse, workspace=workspace)
-    lakeTablesDelta = lakeTables[lakeTables["Format"] == "delta"]
+    df = get_lakehouse_tables(lakehouse=lakehouse, workspace=workspace)
+    df_delta = df[df["Format"] == "delta"]
 
     if isinstance(tables, str):
         tables = [tables]
 
-    if tables is not None:
-        tables_filt = lakeTablesDelta[lakeTablesDelta["Table Name"].isin(tables)]
-    else:
-        tables_filt = lakeTablesDelta.copy()
-
-    spark = _create_spark_session()
-    spark.conf.set("spark.databricks.delta.vacuum.parallelDelete.enabled", "true")
-
-    for _, r in (bar := tqdm(tables_filt.iterrows())):
-        tableName = r["Table Name"]
-        tablePath = r["Location"]
-        bar.set_description(f"Vacuuming the '{tableName}' table...")
-        deltaTable = DeltaTable.forPath(spark, tablePath)
+    df_tables = df_delta[df_delta["Table Name"].isin(tables)] if tables else df_delta
 
-        if retain_n_hours is None:
-            deltaTable.vacuum()
-        else:
-            deltaTable.vacuum(retain_n_hours)
+    for _, r in (bar := tqdm(df_tables.iterrows())):
+        table_name = r["Table Name"]
+        path = r["Location"]
+        bar.set_description(f"Vacuuming the '{table_name}' table...")
+        _vacuum_table(path=path, retain_n_hours=retain_n_hours)
 
 
 def run_table_maintenance(

sempy_labs/lakehouse/_livy_sessions.py

@@ -0,0 +1,137 @@
+from sempy_labs._helper_functions import (
+    resolve_workspace_id,
+    resolve_lakehouse_id,
+    _base_api,
+    _create_dataframe,
+    _update_dataframe_datatypes,
+)
+import pandas as pd
+from typing import Optional
+from uuid import UUID
+
+
+def list_livy_sessions(
+    lakehouse: Optional[str | UUID] = None, workspace: Optional[str | UUID] = None
+) -> pd.DataFrame:
+    """
+    Shows a list of livy sessions from the specified item identifier.
+
+    This is a wrapper function for the following API: `Livy Sessions - List Livy Sessions <https://learn.microsoft.com/rest/api/fabric/lakehouse/livy-sessions/list-livy-sessions>`_.
+
+    Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
+
+    Parameters
+    ----------
+    lakehouse : str | uuid.UUID, default=None
+        The Fabric lakehouse name or ID.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing a list of livy sessions from the specified item identifier.
+    """
+
+    columns = {
+        "Spark Application Id": "string",
+        "State:": "string",
+        "Livy Id": "string",
+        "Origin": "string",
+        "Attempt Number": "int",
+        "Max Number Of Attempts": "int",
+        "Livy Name": "string",
+        "Submitter Id": "string",
+        "Submitter Type": "string",
+        "Item Workspace Id": "string",
+        "Item Id": "string",
+        "Item Reference Type": "string",
+        "Item Name": "string",
+        "Item Type": "string",
+        "Job Type": "string",
+        "Submitted Date Time": "str",
+        "Start Date Time": "str",
+        "End Date Time": "string",
+        "Queued Duration Value": "int",
+        "Queued Duration Time Unit": "string",
+        "Running Duration Value": "int",
+        "Running Duration Time Unit": "string",
+        "Total Duration Value": "int",
+        "Total Duration Time Unit": "string",
+        "Job Instance Id": "string",
+        "Creator Item Workspace Id": "string",
+        "Creator Item Id": "string",
+        "Creator Item Reference Type": "string",
+        "Creator Item Name": "string",
+        "Creator Item Type": "string",
+        "Cancellation Reason": "string",
+        "Capacity Id": "string",
+        "Operation Name": "string",
+        "Runtime Version": "string",
+        "Livy Session Item Resource Uri": "string",
+    }
+    df = _create_dataframe(columns=columns)
+
+    workspace_id = resolve_workspace_id(workspace)
+    lakehouse_id = resolve_lakehouse_id(lakehouse, workspace_id)
+
+    responses = _base_api(
+        request=f"/v1/workspaces/{workspace_id}/lakehouses/{lakehouse_id}/livySessions",
+        uses_pagination=True,
+        client="fabric_sp",
+    )
+
+    dfs = []
+
+    for r in responses:
+        for v in r.get("value", []):
+            queued_duration = v.get("queuedDuration", {})
+            running_duration = v.get("runningDuration", {})
+            total_duration = v.get("totalDuration", {})
+            new_data = {
+                "Spark Application Id": v.get("sparkApplicationId"),
+                "State:": v.get("state"),
+                "Livy Id": v.get("livyId"),
+                "Origin": v.get("origin"),
+                "Attempt Number": v.get("attemptNumber"),
+                "Max Number Of Attempts": v.get("maxNumberOfAttempts"),
+                "Livy Name": v.get("livyName"),
+                "Submitter Id": v["submitter"].get("id"),
+                "Submitter Type": v["submitter"].get("type"),
+                "Item Workspace Id": v["item"].get("workspaceId"),
+                "Item Id": v["item"].get("itemId"),
+                "Item Reference Type": v["item"].get("referenceType"),
+                "Item Name": v.get("itemName"),
+                "Item Type": v.get("itemType"),
+                "Job Type": v.get("jobType"),
+                "Submitted Date Time": v.get("submittedDateTime"),
+                "Start Date Time": v.get("startDateTime"),
+                "End Date Time": v.get("endDateTime"),
+                "Queued Duration Value": queued_duration.get("value"),
+                "Queued Duration Time Unit": queued_duration.get("timeUnit"),
+                "Running Duration Value": running_duration.get("value"),
+                "Running Duration Time Unit": running_duration.get("timeUnit"),
+                "Total Duration Value": total_duration.get("value"),
+                "Total Duration Time Unit": total_duration.get("timeUnit"),
+                "Job Instance Id": v.get("jobInstanceId"),
+                "Creator Item Workspace Id": v["creatorItem"].get("workspaceId"),
+                "Creator Item Id": v["creatorItem"].get("itemId"),
+                "Creator Item Reference Type": v["creatorItem"].get("referenceType"),
+                "Creator Item Name": v.get("creatorItemName"),
+                "Creator Item Type": v.get("creatorItemType"),
+                "Cancellation Reason": v.get("cancellationReason"),
+                "Capacity Id": v.get("capacityId"),
+                "Operation Name": v.get("operationName"),
+                "Runtime Version": v.get("runtimeVersion"),
+                "Livy Session Item Resource Uri": v.get("livySessionItemResourceUri"),
+            }
+            dfs.append(pd.DataFrame(new_data, index=[0]))
+
+    if dfs:
+        df = pd.concat(dfs, ignore_index=True)
+        _update_dataframe_datatypes(dataframe=df, column_map=columns)
+
+    return df