semantic-link-labs 0.10.0__py3-none-any.whl → 0.11.0__py3-none-any.whl

sempy_labs/lakehouse/_lakehouse.py

@@ -11,8 +11,14 @@ from sempy_labs._helper_functions import (
 )
 import sempy_labs._icons as icons
 import re
+import time
+import pandas as pd
+from sempy_labs._job_scheduler import (
+    _get_item_job_instance,
+)
 
 
+@log
 def lakehouse_attached() -> bool:
     """
     Identifies if a lakehouse is attached to the notebook.
@@ -33,6 +39,7 @@ def lakehouse_attached() -> bool:
         return False
 
 
+@log
 def _optimize_table(path):
 
     if _pure_python_notebook():
@@ -46,6 +53,7 @@ def _optimize_table(path):
         DeltaTable.forPath(spark, path).optimize().executeCompaction()
 
 
+@log
 def _vacuum_table(path, retain_n_hours):
 
     if _pure_python_notebook():
@@ -145,6 +153,7 @@ def vacuum_lakehouse_tables(
         _vacuum_table(path=path, retain_n_hours=retain_n_hours)
 
 
+@log
 def run_table_maintenance(
     table_name: str,
     optimize: bool = False,
@@ -154,7 +163,7 @@ def run_table_maintenance(
     schema: Optional[str] = None,
     lakehouse: Optional[str | UUID] = None,
     workspace: Optional[str | UUID] = None,
-):
+) -> pd.DataFrame:
     """
     Runs table maintenance operations on the specified table within the lakehouse.
 
@@ -181,6 +190,11 @@ def run_table_maintenance(
         The Fabric workspace name or ID used by the lakehouse.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A DataFrame containing the job instance details of the table maintenance operation.
     """
 
     (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
@@ -223,13 +237,32 @@ def run_table_maintenance(
     if vacuum and retention_period is not None:
         payload["executionData"]["vacuumSettings"]["retentionPeriod"] = retention_period
 
-    _base_api(
+    response = _base_api(
         request=f"/v1/workspaces/{workspace_id}/lakehouses/{lakehouse_id}/jobs/instances?jobType=TableMaintenance",
         method="post",
         payload=payload,
         status_codes=202,
     )
 
-    print(
-        f"{icons.green_dot} The table maintenance job for the '{table_name}' table in the '{lakehouse_name}' lakehouse within the '{workspace_name}' workspace has been initiated."
-    )
+    f"{icons.in_progress} The table maintenance job for the '{table_name}' table in the '{lakehouse_name}' lakehouse within the '{workspace_name}' workspace has been initiated."
+
+    status_url = response.headers.get("Location").split("fabric.microsoft.com")[1]
+    status = None
+    while status not in ["Completed", "Failed"]:
+        response = _base_api(request=status_url)
+        status = response.json().get("status")
+        time.sleep(10)
+
+    df = _get_item_job_instance(url=status_url)
+
+    if status == "Completed":
+        print(
+            f"{icons.green_dot} The table maintenance job for the '{table_name}' table in the '{lakehouse_name}' lakehouse within the '{workspace_name}' workspace has succeeded."
+        )
+    else:
+        print(status)
+        print(
+            f"{icons.red_dot} The table maintenance job for the '{table_name}' table in the '{lakehouse_name}' lakehouse within the '{workspace_name}' workspace has failed."
+        )
+
+    return df

sempy_labs/lakehouse/_livy_sessions.py

@@ -8,8 +8,10 @@ from sempy_labs._helper_functions import (
 import pandas as pd
 from typing import Optional
 from uuid import UUID
+from sempy._utils._log import log
 
 
+@log
 def list_livy_sessions(
     lakehouse: Optional[str | UUID] = None, workspace: Optional[str | UUID] = None
 ) -> pd.DataFrame:
@@ -85,7 +87,6 @@ def list_livy_sessions(
     )
 
     dfs = []
-
     for r in responses:
         for v in r.get("value", []):
             queued_duration = v.get("queuedDuration", {})

sempy_labs/lakehouse/_shortcuts.py

@@ -216,6 +216,7 @@ def create_shortcut(
     )
 
 
+@log
 def delete_shortcut(
     shortcut_name: str,
     shortcut_path: str = "Tables",
@@ -260,6 +261,7 @@ def delete_shortcut(
     )
 
 
+@log
 def reset_shortcut_cache(workspace: Optional[str | UUID] = None):
     """
     Deletes any cached files that were stored while reading from shortcuts.
@@ -369,6 +371,7 @@ def list_shortcuts(
         "S3Compatible": "s3Compatible",
     }
 
+    dfs = []
     for r in responses:
         for i in r.get("value", []):
             tgt = i.get("target", {})
@@ -415,6 +418,9 @@ def list_shortcuts(
                 "SubPath": sub_path,
                 "Source Properties Raw": str(tgt),
             }
-            df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
+            dfs.append(pd.DataFrame(new_data, index=[0]))
+
+    if dfs:
+        df = pd.concat(dfs, ignore_index=True)
 
     return df

sempy_labs/migration/_direct_lake_to_import.py

@@ -2,8 +2,10 @@ import sempy
 from uuid import UUID
 import sempy_labs._icons as icons
 from typing import Optional
+from sempy._utils._log import log
 
 
+@log
 def migrate_direct_lake_to_import(
     dataset: str | UUID,
     workspace: Optional[str | UUID] = None,

sempy_labs/mirrored_azure_databricks_catalog/__init__.py

@@ -0,0 +1,15 @@
+from sempy_labs.mirrored_azure_databricks_catalog._refresh_catalog_metadata import (
+    refresh_catalog_metadata,
+)
+from sempy_labs.mirrored_azure_databricks_catalog._discover import (
+    discover_catalogs,
+    discover_schemas,
+    discover_tables,
+)
+
+__all__ = [
+    "refresh_catalog_metadata",
+    "discover_catalogs",
+    "discover_schemas",
+    "discover_tables",
+]

sempy_labs/mirrored_azure_databricks_catalog/_discover.py

@@ -0,0 +1,213 @@
+from uuid import UUID
+from typing import Optional
+from sempy_labs._helper_functions import (
+    resolve_workspace_id,
+    _base_api,
+    _create_dataframe,
+)
+import pandas as pd
+from sempy._utils._log import log
+
+
+@log
+def discover_catalogs(
+    databricks_workspace_connection_id: UUID,
+    workspace: Optional[str | UUID] = None,
+    max_results: Optional[int] = None,
+) -> pd.DataFrame:
+    """
+    Returns a list of catalogs from Unity Catalog.
+
+    This is a wrapper function for the following API: `Databricks Metadata Discovery - Discover Catalogs <https://learn.microsoft.comrest/api/fabric/mirroredazuredatabrickscatalog/databricks-metadata-discovery/discover-catalogs>`_.
+
+    Parameters
+    ----------
+    databricks_workspace_connection_id : uuid.UUID
+        The ID of the Databricks workspace connection.
+    workspace : str | uuid.UUID, default=None
+        The workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    max_results : int, default=None
+        The maximum number of results to return. If not specified, all results are returned.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing a list of catalogs from Unity Catalog.
+    """
+
+    workspace_id = resolve_workspace_id(workspace)
+
+    url = f"/v1/workspaces/{workspace_id}/azuredatabricks/catalogs?databricksWorkspaceConnectionId={databricks_workspace_connection_id}"
+    if max_results:
+        url += f"&maxResults={max_results}"
+
+    responses = _base_api(request=url, uses_pagination=True)
+
+    columns = {
+        "Catalog Name": "str",
+        "Catalog Full Name": "str",
+        "Catalog Type": "str",
+        "Storage Location": "str",
+    }
+
+    df = _create_dataframe(columns=columns)
+
+    dfs = []
+    for r in responses:
+        for i in r.get("value", []):
+            new_data = {
+                "Catalog Name": i.get("name"),
+                "Catalog Full Name": i.get("fullName"),
+                "Catalog Type": i.get("catalogType"),
+                "Storage Location": i.get("storageLocation"),
+            }
+
+            dfs.append(pd.DataFrame(new_data, index=[0]))
+
+    if dfs:
+        df = pd.concat(dfs, ignore_index=True)
+
+    return df
+
+
+@log
+def discover_schemas(
+    catalog: str,
+    databricks_workspace_connection_id: UUID,
+    workspace: Optional[str | UUID] = None,
+    max_results: Optional[int] = None,
+) -> pd.DataFrame:
+    """
+    Returns a list of schemas in the given catalog from Unity Catalog.
+
+    This is a wrapper function for the following API: `Databricks Metadata Discovery - Discover Schemas <https://learn.microsoft.comrest/api/fabric/mirroredazuredatabrickscatalog/databricks-metadata-discovery/discover-schemas>`_.
+
+    Parameters
+    ----------
+    catalog : str
+        The name of the catalog.
+    databricks_workspace_connection_id : uuid.UUID
+        The ID of the Databricks workspace connection.
+    workspace : str | uuid.UUID, default=None
+        The workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    max_results : int, default=None
+        The maximum number of results to return. If not specified, all results are returned.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing a list of schemas in the given catalog from Unity Catalog.
+    """
+
+    workspace_id = resolve_workspace_id(workspace)
+
+    url = f"/v1/workspaces/{workspace_id}/azuredatabricks/catalogs/{catalog}/schemas?databricksWorkspaceConnectionId={databricks_workspace_connection_id}"
+    if max_results:
+        url += f"&maxResults={max_results}"
+
+    responses = _base_api(request=url, uses_pagination=True)
+
+    columns = {
+        "Catalog Name": "str",
+        "Schema Name": "str",
+        "Schema Full Name": "str",
+        "Storage Location": "str",
+    }
+
+    df = _create_dataframe(columns=columns)
+
+    dfs = []
+    for r in responses:
+        for i in r.get("value", []):
+            new_data = {
+                "Catalog Name": catalog,
+                "Schema Name": i.get("name"),
+                "Schema Full Name": i.get("fullName"),
+                "Storage Location": i.get("storageLocation"),
+            }
+
+            dfs.append(pd.DataFrame(new_data, index=[0]))
+
+    if dfs:
+        df = pd.concat(dfs, ignore_index=True)
+
+    return df
+
+
+@log
+def discover_tables(
+    catalog: str,
+    schema: str,
+    databricks_workspace_connection_id: UUID,
+    workspace: Optional[str | UUID] = None,
+    max_results: Optional[int] = None,
+) -> pd.DataFrame:
+    """
+    Returns a list of schemas in the given catalog from Unity Catalog.
+
+    This is a wrapper function for the following API: `Databricks Metadata Discovery - Discover Tables <https://learn.microsoft.comrest/api/fabric/mirroredazuredatabrickscatalog/databricks-metadata-discovery/discover-tables>`_.
+
+    Parameters
+    ----------
+    catalog : str
+        The name of the catalog.
+    schema : str
+        The name of the schema.
+    databricks_workspace_connection_id : uuid.UUID
+        The ID of the Databricks workspace connection.
+    workspace : str | uuid.UUID, default=None
+        The workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    max_results : int, default=None
+        The maximum number of results to return. If not specified, all results are returned.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing a list of schemas in the given catalog from Unity Catalog.
+    """
+
+    workspace_id = resolve_workspace_id(workspace)
+
+    url = f"/v1/workspaces/{workspace_id}/azuredatabricks/catalogs/{catalog}/schemas/{schema}/tables?databricksWorkspaceConnectionId={databricks_workspace_connection_id}"
+    if max_results:
+        url += f"&maxResults={max_results}"
+
+    responses = _base_api(request=url, uses_pagination=True)
+
+    columns = {
+        "Catalog Name": "str",
+        "Schema Name": "str",
+        "Table Name": "str",
+        "Table Full Name": "str",
+        "Storage Location": "str",
+        "Table Type": "str",
+        "Data Source Format": "str",
+    }
+
+    df = _create_dataframe(columns=columns)
+
+    dfs = []
+    for r in responses:
+        for i in r.get("value", []):
+            new_data = {
+                "Catalog Name": catalog,
+                "Schema Name": schema,
+                "Table Name": i.get("name"),
+                "Table Full Name": i.get("fullName"),
+                "Storage Location": i.get("storageLocation"),
+                "Table Type": i.get("tableType"),
+                "Data Source Format": i.get("dataSourceFormat"),
+            }
+
+            dfs.append(pd.DataFrame(new_data, index=[0]))
+
+    if dfs:
+        df = pd.concat(dfs, ignore_index=True)
+
+    return df

sempy_labs/mirrored_azure_databricks_catalog/_refresh_catalog_metadata.py

@@ -0,0 +1,45 @@
+from uuid import UUID
+from typing import Optional
+from sempy_labs._helper_functions import (
+    resolve_workspace_name_and_id,
+    resolve_item_name_and_id,
+    _base_api,
+)
+import sempy_labs._icons as icons
+from sempy._utils._log import log
+
+
+@log
+def refresh_catalog_metadata(
+    mirrored_azure_databricks_catalog: str | UUID,
+    workspace: Optional[str | UUID] = None,
+):
+    """
+    Refresh Databricks catalog metadata in mirroredAzureDatabricksCatalogs Item.
+
+    This is a wrapper function for the following API: `Refresh Metadata - Items RefreshCatalogMetadata <https://learn.microsoft.com/rest/api/fabric/mirroredazuredatabrickscatalog/refresh-metadata/items-refresh-catalog-metadata>`_.
+
+    Parameters
+    ----------
+    mirrored_azure_databricks_catalog : str | uuid.UUID
+        The name or ID of the mirrored Azure Databricks catalog.
+    workspace : str | uuie.UUID, default=None
+        The workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook
+    """
+
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    (catalog_name, catalog_id) = resolve_item_name_and_id(
+        mirrored_azure_databricks_catalog
+    )
+
+    _base_api(
+        request=f"/v1/workspaces/{workspace_id}/mirroredAzureDatabricksCatalogs/{catalog_id}/refreshCatalogMetadata",
+        method="post",
+        lro_return_status_code=True,
+    )
+
+    print(
+        f"{icons.green_dot} The '{catalog_name}' Databricks Catalog metadata within the '{workspace_name}' workspace has been refreshed."
+    )

sempy_labs/report/_download_report.py

@@ -1,4 +1,3 @@
-import sempy.fabric as fabric
 import sempy_labs._icons as icons
 from typing import Optional
 from sempy_labs._helper_functions import (
@@ -11,8 +10,10 @@ from sempy_labs._helper_functions import (
 )
 from sempy_labs.lakehouse._lakehouse import lakehouse_attached
 from uuid import UUID
+from sempy._utils._log import log
 
 
+@log
 def download_report(
     report: str | UUID,
     file_name: Optional[str] = None,

sempy_labs/report/_generate_report.py

@@ -18,6 +18,7 @@ from sempy._utils._log import log
 from uuid import UUID
 
 
+@log
 def create_report_from_reportjson(
     report: str,
     dataset: str | UUID,
@@ -119,6 +120,7 @@ def create_report_from_reportjson(
     )
 
 
+@log
 def update_report_from_reportjson(
     report: str | UUID, report_json: dict, workspace: Optional[str | UUID] = None
 ):

sempy_labs/report/_paginated.py

@@ -7,8 +7,10 @@ from sempy_labs._helper_functions import (
     resolve_item_id,
     _create_dataframe,
 )
+from sempy._utils._log import log
 
 
+@log
 def get_report_datasources(
     report: str | UUID,
     workspace: Optional[str | UUID] = None,