semantic-link-labs 0.9.11__py3-none-any.whl → 0.10.1__py3-none-any.whl

sempy_labs/mirrored_azure_databricks_catalog/__init__.py

@@ -0,0 +1,15 @@
+from sempy_labs.mirrored_azure_databricks_catalog._refresh_catalog_metadata import (
+    refresh_catalog_metadata,
+)
+from sempy_labs.mirrored_azure_databricks_catalog._discover import (
+    discover_catalogs,
+    discover_schemas,
+    discover_tables,
+)
+
+__all__ = [
+    "refresh_catalog_metadata",
+    "discover_catalogs",
+    "discover_schemas",
+    "discover_tables",
+]

sempy_labs/mirrored_azure_databricks_catalog/_discover.py

@@ -0,0 +1,209 @@
+from uuid import UUID
+from typing import Optional
+from sempy_labs._helper_functions import (
+    resolve_workspace_id,
+    _base_api,
+    _create_dataframe,
+)
+import pandas as pd
+
+
+def discover_catalogs(
+    databricks_workspace_connection_id: UUID,
+    workspace: Optional[str | UUID] = None,
+    max_results: Optional[int] = None,
+) -> pd.DataFrame:
+    """
+    Returns a list of catalogs from Unity Catalog.
+
+    This is a wrapper function for the following API: `Databricks Metadata Discovery - Discover Catalogs <https://learn.microsoft.comrest/api/fabric/mirroredazuredatabrickscatalog/databricks-metadata-discovery/discover-catalogs>`_.
+
+    Parameters
+    ----------
+    databricks_workspace_connection_id : uuid.UUID
+        The ID of the Databricks workspace connection.
+    workspace : str | uuid.UUID, default=None
+        The workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    max_results : int, default=None
+        The maximum number of results to return. If not specified, all results are returned.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing a list of catalogs from Unity Catalog.
+    """
+
+    workspace_id = resolve_workspace_id(workspace)
+
+    url = f"/v1/workspaces/{workspace_id}/azuredatabricks/catalogs?databricksWorkspaceConnectionId={databricks_workspace_connection_id}"
+    if max_results:
+        url += f"&maxResults={max_results}"
+
+    responses = _base_api(request=url, uses_pagination=True)
+
+    columns = {
+        "Catalog Name": "str",
+        "Catalog Full Name": "str",
+        "Catalog Type": "str",
+        "Storage Location": "str",
+    }
+
+    df = _create_dataframe(columns=columns)
+
+    dfs = []
+    for r in responses:
+        for i in r.get("value", []):
+            new_data = {
+                "Catalog Name": i.get("name"),
+                "Catalog Full Name": i.get("fullName"),
+                "Catalog Type": i.get("catalogType"),
+                "Storage Location": i.get("storageLocation"),
+            }
+
+            dfs.append(pd.DataFrame(new_data, index=[0]))
+
+    if dfs:
+        df = pd.concat(dfs, ignore_index=True)
+
+    return df
+
+
+def discover_schemas(
+    catalog: str,
+    databricks_workspace_connection_id: UUID,
+    workspace: Optional[str | UUID] = None,
+    max_results: Optional[int] = None,
+) -> pd.DataFrame:
+    """
+    Returns a list of schemas in the given catalog from Unity Catalog.
+
+    This is a wrapper function for the following API: `Databricks Metadata Discovery - Discover Schemas <https://learn.microsoft.comrest/api/fabric/mirroredazuredatabrickscatalog/databricks-metadata-discovery/discover-schemas>`_.
+
+    Parameters
+    ----------
+    catalog : str
+        The name of the catalog.
+    databricks_workspace_connection_id : uuid.UUID
+        The ID of the Databricks workspace connection.
+    workspace : str | uuid.UUID, default=None
+        The workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    max_results : int, default=None
+        The maximum number of results to return. If not specified, all results are returned.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing a list of schemas in the given catalog from Unity Catalog.
+    """
+
+    workspace_id = resolve_workspace_id(workspace)
+
+    url = f"/v1/workspaces/{workspace_id}/azuredatabricks/catalogs/{catalog}/schemas?databricksWorkspaceConnectionId={databricks_workspace_connection_id}"
+    if max_results:
+        url += f"&maxResults={max_results}"
+
+    responses = _base_api(request=url, uses_pagination=True)
+
+    columns = {
+        "Catalog Name": "str",
+        "Schema Name": "str",
+        "Schema Full Name": "str",
+        "Storage Location": "str",
+    }
+
+    df = _create_dataframe(columns=columns)
+
+    dfs = []
+    for r in responses:
+        for i in r.get("value", []):
+            new_data = {
+                "Catalog Name": catalog,
+                "Schema Name": i.get("name"),
+                "Schema Full Name": i.get("fullName"),
+                "Storage Location": i.get("storageLocation"),
+            }
+
+            dfs.append(pd.DataFrame(new_data, index=[0]))
+
+    if dfs:
+        df = pd.concat(dfs, ignore_index=True)
+
+    return df
+
+
+def discover_tables(
+    catalog: str,
+    schema: str,
+    databricks_workspace_connection_id: UUID,
+    workspace: Optional[str | UUID] = None,
+    max_results: Optional[int] = None,
+) -> pd.DataFrame:
+    """
+    Returns a list of schemas in the given catalog from Unity Catalog.
+
+    This is a wrapper function for the following API: `Databricks Metadata Discovery - Discover Tables <https://learn.microsoft.comrest/api/fabric/mirroredazuredatabrickscatalog/databricks-metadata-discovery/discover-tables>`_.
+
+    Parameters
+    ----------
+    catalog : str
+        The name of the catalog.
+    schema : str
+        The name of the schema.
+    databricks_workspace_connection_id : uuid.UUID
+        The ID of the Databricks workspace connection.
+    workspace : str | uuid.UUID, default=None
+        The workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    max_results : int, default=None
+        The maximum number of results to return. If not specified, all results are returned.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing a list of schemas in the given catalog from Unity Catalog.
+    """
+
+    workspace_id = resolve_workspace_id(workspace)
+
+    url = f"/v1/workspaces/{workspace_id}/azuredatabricks/catalogs/{catalog}/schemas/{schema}/tables?databricksWorkspaceConnectionId={databricks_workspace_connection_id}"
+    if max_results:
+        url += f"&maxResults={max_results}"
+
+    responses = _base_api(request=url, uses_pagination=True)
+
+    columns = {
+        "Catalog Name": "str",
+        "Schema Name": "str",
+        "Table Name": "str",
+        "Table Full Name": "str",
+        "Storage Location": "str",
+        "Table Type": "str",
+        "Data Source Format": "str",
+    }
+
+    df = _create_dataframe(columns=columns)
+
+    dfs = []
+    for r in responses:
+        for i in r.get("value", []):
+            new_data = {
+                "Catalog Name": catalog,
+                "Schema Name": schema,
+                "Table Name": i.get("name"),
+                "Table Full Name": i.get("fullName"),
+                "Storage Location": i.get("storageLocation"),
+                "Table Type": i.get("tableType"),
+                "Data Source Format": i.get("dataSourceFormat"),
+            }
+
+            dfs.append(pd.DataFrame(new_data, index=[0]))
+
+    if dfs:
+        df = pd.concat(dfs, ignore_index=True)
+
+    return df

sempy_labs/mirrored_azure_databricks_catalog/_refresh_catalog_metadata.py

@@ -0,0 +1,43 @@
+from uuid import UUID
+from typing import Optional
+from sempy_labs._helper_functions import (
+    resolve_workspace_name_and_id,
+    resolve_item_name_and_id,
+    _base_api,
+)
+import sempy_labs._icons as icons
+
+
+def refresh_catalog_metadata(
+    mirrored_azure_databricks_catalog: str | UUID,
+    workspace: Optional[str | UUID] = None,
+):
+    """
+    Refresh Databricks catalog metadata in mirroredAzureDatabricksCatalogs Item.
+
+    This is a wrapper function for the following API: `Refresh Metadata - Items RefreshCatalogMetadata <https://learn.microsoft.com/rest/api/fabric/mirroredazuredatabrickscatalog/refresh-metadata/items-refresh-catalog-metadata>`_.
+
+    Parameters
+    ----------
+    mirrored_azure_databricks_catalog : str | uuid.UUID
+        The name or ID of the mirrored Azure Databricks catalog.
+    workspace : str | uuie.UUID, default=None
+        The workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook
+    """
+
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    (catalog_name, catalog_id) = resolve_item_name_and_id(
+        mirrored_azure_databricks_catalog
+    )
+
+    _base_api(
+        request=f"/v1/workspaces/{workspace_id}/mirroredAzureDatabricksCatalogs/{catalog_id}/refreshCatalogMetadata",
+        method="post",
+        lro_return_status_code=True,
+    )
+
+    print(
+        f"{icons.green_dot} The '{catalog_name}' Databricks Catalog metadata within the '{workspace_name}' workspace has been refreshed."
+    )

sempy_labs/report/__init__.py

@@ -3,6 +3,7 @@ from sempy_labs.report._save_report import (
 )
 from sempy_labs.report._reportwrapper import (
     ReportWrapper,
+    connect_report,
 )
 from sempy_labs.report._paginated import (
     get_report_datasources,
@@ -50,4 +51,5 @@ __all__ = [
     "get_report_datasources",
     "download_report",
     "save_report_as_pbip",
+    "connect_report",
 ]

sempy_labs/report/_report_helper.py

@@ -1,14 +1,5 @@
-import sempy.fabric as fabric
-from typing import Tuple, Optional
-import sempy_labs._icons as icons
-import re
-import base64
-import json
 import requests
-from uuid import UUID
-from sempy_labs._helper_functions import (
-    resolve_workspace_name_and_id,
-)
+import sempy_labs._icons as icons
 
 
 vis_type_mapping = {
@@ -47,10 +38,9 @@ vis_type_mapping = {
     "decompositionTreeVisual": "Decomposition tree",
     "qnaVisual": "Q&A",
     "aiNarratives": "Narrative",
-    "scorecard": "Metrics (Preview)",
+    "scorecard": "Goals (Preview)",
     "rdlVisual": "Paginated report",
     "cardVisual": "Card (new)",
-    "advancedSlicerVisual": "Slicer (new)",
     "actionButton": "Button",
     "bookmarkNavigator": "Bookmark navigator",
     "image": "Image",
@@ -58,8 +48,33 @@ vis_type_mapping = {
     "pageNavigator": "Page navigator",
     "shape": "Shape",
     "Group": "Group",
+    "listSlicer": "List Slicer",
+    "advancedSlicerVisual": "Button Slicer",
+    "FlowVisual_C29F1DCC_81F5_4973_94AD_0517D44CC06A": "Power Automate for Power BI",
 }
 
+
+def generate_visual_file_path(page_file_path: str, visual_id: str) -> str:
+
+    return page_file_path.split("/page.json")[0] + f"/visuals/{visual_id}.json"
+
+
+def resolve_visual_type(visual_type: str) -> str:
+    vt_lower = visual_type.lower()
+
+    vis_map_lower = {k.lower(): v for k, v in vis_type_mapping.items()}
+    flipped_lower = {v.lower(): k for k, v in vis_type_mapping.items()}
+
+    if vt_lower in vis_map_lower:
+        resolved = vis_map_lower.get(vt_lower)
+    elif vt_lower in flipped_lower:
+        resolved = flipped_lower.get(vt_lower)
+    else:
+        raise ValueError(f"{icons.red_dot} Unknown visual type: {visual_type}")
+
+    return resolved
+
+
 page_type_mapping = {
     (320, 240): "Tooltip",
     (816, 1056): "Letter",
@@ -70,22 +85,6 @@ page_type_mapping = {
 page_types = ["Tooltip", "Letter", "4:3", "16:9"]
 
 
-def get_web_url(report: str, workspace: Optional[str | UUID] = None):
-
-    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
-
-    dfR = fabric.list_reports(workspace=workspace_id)
-    dfR_filt = dfR[dfR["Name"] == report]
-
-    if len(dfR_filt) == 0:
-        raise ValueError(
-            f"{icons.red_dot} The '{report}' report does not exist within the '{workspace_name}' workspace."
-        )
-    web_url = dfR_filt["Web Url"].iloc[0]
-
-    return web_url
-
-
 def populate_custom_visual_display_names():
 
     url = "https://catalogapi.azure.com/offers?api-version=2018-08-01-beta&storefront=appsource&$filter=offerType+eq+%27PowerBIVisuals%27"
@@ -128,106 +127,6 @@ def populate_custom_visual_display_names():
         vis_type_mapping[vizId] = displayName
 
 
-def resolve_page_name(self, page_name: str) -> Tuple[str, str, str]:
-
-    dfP = self.list_pages()
-    if any(r["Page Name"] == page_name for _, r in dfP.iterrows()):
-        valid_page_name = page_name
-        dfP_filt = dfP[dfP["Page Name"] == page_name]
-        valid_display_name = dfP_filt["Page Display Name"].iloc[0]
-        file_path = dfP_filt["File Path"].iloc[0]
-    elif any(r["Page Display Name"] == page_name for _, r in dfP.iterrows()):
-        valid_display_name = page_name
-        dfP_filt = dfP[dfP["Page Display Name"] == page_name]
-        valid_page_name = dfP_filt["Page Name"].iloc[0]
-        file_path = dfP_filt["File Path"].iloc[0]
-    else:
-        raise ValueError(
-            f"{icons.red_dot} Invalid page name. The '{page_name}' page does not exist in the '{self._report}' report within the '{self._workspace}' workspace."
-        )
-
-    return valid_page_name, valid_display_name, file_path
-
-
-def visual_page_mapping(self) -> Tuple[dict, dict]:
-
-    page_mapping = {}
-    visual_mapping = {}
-    rd = self.rdef
-    for _, r in rd.iterrows():
-        file_path = r["path"]
-        payload = r["payload"]
-        if file_path.endswith("/page.json"):
-            pattern_page = r"/pages/(.*?)/page.json"
-            page_name = re.search(pattern_page, file_path).group(1)
-            obj_file = base64.b64decode(payload).decode("utf-8")
-            obj_json = json.loads(obj_file)
-            page_id = obj_json.get("name")
-            page_display = obj_json.get("displayName")
-            page_mapping[page_name] = (page_id, page_display)
-    for _, r in rd.iterrows():
-        file_path = r["path"]
-        payload = r["payload"]
-        if file_path.endswith("/visual.json"):
-            pattern_page = r"/pages/(.*?)/visuals/"
-            page_name = re.search(pattern_page, file_path).group(1)
-            visual_mapping[file_path] = (
-                page_mapping.get(page_name)[0],
-                page_mapping.get(page_name)[1],
-            )
-
-    return page_mapping, visual_mapping
-
-
-def resolve_visual_name(
-    self, page_name: str, visual_name: str
-) -> Tuple[str, str, str, str]:
-    """
-    Obtains the page name, page display name, and the file path for a given page in a report.
-
-    Parameters
-    ----------
-    page_name : str
-        The name of the page of the report - either the page name (GUID) or the page display name.
-    visual_name : str
-        The name of the visual of the report.
-
-    Returns
-    -------
-    Tuple[str, str, str, str] Page name, page display name, visual name, file path from the report definition.
-
-    """
-
-    dfV = self.list_visuals()
-    if any(
-        (r["Page Name"] == page_name) & (r["Visual Name"] == visual_name)
-        for _, r in dfV.iterrows()
-    ):
-        valid_page_name = page_name
-        dfV_filt = dfV[
-            (dfV["Page Name"] == page_name) & (dfV["Visual Name"] == visual_name)
-        ]
-        file_path = dfV_filt["File Path"].iloc[0]
-        valid_display_name = dfV_filt["Page Display Name"].iloc[0]
-    elif any(
-        (r["Page Display Name"] == page_name) & (r["Visual Name"] == visual_name)
-        for _, r in dfV.iterrows()
-    ):
-        valid_display_name = page_name
-        dfV_filt = dfV[
-            (dfV["Page Display Name"] == page_name)
-            & (dfV["Visual Name"] == visual_name)
-        ]
-        file_path = dfV_filt["File Path"].iloc[0]
-        valid_page_name = dfV_filt["Page Name"].iloc[0]
-    else:
-        raise ValueError(
-            f"{icons.red_dot} Invalid page/visual name. The '{visual_name}' visual on the '{page_name}' page does not exist in the '{self._report}' report within the '{self._workspace}' workspace."
-        )
-
-    return valid_page_name, valid_display_name, visual_name, file_path
-
-
 def find_entity_property_pairs(data, result=None, keys_path=None):
 
     if result is None: