semantic-link-labs 0.10.0__py3-none-any.whl → 0.11.0__py3-none-any.whl

sempy_labs/_ml_models.py

@@ -1,15 +1,17 @@
 import pandas as pd
 from typing import Optional
 from sempy_labs._helper_functions import (
-    resolve_workspace_name_and_id,
+    resolve_workspace_id,
     _base_api,
     delete_item,
     _create_dataframe,
     create_item,
 )
 from uuid import UUID
+from sempy._utils._log import log
 
 
+@log
 def list_ml_models(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
     """
     Shows the ML models within a workspace.
@@ -36,7 +38,7 @@ def list_ml_models(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
     }
     df = _create_dataframe(columns=columns)
 
-    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    workspace_id = resolve_workspace_id(workspace)
 
     responses = _base_api(
         request=f"/v1/workspaces/{workspace_id}/mlModels",
@@ -44,6 +46,7 @@ def list_ml_models(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
         uses_pagination=True,
     )
 
+    dfs = []
     for r in responses:
         for v in r.get("value", []):
             model_id = v.get("id")
@@ -55,11 +58,15 @@ def list_ml_models(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
                 "ML Model Id": model_id,
                 "Description": desc,
             }
-            df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
+            dfs.append(pd.DataFrame(new_data, index=[0]))
+
+    if dfs:
+        df = pd.concat(dfs, ignore_index=True)
 
     return df
 
 
+@log
 def create_ml_model(
     name: str, description: Optional[str] = None, workspace: Optional[str | UUID] = None
 ):
@@ -83,6 +90,7 @@ def create_ml_model(
     create_item(name=name, description=description, type="MLModel", workspace=workspace)
 
 
+@log
 def delete_ml_model(name: str | UUID, workspace: Optional[str | UUID] = None):
     """
     Deletes a Fabric ML model.

sempy_labs/_model_bpa_rules.py

@@ -2,8 +2,10 @@ import sempy
 import pandas as pd
 import re
 from typing import Optional
+from sempy._utils._log import log
 
 
+@log
 def model_bpa_rules(
     dependencies: Optional[pd.DataFrame] = None,
     **kwargs,

sempy_labs/_mounted_data_factories.py

@@ -1,20 +1,19 @@
 import pandas as pd
-import json
 from typing import Optional
 from sempy_labs._helper_functions import (
-    resolve_workspace_name_and_id,
+    resolve_workspace_id,
     _base_api,
     _create_dataframe,
     _update_dataframe_datatypes,
-    resolve_item_id,
-    _decode_b64,
-    delete_item,
     get_item_definition,
+    delete_item,
 )
 
 from uuid import UUID
+from sempy._utils._log import log
 
 
+@log
 def list_mounted_data_factories(
     workspace: Optional[str | UUID] = None,
 ) -> pd.DataFrame:
@@ -36,7 +35,7 @@ def list_mounted_data_factories(
         A pandas dataframe showing a list of mounted data factories from the specified workspace.
     """
 
-    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    workspace_id = resolve_workspace_id(workspace)
 
     columns = {
         "Mounted Data Factory Name": "str",
@@ -50,6 +49,7 @@ def list_mounted_data_factories(
         uses_pagination=True,
     )
 
+    dfs = []
     for r in responses:
         for v in r.get("value", []):
             new_data = {
@@ -58,13 +58,16 @@ def list_mounted_data_factories(
                 "Description": v.get("description"),
             }
 
-            df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
+            dfs.append(pd.DataFrame(new_data, index=[0]))
 
-    _update_dataframe_datatypes(dataframe=df, column_map=columns)
+    if dfs:
+        df = pd.concat(dfs, ignore_index=True)
+        _update_dataframe_datatypes(dataframe=df, column_map=columns)
 
     return df
 
 
+@log
 def get_mounted_data_factory_definition(
     mounted_data_factory: str | UUID, workspace: Optional[str | UUID] = None
 ) -> dict:
@@ -96,6 +99,7 @@ def get_mounted_data_factory_definition(
     )
 
 
+@log
 def delete_mounted_data_factory(
     mounted_data_factory: str | UUID, workspace: Optional[str | UUID]
 ):

sempy_labs/_notebooks.py

@@ -60,6 +60,7 @@ def _get_notebook_type(
     return file_extension[1:]
 
 
+@log
 def get_notebook_definition(
     notebook_name: str,
     workspace: Optional[str | UUID] = None,
@@ -159,6 +160,7 @@ def import_notebook_from_web(
             notebook_content=response.content,
             workspace=workspace_id,
             description=description,
+            format="ipynb",
         )
     elif len(dfI_filt) > 0 and overwrite:
         print(f"{icons.info} Overwrite of notebooks is currently not supported.")
@@ -171,6 +173,7 @@ def import_notebook_from_web(
         )
 
 
+@log
 def create_notebook(
     name: str,
     notebook_content: str,
@@ -202,9 +205,8 @@ def create_notebook(
         otherwise notebook_content should be GIT friendly format
     """
 
-    notebook_payload = base64.b64encode(notebook_content.encode("utf-8")).decode(
-        "utf-8"
-    )
+    notebook_payload = base64.b64encode(notebook_content).decode("utf-8")
+
     definition_payload = {
         "parts": [
             {
@@ -227,6 +229,7 @@ def create_notebook(
     )
 
 
+@log
 def update_notebook_definition(
     name: str,
     notebook_content: str,

sempy_labs/_refresh_semantic_model.py

@@ -321,6 +321,7 @@ def cancel_dataset_refresh(
     )
 
 
+@log
 def get_semantic_model_refresh_history(
     dataset: str | UUID,
     request_id: Optional[str] = None,

sempy_labs/_semantic_models.py

@@ -8,11 +8,15 @@ from sempy_labs._helper_functions import (
     resolve_workspace_name_and_id,
     resolve_dataset_name_and_id,
     delete_item,
+    resolve_dataset_id,
+    resolve_workspace_id,
 )
 import sempy_labs._icons as icons
 import re
+from sempy._utils._log import log
 
 
+@log
 def get_semantic_model_refresh_schedule(
     dataset: str | UUID, workspace: Optional[str | UUID] = None
 ) -> pd.DataFrame:
@@ -70,6 +74,7 @@ def get_semantic_model_refresh_schedule(
     return df
 
 
+@log
 def enable_semantic_model_scheduled_refresh(
     dataset: str | UUID,
     workspace: Optional[str | UUID] = None,
@@ -119,6 +124,7 @@ def enable_semantic_model_scheduled_refresh(
         )
 
 
+@log
 def delete_semantic_model(dataset: str | UUID, workspace: Optional[str | UUID] = None):
     """
     Deletes a semantic model.
@@ -138,6 +144,7 @@ def delete_semantic_model(dataset: str | UUID, workspace: Optional[str | UUID] =
     delete_item(item=dataset, type="SemanticModel", workspace=workspace)
 
 
+@log
 def update_semantic_model_refresh_schedule(
     dataset: str | UUID,
     days: Optional[str | List[str]] = None,
@@ -227,3 +234,103 @@ def update_semantic_model_refresh_schedule(
     print(
         f"{icons.green_dot} Refresh schedule for the '{dataset_name}' within the '{workspace_name}' workspace has been updated."
     )
+
+
+@log
+def list_semantic_model_datasources(
+    dataset: str | UUID,
+    workspace: Optional[str | UUID] = None,
+    expand_details: bool = True,
+) -> pd.DataFrame:
+    """
+    Lists the data sources for the specified semantic model.
+
+    This is a wrapper function for the following API: `Datasets - Get Datasources In Group <https://learn.microsoft.com/rest/api/power-bi/datasets/get-datasources-in-group>`_.
+
+    Parameters
+    ----------
+    dataset : str | uuid.UUID
+        Name or ID of the semantic model.
+    workspace : str | uuid.UUID, default=None
+        The workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    expand_details : bool, default=True
+        If True, expands the connection details for each data source.
+
+    Returns
+    -------
+    pandas.DataFrame
+        DataFrame containing the data sources for the specified semantic model.
+    """
+
+    workspace_id = resolve_workspace_id(workspace)
+    dataset_id = resolve_dataset_id(dataset, workspace_id)
+
+    if expand_details:
+        columns = {
+            "Datasource Type": "str",
+            "Connection Server": "str",
+            "Connection Database": "str",
+            "Connection Path": "str",
+            "Connection Account": "str",
+            "Connection Domain": "str",
+            "Connection Kind": "str",
+            "Connection Email Address": "str",
+            "Connection URL": "str",
+            "Connection Class Info": "str",
+            "Connection Login Server": "str",
+            "Datasource Id": "str",
+            "Gateway Id": "str",
+        }
+    else:
+        columns = {
+            "Datasource Type": "str",
+            "Connection Details": "str",
+            "Datasource Id": "str",
+            "Gateway Id": "str",
+        }
+
+    df = _create_dataframe(columns)
+
+    response = _base_api(
+        request=f"/v1.0/myorg/groups/{workspace_id}/datasets/{dataset_id}/datasources",
+        client="fabric_sp",
+    )
+
+    dfs = []
+    for item in response.json().get("value", []):
+        ds_type = item.get("datasourceType")
+        conn_details = item.get("connectionDetails", {})
+        ds_id = item.get("datasourceId")
+        gateway_id = item.get("gatewayId")
+        if expand_details:
+            new_data = {
+                "Datasource Type": ds_type,
+                "Connection Server": conn_details.get("server"),
+                "Connection Database": conn_details.get("database"),
+                "Connection Path": conn_details.get("path"),
+                "Connection Account": conn_details.get("account"),
+                "Connection Domain": conn_details.get("domain"),
+                "Connection Kind": conn_details.get("kind"),
+                "Connection Email Address": conn_details.get("emailAddress"),
+                "Connection URL": conn_details.get("url"),
+                "Connection Class Info": conn_details.get("classInfo"),
+                "Connection Login Server": conn_details.get("loginServer"),
+                "Datasource Id": ds_id,
+                "Gateway Id": gateway_id,
+            }
+            dfs.append(pd.DataFrame(new_data, index=[0]))
+        else:
+            new_data = {
+                "Datasource Type": ds_type,
+                "Connection Details": conn_details,
+                "Datasource Id": ds_id,
+                "Gateway Id": gateway_id,
+            }
+            dfs.append(pd.DataFrame([new_data]))
+
+    if dfs:
+        df = pd.concat(dfs, ignore_index=True)
+
+    return df

sempy_labs/_spark.py

@@ -8,8 +8,10 @@ from sempy_labs._helper_functions import (
     _create_dataframe,
 )
 from uuid import UUID
+from sempy._utils._log import log
 
 
+@log
 def list_custom_pools(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
     """
     Lists all `custom pools <https://learn.microsoft.com/fabric/data-engineering/create-custom-spark-pools>`_ within a workspace.
@@ -73,6 +75,7 @@ def list_custom_pools(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
     return df
 
 
+@log
 def create_custom_pool(
     pool_name: str,
     node_size: str,
@@ -145,6 +148,7 @@ def create_custom_pool(
     )
 
 
+@log
 def update_custom_pool(
     pool_name: str,
     node_size: Optional[str] = None,
@@ -251,6 +255,7 @@ def update_custom_pool(
     )
 
 
+@log
 def delete_custom_pool(pool_name: str, workspace: Optional[str | UUID] = None):
     """
     Deletes a `custom pool <https://learn.microsoft.com/fabric/data-engineering/create-custom-spark-pools>`_ within a workspace.
@@ -286,6 +291,7 @@ def delete_custom_pool(pool_name: str, workspace: Optional[str | UUID] = None):
     )
 
 
+@log
 def get_spark_settings(
     workspace: Optional[str | UUID] = None, return_dataframe: bool = True
 ) -> pd.DataFrame | dict:
@@ -362,6 +368,7 @@ def get_spark_settings(
         return response.json()
 
 
+@log
 def update_spark_settings(
     automatic_log_enabled: Optional[bool] = None,
     high_concurrency_enabled: Optional[bool] = None,

sempy_labs/_sql_endpoints.py

@@ -0,0 +1,208 @@
+from typing import Optional, Literal
+from uuid import UUID
+import pandas as pd
+from sempy_labs._helper_functions import (
+    _base_api,
+    _create_dataframe,
+    resolve_workspace_name_and_id,
+    resolve_item_name_and_id,
+    _update_dataframe_datatypes,
+    resolve_workspace_id,
+)
+import sempy_labs._icons as icons
+from sempy._utils._log import log
+
+
+@log
+def list_sql_endpoints(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
+    """
+    Shows the SQL endpoints within a workspace.
+
+    Parameters
+    ----------
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing the SQL endpoints within a workspace.
+    """
+
+    columns = {
+        "SQL Endpoint Id": "string",
+        "SQL Endpoint Name": "string",
+        "Description": "string",
+    }
+    df = _create_dataframe(columns=columns)
+
+    workspace_id = resolve_workspace_id(workspace)
+
+    responses = _base_api(
+        request=f"/v1/workspaces/{workspace_id}/sqlEndpoints", uses_pagination=True
+    )
+
+    dfs = []
+    for r in responses:
+        for v in r.get("value", []):
+
+            new_data = {
+                "SQL Endpoint Id": v.get("id"),
+                "SQL Endpoint Name": v.get("displayName"),
+                "Description": v.get("description"),
+            }
+            dfs.append(pd.DataFrame(new_data, index=[0]))
+
+    if dfs:
+        df = pd.concat(dfs, ignore_index=True)
+
+    return df
+
+
+@log
+def refresh_sql_endpoint_metadata(
+    item: str | UUID,
+    type: Literal["Lakehouse", "MirroredDatabase"],
+    workspace: Optional[str | UUID] = None,
+    tables: dict[str, list[str]] = None,
+) -> pd.DataFrame:
+    """
+    Refreshes the metadata of a SQL endpoint.
+
+    This is a wrapper function for the following API: `Items - Refresh Sql Endpoint Metadata <https://learn.microsoft.com/rest/api/fabric/sqlendpoint/items/refresh-sql-endpoint-metadata>`_.
+
+    Parameters
+    ----------
+    item : str | uuid.UUID
+        The name or ID of the item (Lakehouse or MirroredDatabase).
+    type : Literal['Lakehouse', 'MirroredDatabase']
+        The type of the item. Must be 'Lakehouse' or 'MirroredDatabase'.
+    workspace : str | uuid.UUID, default=None
+        The Fabric workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    tables : dict[str, list[str]], default=None
+        A dictionary where the keys are schema names and the values are lists of table names.
+        If empty, all table metadata will be refreshed.
+
+        Example:
+        {
+            "dbo": ["DimDate", "DimGeography"],
+            "sls": ["FactSales", "FactBudget"],
+        }
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing the status of the metadata refresh operation.
+    """
+
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+
+    (item_name, item_id) = resolve_item_name_and_id(
+        item=item, type=type, workspace=workspace
+    )
+
+    if type == "Lakehouse":
+        response = _base_api(
+            request=f"/v1/workspaces/{workspace_id}/lakehouses/{item_id}",
+            client="fabric_sp",
+        )
+        sql_endpoint_id = (
+            response.json()
+            .get("properties", {})
+            .get("sqlEndpointProperties", {})
+            .get("id")
+        )
+    elif type == "MirroredDatabase":
+        response = _base_api(
+            request=f"/v1/workspaces/{workspace_id}/mirroredDatabases/{item_id}",
+            client="fabric_sp",
+        )
+        sql_endpoint_id = (
+            response.json()
+            .get("properties", {})
+            .get("sqlEndpointProperties", {})
+            .get("id")
+        )
+    else:
+        raise ValueError("Invalid type. Must be 'Lakehouse' or 'MirroredDatabase'.")
+
+    payload = {}
+    if tables:
+        payload = {
+            "tableDefinitions": [
+                {"schema": schema, "tableNames": tables}
+                for schema, tables in tables.items()
+            ]
+        }
+
+    result = _base_api(
+        request=f"v1/workspaces/{workspace_id}/sqlEndpoints/{sql_endpoint_id}/refreshMetadata?preview=true",
+        method="post",
+        status_codes=[200, 202],
+        lro_return_json=True,
+        payload=payload,
+    )
+
+    columns = {
+        "Table Name": "string",
+        "Status": "string",
+        "Start Time": "datetime",
+        "End Time": "datetime",
+        "Last Successful Sync Time": "datetime",
+        "Error Code": "string",
+        "Error Message": "string",
+    }
+
+    if result:
+        df = pd.json_normalize(result)
+
+        # Extract error code and message, set to None if no error
+        df["Error Code"] = df.get("error.errorCode", None)
+        df["Error Message"] = df.get("error.message", None)
+
+        # Friendly column renaming
+        df.rename(
+            columns={
+                "tableName": "Table Name",
+                "startDateTime": "Start Time",
+                "endDateTime": "End Time",
+                "status": "Status",
+                "lastSuccessfulSyncDateTime": "Last Successful Sync Time",
+            },
+            inplace=True,
+        )
+
+        # Drop the original 'error' column if present
+        df.drop(columns=[col for col in ["error"] if col in df.columns], inplace=True)
+
+        # Optional: Reorder columns
+        column_order = [
+            "Table Name",
+            "Status",
+            "Start Time",
+            "End Time",
+            "Last Successful Sync Time",
+            "Error Code",
+            "Error Message",
+        ]
+        df = df[column_order]
+
+        printout = f"{icons.green_dot} The metadata of the SQL endpoint for the '{item_name}' {type.lower()} within the '{workspace_name}' workspace has been refreshed"
+        if tables:
+            print(f"{printout} for the following tables: {tables}.")
+        else:
+            print(f"{printout} for all tables.")
+    else:
+        # If the target item has no tables to refresh the metadata for
+        df = pd.DataFrame(columns=columns.keys())
+        print(
+            f"{icons.yellow_dot} The SQL endpoint '{item_name}' {type.lower()} within the '{workspace_name}' workspace has no tables to refresh..."
+        )
+
+    _update_dataframe_datatypes(df, columns)
+
+    return df

sempy_labs/_sqldatabase.py

@@ -1,5 +1,5 @@
 from sempy_labs._helper_functions import (
-    resolve_workspace_name_and_id,
+    resolve_workspace_id,
     _base_api,
     _create_dataframe,
     _update_dataframe_datatypes,
@@ -9,8 +9,10 @@ from sempy_labs._helper_functions import (
 import pandas as pd
 from typing import Optional
 from uuid import UUID
+from sempy._utils._log import log
 
 
+@log
 def create_sql_database(
     name: str, description: Optional[str] = None, workspace: Optional[str | UUID] = None
 ):
@@ -36,6 +38,7 @@ def create_sql_database(
     )
 
 
+@log
 def delete_sql_database(
     sql_database: str | UUID, workspace: Optional[str | UUID] = None
 ):
@@ -57,6 +60,7 @@ def delete_sql_database(
     delete_item(item=sql_database, type="SQLDatabase", workspace=workspace)
 
 
+@log
 def list_sql_databases(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
     """
     Lists all SQL databases in the Fabric workspace.
@@ -78,7 +82,7 @@ def list_sql_databases(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
         A pandas dataframe showing a list of SQL databases in the Fabric workspace.
     """
 
-    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    workspace_id = resolve_workspace_id(workspace)
 
     columns = {
         "SQL Database Name": "string",
@@ -96,6 +100,7 @@ def list_sql_databases(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
         client="fabric_sp",
     )
 
+    dfs = []
     for r in responses:
         for v in r.get("value", []):
             prop = v.get("properties", {})
@@ -108,13 +113,16 @@ def list_sql_databases(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
                 "Server FQDN": prop.get("serverFqdn"),
             }
 
-            df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
+            dfs.append(pd.DataFrame(new_data, index=[0]))
 
-    _update_dataframe_datatypes(dataframe=df, column_map=columns)
+    if dfs:
+        df = pd.concat(dfs, ignore_index=True)
+        _update_dataframe_datatypes(dataframe=df, column_map=columns)
 
     return df
 
 
+@log
 def get_sql_database_tables(
     sql_database: str | UUID, workspace: Optional[str | UUID] = None
 ) -> pd.DataFrame:
@@ -150,6 +158,7 @@ def get_sql_database_tables(
     return df
 
 
+@log
 def get_sql_database_columns(
     sql_database: str | UUID, workspace: Optional[str | UUID] = None
 ) -> pd.DataFrame: