semantic-link-labs 0.7.4__py3-none-any.whl → 0.8.1__py3-none-any.whl

sempy_labs/_mirrored_warehouses.py

@@ -0,0 +1,50 @@
+import sempy.fabric as fabric
+import pandas as pd
+from typing import Optional
+from sempy_labs._helper_functions import (
+    resolve_workspace_name_and_id,
+    pagination,
+)
+from sempy.fabric.exceptions import FabricHTTPException
+
+
+def list_mirrored_warehouses(workspace: Optional[str] = None) -> pd.DataFrame:
+    """
+    Shows the mirrored warehouses within a workspace.
+
+    Parameters
+    ----------
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing the mirrored warehouses within a workspace.
+    """
+
+    df = pd.DataFrame(
+        columns=["Mirrored Warehouse Name", "Mirrored Warehouse Id", "Description"]
+    )
+
+    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
+
+    client = fabric.FabricRestClient()
+    response = client.get(f"/v1/workspaces/{workspace_id}/mirroredWarehouses")
+    if response.status_code != 200:
+        raise FabricHTTPException(response)
+    responses = pagination(client, response)
+
+    for r in responses:
+        for v in r.get("value", []):
+
+            new_data = {
+                "Mirrored Warehouse Name": v.get("displayName"),
+                "Mirrored Warehouse Id": v.get("id"),
+                "Description": v.get("description"),
+            }
+            df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
+
+    return df

sempy_labs/_ml_experiments.py

@@ -0,0 +1,122 @@
+import sempy.fabric as fabric
+import pandas as pd
+import sempy_labs._icons as icons
+from typing import Optional
+from sempy_labs._helper_functions import (
+    resolve_workspace_name_and_id,
+    lro,
+    pagination,
+)
+from sempy.fabric.exceptions import FabricHTTPException
+
+
+def list_ml_experiments(workspace: Optional[str] = None) -> pd.DataFrame:
+    """
+    Shows the ML experiments within a workspace.
+
+    Parameters
+    ----------
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing the ML models within a workspace.
+    """
+
+    df = pd.DataFrame(columns=["ML Experiment Name", "ML Experiment Id", "Description"])
+
+    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
+
+    client = fabric.FabricRestClient()
+    response = client.get(f"/v1/workspaces/{workspace_id}/mlExperiments")
+    if response.status_code != 200:
+        raise FabricHTTPException(response)
+
+    responses = pagination(client, response)
+
+    for r in responses:
+        for v in r.get("value", []):
+            model_id = v.get("id")
+            modelName = v.get("displayName")
+            desc = v.get("description")
+
+            new_data = {
+                "ML Experiment Name": modelName,
+                "ML Experiment Id": model_id,
+                "Description": desc,
+            }
+            df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
+
+    return df
+
+
+def create_ml_experiment(
+    name: str, description: Optional[str] = None, workspace: Optional[str] = None
+):
+    """
+    Creates a Fabric ML experiment.
+
+    Parameters
+    ----------
+    name: str
+        Name of the ML experiment.
+    description : str, default=None
+        A description of the environment.
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
+
+    request_body = {"displayName": name}
+
+    if description:
+        request_body["description"] = description
+
+    client = fabric.FabricRestClient()
+    response = client.post(
+        f"/v1/workspaces/{workspace_id}/mlExperiments", json=request_body
+    )
+
+    lro(client, response, status_codes=[201, 202])
+
+    print(
+        f"{icons.green_dot} The '{name}' ML experiment has been created within the '{workspace}' workspace."
+    )
+
+
+def delete_ml_experiment(name: str, workspace: Optional[str] = None):
+    """
+    Deletes a Fabric ML experiment.
+
+    Parameters
+    ----------
+    name: str
+        Name of the ML experiment.
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
+
+    item_id = fabric.resolve_item_id(
+        item_name=name, type="MLExperiment", workspace=workspace
+    )
+
+    client = fabric.FabricRestClient()
+    response = client.delete(f"/v1/workspaces/{workspace_id}/mlExperiments/{item_id}")
+
+    if response.status_code != 200:
+        raise FabricHTTPException(response)
+
+    print(
+        f"{icons.green_dot} The '{name}' ML experiment within the '{workspace}' workspace has been deleted."
+    )

sempy_labs/_ml_models.py

@@ -0,0 +1,120 @@
+import sempy.fabric as fabric
+import pandas as pd
+import sempy_labs._icons as icons
+from typing import Optional
+from sempy_labs._helper_functions import (
+    resolve_workspace_name_and_id,
+    lro,
+    pagination,
+)
+from sempy.fabric.exceptions import FabricHTTPException
+
+
+def list_ml_models(workspace: Optional[str] = None) -> pd.DataFrame:
+    """
+    Shows the ML models within a workspace.
+
+    Parameters
+    ----------
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas dataframe showing the ML models within a workspace.
+    """
+
+    df = pd.DataFrame(columns=["ML Model Name", "ML Model Id", "Description"])
+
+    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
+
+    client = fabric.FabricRestClient()
+    response = client.get(f"/v1/workspaces/{workspace_id}/mlModels")
+    if response.status_code != 200:
+        raise FabricHTTPException(response)
+
+    responses = pagination(client, response)
+
+    for r in responses:
+        for v in r.get("value", []):
+            model_id = v.get("id")
+            modelName = v.get("displayName")
+            desc = v.get("description")
+
+            new_data = {
+                "ML Model Name": modelName,
+                "ML Model Id": model_id,
+                "Description": desc,
+            }
+            df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
+
+    return df
+
+
+def create_ml_model(
+    name: str, description: Optional[str] = None, workspace: Optional[str] = None
+):
+    """
+    Creates a Fabric ML model.
+
+    Parameters
+    ----------
+    name: str
+        Name of the ML model.
+    description : str, default=None
+        A description of the environment.
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
+
+    request_body = {"displayName": name}
+
+    if description:
+        request_body["description"] = description
+
+    client = fabric.FabricRestClient()
+    response = client.post(f"/v1/workspaces/{workspace_id}/mlModels", json=request_body)
+
+    lro(client, response, status_codes=[201, 202])
+
+    print(
+        f"{icons.green_dot} The '{name}' ML model has been created within the '{workspace}' workspace."
+    )
+
+
+def delete_ml_model(name: str, workspace: Optional[str] = None):
+    """
+    Deletes a Fabric ML model.
+
+    Parameters
+    ----------
+    name: str
+        Name of the ML model.
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
+
+    item_id = fabric.resolve_item_id(
+        item_name=name, type="MLModel", workspace=workspace
+    )
+
+    client = fabric.FabricRestClient()
+    response = client.delete(f"/v1/workspaces/{workspace_id}/mlModels/{item_id}")
+
+    if response.status_code != 200:
+        raise FabricHTTPException(response)
+
+    print(
+        f"{icons.green_dot} The '{name}' ML model within the '{workspace}' workspace has been deleted."
+    )

sempy_labs/_model_auto_build.py

@@ -34,10 +34,6 @@ def model_auto_build(
         The Fabric workspace used by the lakehouse.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
-
-    Returns
-    -------
-
     """
 
     workspace = fabric.resolve_workspace_name(workspace)

sempy_labs/_model_bpa.py

@@ -3,7 +3,6 @@ import pandas as pd
 import warnings
 import datetime
 from IPython.display import display, HTML
-from pyspark.sql import SparkSession
 from sempy_labs._model_dependencies import get_model_calc_dependencies
 from sempy_labs._helper_functions import (
     format_dax_object_name,
@@ -13,6 +12,7 @@ from sempy_labs._helper_functions import (
     resolve_workspace_capacity,
     resolve_dataset_id,
     get_language_codes,
+    get_max_run_id,
 )
 from sempy_labs.lakehouse import get_lakehouse_tables, lakehouse_attached
 from sempy_labs.tom import connect_semantic_model
@@ -30,9 +30,9 @@ def run_model_bpa(
     dataset: str,
     rules: Optional[pd.DataFrame] = None,
     workspace: Optional[str] = None,
-    export: Optional[bool] = False,
-    return_dataframe: Optional[bool] = False,
-    extended: Optional[bool] = False,
+    export: bool = False,
+    return_dataframe: bool = False,
+    extended: bool = False,
     language: Optional[str] = None,
     **kwargs,
 ):
@@ -151,6 +151,7 @@ def run_model_bpa(
             def translate_using_spark(rule_file):
 
                 from synapse.ml.services import Translate
+                from pyspark.sql import SparkSession
 
                 rules_temp = rule_file.copy()
                 rules_temp = rules_temp.drop(["Expression", "URL", "Severity"], axis=1)
@@ -346,15 +347,11 @@ def run_model_bpa(
 
         dfExport["Severity"].replace(icons.severity_mapping, inplace=True)
 
-        spark = SparkSession.builder.getOrCreate()
-        query = f"SELECT MAX(RunId) FROM {lakehouse}.{delta_table_name}"
-
         if len(lakeT_filt) == 0:
             runId = 1
         else:
-            dfSpark = spark.sql(query)
-            maxRunId = dfSpark.collect()[0][0]
-            runId = maxRunId + 1
+            max_run_id = get_max_run_id(table_name=delta_table_name)
+            runId = max_run_id + 1
 
         now = datetime.datetime.now()
         dfD = fabric.list_datasets(workspace=workspace, mode="rest")
@@ -506,7 +503,7 @@ def run_model_bpa(
             content_html += f'<td>{row["Object Type"]}</td>'
             content_html += f'<td>{row["Object Name"]}</td>'
             content_html += f'<td style="text-align: center;">{row["Severity"]}</td>'
-            #content_html += f'<td>{row["Severity"]}</td>'
+            # content_html += f'<td>{row["Severity"]}</td>'
             content_html += "</tr>"
         content_html += "</table>"
 
@@ -514,4 +511,5 @@ def run_model_bpa(
     tab_html += "</div>"
 
     # Display the tabs, tab contents, and run the script
-    return display(HTML(styles + tab_html + content_html + script))
+    if not export:
+        return display(HTML(styles + tab_html + content_html + script))

sempy_labs/_model_bpa_bulk.py

@@ -1,14 +1,17 @@
 import sempy.fabric as fabric
 import pandas as pd
 import datetime
-from pyspark.sql import SparkSession
 from sempy_labs._helper_functions import (
     resolve_lakehouse_name,
     save_as_delta_table,
     resolve_workspace_capacity,
     retry,
+    get_max_run_id,
+)
+from sempy_labs.lakehouse import (
+    get_lakehouse_tables,
+    lakehouse_attached,
 )
-from sempy_labs.lakehouse import get_lakehouse_tables, lakehouse_attached
 from sempy_labs._model_bpa import run_model_bpa
 from typing import Optional, List
 from sempy._utils._log import log
@@ -18,7 +21,7 @@ import sempy_labs._icons as icons
 @log
 def run_model_bpa_bulk(
     rules: Optional[pd.DataFrame] = None,
-    extended: Optional[bool] = False,
+    extended: bool = False,
     language: Optional[str] = None,
     workspace: Optional[str | List[str]] = None,
     skip_models: Optional[str | List[str]] = ["ModelBPA", "Fabric Capacity Metrics"],
@@ -78,7 +81,6 @@ def run_model_bpa_bulk(
     ]
     now = datetime.datetime.now()
     output_table = "modelbparesults"
-    spark = SparkSession.builder.getOrCreate()
     lakehouse_workspace = fabric.resolve_workspace_name()
     lakehouse_id = fabric.get_lakehouse_id()
     lakehouse = resolve_lakehouse_name(
@@ -90,9 +92,8 @@ def run_model_bpa_bulk(
     if len(lakeT_filt) == 0:
         runId = 1
     else:
-        dfSpark = spark.table(f"`{lakehouse_id}`.{output_table}").select(F.max("RunId"))
-        maxRunId = dfSpark.collect()[0][0]
-        runId = maxRunId + 1
+        max_run_id = get_max_run_id(table_name=output_table)
+        runId = max_run_id + 1
 
     if isinstance(workspace, str):
         workspace = [workspace]

sempy_labs/_model_dependencies.py

@@ -74,7 +74,7 @@ def get_measure_dependencies(
 
                 for index, dependency in dep_filt.iterrows():
                     d = True
-                    if dependency[5] == "Measure":
+                    if dependency.iloc[5] == "Measure":
                         d = False
                         df = pd.concat(
                             [
@@ -85,12 +85,14 @@ def get_measure_dependencies(
                                             "Table Name": r["Table Name"],
                                             "Object Name": r["Object Name"],
                                             "Object Type": r["Object Type"],
-                                            "Referenced Object": dependency[4],
-                                            "Referenced Table": dependency[3],
-                                            "Referenced Object Type": dependency[5],
+                                            "Referenced Object": dependency.iloc[4],
+                                            "Referenced Table": dependency.iloc[3],
+                                            "Referenced Object Type": dependency.iloc[
+                                                5
+                                            ],
                                             "Done": d,
                                             "Full Object Name": r["Full Object Name"],
-                                            "Referenced Full Object Name": dependency[
+                                            "Referenced Full Object Name": dependency.iloc[
                                                 7
                                             ],
                                             "Parent Node": rObj,
@@ -110,12 +112,14 @@ def get_measure_dependencies(
                                             "Table Name": r["Table Name"],
                                             "Object Name": r["Object Name"],
                                             "Object Type": r["Object Type"],
-                                            "Referenced Object": dependency[4],
-                                            "Referenced Table": dependency[3],
-                                            "Referenced Object Type": dependency[5],
+                                            "Referenced Object": dependency.iloc[4],
+                                            "Referenced Table": dependency.iloc[3],
+                                            "Referenced Object Type": dependency.iloc[
+                                                5
+                                            ],
                                             "Done": d,
                                             "Full Object Name": r["Full Object Name"],
-                                            "Referenced Full Object Name": dependency[
+                                            "Referenced Full Object Name": dependency.iloc[
                                                 7
                                             ],
                                             "Parent Node": rObj,
@@ -203,7 +207,7 @@ def get_model_calc_dependencies(
 
                 for index, dependency in dep_filt.iterrows():
                     d = True
-                    if dependency[5] in objs:
+                    if dependency.iloc[5] in objs:
                         d = False
                         df = pd.concat(
                             [
@@ -214,12 +218,14 @@ def get_model_calc_dependencies(
                                             "Table Name": r["Table Name"],
                                             "Object Name": r["Object Name"],
                                             "Object Type": r["Object Type"],
-                                            "Referenced Object": dependency[4],
-                                            "Referenced Table": dependency[3],
-                                            "Referenced Object Type": dependency[5],
+                                            "Referenced Object": dependency.iloc[4],
+                                            "Referenced Table": dependency.iloc[3],
+                                            "Referenced Object Type": dependency.iloc[
+                                                5
+                                            ],
                                             "Done": d,
                                             "Full Object Name": r["Full Object Name"],
-                                            "Referenced Full Object Name": dependency[
+                                            "Referenced Full Object Name": dependency.iloc[
                                                 7
                                             ],
                                             "Parent Node": rObj,
@@ -239,12 +245,14 @@ def get_model_calc_dependencies(
                                             "Table Name": r["Table Name"],
                                             "Object Name": r["Object Name"],
                                             "Object Type": r["Object Type"],
-                                            "Referenced Object": dependency[5],
-                                            "Referenced Table": dependency[4],
-                                            "Referenced Object Type": dependency[6],
+                                            "Referenced Object": dependency.iloc[5],
+                                            "Referenced Table": dependency.iloc[4],
+                                            "Referenced Object Type": dependency.iloc[
+                                                6
+                                            ],
                                             "Done": d,
                                             "Full Object Name": r["Full Object Name"],
-                                            "Referenced Full Object Name": dependency[
+                                            "Referenced Full Object Name": dependency.iloc[
                                                 7
                                             ],
                                             "Parent Node": rObj,

sempy_labs/_notebooks.py

@@ -8,13 +8,14 @@ from sempy_labs._helper_functions import (
     resolve_workspace_name_and_id,
     lro,
     _decode_b64,
+    resolve_notebook_id,
 )
 from sempy.fabric.exceptions import FabricHTTPException
 
 
 def get_notebook_definition(
-    notebook_name: str, workspace: Optional[str] = None, decode: Optional[bool] = True
-):
+    notebook_name: str, workspace: Optional[str] = None, decode: bool = True
+) -> str:
     """
     Obtains the notebook definition.
 
@@ -32,21 +33,12 @@ def get_notebook_definition(
 
     Returns
     -------
-    ipynb
+    str
         The notebook definition.
     """
 
     (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
-
-    dfI = fabric.list_items(workspace=workspace, type="Notebook")
-    dfI_filt = dfI[dfI["Display Name"] == notebook_name]
-
-    if len(dfI_filt) == 0:
-        raise ValueError(
-            f"{icons.red_dot} The '{notebook_name}' notebook does not exist within the '{workspace}' workspace."
-        )
-
-    notebook_id = dfI_filt["Id"].iloc[0]
+    notebook_id = resolve_notebook_id(notebook=notebook_name, workspace=workspace)
     client = fabric.FabricRestClient()
     response = client.post(
         f"v1/workspaces/{workspace_id}/notebooks/{notebook_id}/getDefinition",
@@ -90,9 +82,6 @@ def import_notebook_from_web(
         The name of the workspace.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
-
-    Returns
-    -------
     """
 
     (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)

sempy_labs/_query_scale_out.py

@@ -181,8 +181,8 @@ def disable_qso(dataset: str, workspace: Optional[str] = None) -> pd.DataFrame:
 
 def set_qso(
     dataset: str,
-    auto_sync: Optional[bool] = True,
-    max_read_only_replicas: Optional[int] = -1,
+    auto_sync: bool = True,
+    max_read_only_replicas: int = -1,
     workspace: Optional[str] = None,
 ) -> pd.DataFrame:
     """
@@ -339,7 +339,6 @@ def list_qso_settings(
     if dataset is not None:
         dataset_id = resolve_dataset_id(dataset, workspace)
 
-    workspace_id = fabric.get_workspace_id()
     df = pd.DataFrame(
         columns=[
             "Dataset Id",
@@ -413,10 +412,12 @@ def set_workspace_default_storage_format(
     dfW = fabric.list_workspaces(filter=f"name eq '{workspace}'")
     if len(dfW) == 0:
         raise ValueError()
-    current_storage_format = dfW['Default Dataset Storage Format'].iloc[0]
+    current_storage_format = dfW["Default Dataset Storage Format"].iloc[0]
 
     if current_storage_format == storage_format:
-        print(f"{icons.info} The '{workspace}' is already set to a default storage format of '{current_storage_format}'.")
+        print(
+            f"{icons.info} The '{workspace}' is already set to a default storage format of '{current_storage_format}'."
+        )
         return
 
     request_body = {

sempy_labs/_refresh_semantic_model.py

@@ -13,10 +13,10 @@ def refresh_semantic_model(
     dataset: str,
     tables: Optional[Union[str, List[str]]] = None,
     partitions: Optional[Union[str, List[str]]] = None,
-    refresh_type: Optional[str] = None,
-    retry_count: Optional[int] = 0,
-    apply_refresh_policy: Optional[bool] = True,
-    max_parallelism: Optional[int] = 10,
+    refresh_type: str = "full",
+    retry_count: int = 0,
+    apply_refresh_policy: bool = True,
+    max_parallelism: int = 10,
     workspace: Optional[str] = None,
 ):
     """
@@ -30,7 +30,7 @@ def refresh_semantic_model(
         A string or a list of tables to refresh.
     partitions: str, List[str], default=None
         A string or a list of partitions to refresh. Partitions must be formatted as such: 'Table Name'[Partition Name].
-    refresh_type : str, default='full'
+    refresh_type : str, default="full"
         The type of processing to perform. Types align with the TMSL refresh command types: full, clearValues, calculate, dataOnly, automatic, and defragment. The add type isn't supported. Defaults to "full".
     retry_count : int, default=0
         Number of times the operation retries before failing.
@@ -48,9 +48,6 @@ def refresh_semantic_model(
 
     workspace = fabric.resolve_workspace_name(workspace)
 
-    if refresh_type is None:
-        refresh_type = "full"
-
     if isinstance(tables, str):
         tables = [tables]
     if isinstance(partitions, str):
@@ -74,18 +71,9 @@ def refresh_semantic_model(
         refresh_type.lower().replace("only", "Only").replace("values", "Values")
     )
 
-    refreshTypes = [
-        "full",
-        "automatic",
-        "dataOnly",
-        "calculate",
-        "clearValues",
-        "defragment",
-    ]
-
-    if refresh_type not in refreshTypes:
+    if refresh_type not in icons.refreshTypes:
         raise ValueError(
-            f"{icons.red_dot} Invalid refresh type. Refresh type must be one of these values: {refreshTypes}."
+            f"{icons.red_dot} Invalid refresh type. Refresh type must be one of these values: {icons.refreshTypes}."
         )
 
     if len(objects) == 0: