semantic-link-labs 0.8.2__py3-none-any.whl → 0.8.4__py3-none-any.whl

sempy_labs/_model_bpa.py

@@ -12,7 +12,7 @@ from sempy_labs._helper_functions import (
     resolve_workspace_capacity,
     resolve_dataset_id,
     get_language_codes,
-    get_max_run_id,
+    _get_max_run_id,
 )
 from sempy_labs.lakehouse import get_lakehouse_tables, lakehouse_attached
 from sempy_labs.tom import connect_semantic_model
@@ -56,7 +56,7 @@ def run_model_bpa(
     extended : bool, default=False
         If True, runs the set_vertipaq_annotations function to collect Vertipaq Analyzer statistics to be used in the analysis of the semantic model.
     language : str, default=None
-        Specifying a language code (i.e. 'it-IT' for Italian) will auto-translate the Category, Rule Name and Description into the specified language.
+        Specifying a language name or code (i.e. 'it-IT' for Italian) will auto-translate the Category, Rule Name and Description into the specified language.
         Defaults to None which resolves to English.
 
     Returns
@@ -113,6 +113,7 @@ def run_model_bpa(
         )
 
     if extended:
+        icons.sll_tags.append("ModelBPAExtended")
         with connect_semantic_model(
             dataset=dataset, workspace=workspace, readonly=False
         ) as tom:
@@ -220,9 +221,9 @@ def run_model_bpa(
 
             rules = translate_using_spark(rules)
 
-        rules["Severity"].replace("Warning", icons.warning, inplace=True)
-        rules["Severity"].replace("Error", icons.error, inplace=True)
-        rules["Severity"].replace("Info", icons.info, inplace=True)
+        rules.loc[rules["Severity"] == "Warning", "Severity"] = icons.warning
+        rules.loc[rules["Severity"] == "Error", "Severity"] = icons.error
+        rules.loc[rules["Severity"] == "Info", "Severity"] = icons.info
 
         pd.set_option("display.max_colwidth", 1000)
 
@@ -350,7 +351,7 @@ def run_model_bpa(
         if len(lakeT_filt) == 0:
             runId = 1
         else:
-            max_run_id = get_max_run_id(
+            max_run_id = _get_max_run_id(
                 lakehouse=lakehouse, table_name=delta_table_name
             )
             runId = max_run_id + 1
@@ -387,10 +388,14 @@ def run_model_bpa(
         dfExport.insert(5, colName, dfExport.pop(colName))
 
         dfExport.columns = dfExport.columns.str.replace(" ", "_")
+        schema = {
+            key.replace(" ", "_"): value for key, value in icons.bpa_schema.items()
+        }
         save_as_delta_table(
             dataframe=dfExport,
             delta_table_name=delta_table_name,
             write_mode="append",
+            schema=schema,
             merge_schema=True,
         )
 

sempy_labs/_model_bpa_bulk.py

@@ -6,7 +6,7 @@ from sempy_labs._helper_functions import (
     save_as_delta_table,
     resolve_workspace_capacity,
     retry,
-    get_max_run_id,
+    _get_max_run_id,
 )
 from sempy_labs.lakehouse import (
     get_lakehouse_tables,
@@ -49,8 +49,6 @@ def run_model_bpa_bulk(
         The semantic models to always skip when running this analysis.
     """
 
-    import pyspark.sql.functions as F
-
     if not lakehouse_attached():
         raise ValueError(
             f"{icons.red_dot} No lakehouse is attached to this notebook. Must attach a lakehouse to the notebook."
@@ -61,24 +59,6 @@ def run_model_bpa_bulk(
 
     skip_models.extend(["ModelBPA", "Fabric Capacity Metrics"])
 
-    cols = [
-        "Capacity Name",
-        "Capacity Id",
-        "Workspace Name",
-        "Workspace Id",
-        "Dataset Name",
-        "Dataset Id",
-        "Configured By",
-        "Rule Name",
-        "Category",
-        "Severity",
-        "Object Type",
-        "Object Name",
-        "Description",
-        "URL",
-        "RunId",
-        "Timestamp",
-    ]
     now = datetime.datetime.now()
     output_table = "modelbparesults"
     lakehouse_workspace = fabric.resolve_workspace_name()
@@ -92,7 +72,7 @@ def run_model_bpa_bulk(
     if len(lakeT_filt) == 0:
         runId = 1
     else:
-        max_run_id = get_max_run_id(lakehouse=lakehouse, table_name=output_table)
+        max_run_id = _get_max_run_id(lakehouse=lakehouse, table_name=output_table)
         runId = max_run_id + 1
 
     if isinstance(workspace, str):
@@ -108,7 +88,7 @@ def run_model_bpa_bulk(
         wksp = r["Name"]
         wksp_id = r["Id"]
         capacity_id, capacity_name = resolve_workspace_capacity(workspace=wksp)
-        df = pd.DataFrame(columns=cols)
+        df = pd.DataFrame(columns=list(icons.bpa_schema.keys()))
         dfD = fabric.list_datasets(workspace=wksp, mode="rest")
 
         # Exclude default semantic models
@@ -139,8 +119,8 @@ def run_model_bpa_bulk(
                             rules=rules,
                             extended=extended,
                         )
-                        bpa_df["Capacity Id"] = capacity_id
                         bpa_df["Capacity Name"] = capacity_name
+                        bpa_df["Capacity Id"] = capacity_id
                         bpa_df["Workspace Name"] = wksp
                         bpa_df["Workspace Id"] = wksp_id
                         bpa_df["Dataset Name"] = dataset_name
@@ -148,7 +128,7 @@ def run_model_bpa_bulk(
                         bpa_df["Configured By"] = config_by
                         bpa_df["Timestamp"] = now
                         bpa_df["RunId"] = runId
-                        bpa_df = bpa_df[cols]
+                        bpa_df = bpa_df[list(icons.bpa_schema.keys())]
 
                         bpa_df["RunId"] = bpa_df["RunId"].astype("int")
 
@@ -162,16 +142,23 @@ def run_model_bpa_bulk(
                         )
                         print(e)
 
-                df["Severity"].replace(icons.severity_mapping, inplace=True)
+                df["Severity"].replace(icons.severity_mapping)
 
                 # Append save results individually for each workspace (so as not to create a giant dataframe)
                 print(
                     f"{icons.in_progress} Saving the Model BPA results of the '{wksp}' workspace to the '{output_table}' within the '{lakehouse}' lakehouse within the '{lakehouse_workspace}' workspace..."
                 )
+
+                schema = {
+                    key.replace(" ", "_"): value
+                    for key, value in icons.bpa_schema.items()
+                }
+
                 save_as_delta_table(
                     dataframe=df,
                     delta_table_name=output_table,
                     write_mode="append",
+                    schema=schema,
                     merge_schema=True,
                 )
                 print(
@@ -205,9 +192,6 @@ def create_model_bpa_semantic_model(
         The workspace in which the lakehouse resides.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
-
-    Returns
-    -------
     """
 
     from sempy_labs._helper_functions import resolve_lakehouse_name
@@ -246,7 +230,7 @@ def create_model_bpa_semantic_model(
             tom.model
 
     dyn_connect()
-
+    icons.sll_tags.append("ModelBPABulk")
     table_exists = False
     with connect_semantic_model(
         dataset=dataset, readonly=False, workspace=lakehouse_workspace

sempy_labs/_model_bpa_rules.py

@@ -158,6 +158,7 @@ def model_bpa_rules(
                     and r.ToTable.Name == obj.Name
                     for r in tom.used_in_relationships(object=obj)
                 ),
+                "When using DirectQuery, dimension tables should be set to Dual mode in order to improve query performance.",
                 "https://learn.microsoft.com/power-bi/transform-model/desktop-storage-mode#propagation-of-the-dual-setting",
             ),
             (
@@ -492,6 +493,7 @@ def model_bpa_rules(
                     obj.Expression,
                     flags=re.IGNORECASE,
                 ),
+                "Adding a constant value may lead to performance degradation.",
             ),
             (
                 "DAX Expressions",
@@ -643,14 +645,15 @@ def model_bpa_rules(
                 "Calculation groups with no calculation items",
                 lambda obj, tom: obj.CalculationGroup is not None
                 and not any(obj.CalculationGroup.CalculationItems),
+                "Calculation groups have no function unless they have calculation items.",
             ),
             (
                 "Maintenance",
-                "Column",
+                ["Column", "Measure", "Table"],
                 "Info",
                 "Visible objects with no description",
                 lambda obj, tom: obj.IsHidden is False and len(obj.Description) == 0,
-                "Calculation groups have no function unless they have calculation items.",
+                "Add descriptions to objects. These descriptions are shown on hover within the Field List in Power BI Desktop. Additionally, you can leverage these descriptions to create an automated data dictionary.",
             ),
             (
                 "Formatting",
@@ -710,6 +713,7 @@ def model_bpa_rules(
                 "Percentages should be formatted with thousands separators and 1 decimal",
                 lambda obj, tom: "%" in obj.FormatString
                 and obj.FormatString != "#,0.0%;-#,0.0%;#,0.0%",
+                "For a better user experience, percengage measures should be formatted with a '%' sign.",
             ),
             (
                 "Formatting",
@@ -719,6 +723,7 @@ def model_bpa_rules(
                 lambda obj, tom: "$" not in obj.FormatString
                 and "%" not in obj.FormatString
                 and obj.FormatString not in ["#,0", "#,0.0"],
+                "For a better user experience, whole numbers should be formatted with commas.",
             ),
             (
                 "Formatting",
@@ -731,7 +736,7 @@ def model_bpa_rules(
                     and r.FromCardinality == TOM.RelationshipEndCardinality.Many
                     for r in tom.used_in_relationships(object=obj)
                 ),
-                "Foreign keys should always be hidden.",
+                "Foreign keys should always be hidden as they should not be used by end users.",
             ),
             (
                 "Formatting",

sempy_labs/_notebooks.py

@@ -8,7 +8,6 @@ from sempy_labs._helper_functions import (
     resolve_workspace_name_and_id,
     lro,
     _decode_b64,
-    resolve_notebook_id,
 )
 from sempy.fabric.exceptions import FabricHTTPException
 
@@ -19,6 +18,8 @@ def get_notebook_definition(
     """
     Obtains the notebook definition.
 
+    This is a wrapper function for the following API: `Items - Get Notebook Definition <https://learn.microsoft.com/rest/api/fabric/notebook/items/get-notebook-definition`_.
+
     Parameters
     ----------
     notebook_name : str
@@ -38,10 +39,12 @@ def get_notebook_definition(
     """
 
     (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
-    notebook_id = resolve_notebook_id(notebook=notebook_name, workspace=workspace)
+    item_id = fabric.resolve_item_id(
+        item_name=notebook_name, type="Notebook", workspace=workspace
+    )
     client = fabric.FabricRestClient()
     response = client.post(
-        f"v1/workspaces/{workspace_id}/notebooks/{notebook_id}/getDefinition",
+        f"v1/workspaces/{workspace_id}/notebooks/{item_id}/getDefinition",
     )
 
     result = lro(client, response).json()
@@ -62,6 +65,7 @@ def import_notebook_from_web(
     url: str,
     description: Optional[str] = None,
     workspace: Optional[str] = None,
+    overwrite: bool = False,
 ):
     """
     Creates a new notebook within a workspace based on a Jupyter notebook hosted in the web.
@@ -82,16 +86,12 @@ def import_notebook_from_web(
         The name of the workspace.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
+    overwrite : bool, default=False
+        If set to True, overwrites the existing notebook in the workspace if it exists.
     """
 
-    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
-    client = fabric.FabricRestClient()
-    dfI = fabric.list_items(workspace=workspace, type="Notebook")
-    dfI_filt = dfI[dfI["Display Name"] == notebook_name]
-    if len(dfI_filt) > 0:
-        raise ValueError(
-            f"{icons.red_dot} The '{notebook_name}' already exists within the '{workspace}' workspace."
-        )
+    if workspace is None:
+        workspace = fabric.resolve_workspace_name(workspace)
 
     # Fix links to go to the raw github file
     starting_text = "https://github.com/"
@@ -104,11 +104,56 @@ def import_notebook_from_web(
     response = requests.get(url)
     if response.status_code != 200:
         raise FabricHTTPException(response)
-    file_content = response.content
-    notebook_payload = base64.b64encode(file_content)
+
+    dfI = fabric.list_items(workspace=workspace, type="Notebook")
+    dfI_filt = dfI[dfI["Display Name"] == notebook_name]
+    if len(dfI_filt) == 0:
+        create_notebook(
+            name=notebook_name,
+            notebook_content=response.content,
+            workspace=workspace,
+            description=description,
+        )
+    elif len(dfI_filt) > 0 and overwrite:
+        update_notebook_definition(
+            name=notebook_name, notebook_content=response.content, workspace=workspace
+        )
+    else:
+        raise ValueError(
+            f"{icons.red_dot} The '{notebook_name}' already exists within the '{workspace}' workspace and 'overwrite' is set to False."
+        )
+
+
+def create_notebook(
+    name: str,
+    notebook_content: str,
+    description: Optional[str] = None,
+    workspace: Optional[str] = None,
+):
+    """
+    Creates a new notebook with a definition within a workspace.
+
+    Parameters
+    ----------
+    name : str
+        The name of the notebook to be created.
+    notebook_content : str
+        The Jupyter notebook content (not in Base64 format).
+    description : str, default=None
+        The description of the notebook.
+        Defaults to None which does not place a description.
+    workspace : str, default=None
+        The name of the workspace.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
+    client = fabric.FabricRestClient()
+    notebook_payload = base64.b64encode(notebook_content)
 
     request_body = {
-        "displayName": notebook_name,
+        "displayName": name,
         "definition": {
             "format": "ipynb",
             "parts": [
@@ -128,5 +173,56 @@ def import_notebook_from_web(
     lro(client, response, status_codes=[201, 202])
 
     print(
-        f"{icons.green_dot} The '{notebook_name}' notebook was created within the '{workspace}' workspace."
+        f"{icons.green_dot} The '{name}' notebook was created within the '{workspace}' workspace."
+    )
+
+
+def update_notebook_definition(
+    name: str, notebook_content: str, workspace: Optional[str] = None
+):
+    """
+    Updates an existing notebook with a new definition.
+
+    Parameters
+    ----------
+    name : str
+        The name of the notebook to be created.
+    notebook_content : str
+        The Jupyter notebook content (not in Base64 format).
+    workspace : str, default=None
+        The name of the workspace.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
+    client = fabric.FabricRestClient()
+    notebook_payload = base64.b64encode(notebook_content)
+    notebook_id = fabric.resolve_item_id(
+        item_name=name, type="Notebook", workspace=workspace
+    )
+
+    request_body = {
+        "displayName": name,
+        "definition": {
+            "format": "ipynb",
+            "parts": [
+                {
+                    "path": "notebook-content.py",
+                    "payload": notebook_payload,
+                    "payloadType": "InlineBase64",
+                }
+            ],
+        },
+    }
+
+    response = client.post(
+        f"v1/workspaces/{workspace_id}/notebooks/{notebook_id}/updateDefinition",
+        json=request_body,
+    )
+
+    lro(client, response, return_status_code=True)
+
+    print(
+        f"{icons.green_dot} The '{name}' notebook was updated within the '{workspace}' workspace."
     )

sempy_labs/_query_scale_out.py

@@ -13,6 +13,8 @@ def qso_sync(dataset: str, workspace: Optional[str] = None):
     """
     Triggers a query scale-out sync of read-only replicas for the specified dataset from the specified workspace.
 
+    This is a wrapper function for the following API: `Datasets - Trigger Query Scale Out Sync In Group <https://learn.microsoft.com/rest/api/power-bi/datasets/trigger-query-scale-out-sync-in-group`_.
+
     Parameters
     ----------
     dataset : str
@@ -23,8 +25,6 @@ def qso_sync(dataset: str, workspace: Optional[str] = None):
         or if no lakehouse attached, resolves to the workspace of the notebook.
     """
 
-    # https://learn.microsoft.com/en-us/rest/api/power-bi/datasets/trigger-query-scale-out-sync-in-group
-
     (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
     dataset_id = resolve_dataset_id(dataset, workspace)
 
@@ -46,6 +46,8 @@ def qso_sync_status(
     """
     Returns the query scale-out sync status for the specified dataset from the specified workspace.
 
+    This is a wrapper function for the following API: `Datasets - Get Query Scale Out Sync Status In Group <https://learn.microsoft.com/rest/api/power-bi/datasets/get-query-scale-out-sync-status-in-group`_.
+
     Parameters
     ----------
     dataset : str
@@ -61,8 +63,6 @@ def qso_sync_status(
         2 pandas dataframes showing the query scale-out sync status.
     """
 
-    # https://learn.microsoft.com/en-us/rest/api/power-bi/datasets/get-query-scale-out-sync-status-in-group
-
     df = pd.DataFrame(
         columns=[
             "Scale Out Status",
@@ -143,6 +143,8 @@ def disable_qso(dataset: str, workspace: Optional[str] = None) -> pd.DataFrame:
     """
     Sets the max read-only replicas to 0, disabling query scale out.
 
+    This is a wrapper function for the following API: `Datasets - Update Dataset In Group <https://learn.microsoft.com/rest/api/power-bi/datasets/update-dataset-in-group`_.
+
     Parameters
     ----------
     dataset : str
@@ -188,6 +190,8 @@ def set_qso(
     """
     Sets the query scale out settings for a semantic model.
 
+    This is a wrapper function for the following API: `Datasets - Update Dataset In Group <https://learn.microsoft.com/rest/api/power-bi/datasets/update-dataset-in-group`_.
+
     Parameters
     ----------
     dataset : str
@@ -209,8 +213,6 @@ def set_qso(
 
     from sempy_labs._helper_functions import is_default_semantic_model
 
-    # https://learn.microsoft.com/en-us/rest/api/power-bi/datasets/update-dataset-in-group
-
     (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
     dataset_id = resolve_dataset_id(dataset, workspace)