semantic-link-labs 0.9.2__py3-none-any.whl → 0.9.4__py3-none-any.whl

sempy_labs/_job_scheduler.py

@@ -1,6 +1,6 @@
 from sempy._utils._log import log
 import pandas as pd
-from typing import Optional
+from typing import Optional, List
 from sempy_labs._helper_functions import (
     resolve_workspace_name_and_id,
     resolve_item_name_and_id,
@@ -189,7 +189,7 @@ def run_on_demand_item_job(
     Parameters
     ----------
     item : str | uuid.UUID
-        The item name or ID
+        The item name or ID.
     type : str, default=None
         The item `type <https://learn.microsoft.com/rest/api/fabric/core/items/list-items?tabs=HTTP#itemtype>`_. If specifying the item name as the item, the item type is required.
     job_type : str, default="DefaultJob"
@@ -213,3 +213,227 @@ def run_on_demand_item_job(
     )
 
     print(f"{icons.green_dot} The '{item_name}' {type.lower()} has been executed.")
+
+
+def create_item_schedule_cron(
+    item: str | UUID,
+    type: str,
+    start_date_time: str,
+    end_date_time: str,
+    local_time_zone: str,
+    job_type: str = "DefaultJob",
+    interval_minutes: int = 10,
+    enabled: bool = True,
+    workspace: Optional[str | UUID] = None,
+):
+    """
+    Create a new schedule for an item based on a `chronological time <https://learn.microsoft.com/rest/api/fabric/core/job-scheduler/create-item-schedule?tabs=HTTP#cronscheduleconfig>`_.
+
+    This is a wrapper function for the following API: `Job Scheduler - Create Item Schedule <https://learn.microsoft.com/rest/api/fabric/core/job-scheduler/create-item-schedule>`_.
+
+    Parameters
+    ----------
+    item : str | uuid.UUID
+        The item name or ID.
+    type : str
+        The item `type <https://learn.microsoft.com/rest/api/fabric/core/items/list-items?tabs=HTTP#itemtype>`_. If specifying the item name as the item, the item type is required.
+    start_date_time: str
+        The start date and time of the schedule. Example: "2024-04-28T00:00:00".
+    end_date_time: str
+        The end date and time of the schedule. Must be later than the start_date_time. Example: "2024-04-30T23:59:00".
+    local_time_zone: str
+        The `time zone <https://learn.microsoft.com/windows-hardware/manufacture/desktop/default-time-zones?view=windows-11>`_ of the schedule. Example: "Central Standard Time".
+    job_type : str, default="DefaultJob"
+        The job type.
+    interval_minutes: int, default=10
+        The schedule interval (in minutes).
+    enabled: bool, default=True
+        Whether the schedule is enabled.
+    workspace : str | uuid.UUID, default=None
+        The workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    (item_name, item_id) = resolve_item_name_and_id(
+        item=item, type=type, workspace=workspace
+    )
+
+    payload = {
+        "enabled": enabled,
+        "configuration": {
+            "startDateTime": start_date_time,
+            "endDateTime": end_date_time,
+            "localTimeZoneId": local_time_zone,
+            "type": "Cron",
+            "interval": interval_minutes,
+        },
+    }
+
+    _base_api(
+        request=f"v1/workspaces/{workspace_id}/items/{item_id}/jobs/{job_type}/schedules",
+        method="post",
+        payload=payload,
+        status_codes=201,
+    )
+
+    print(
+        f"{icons.green_dot} The schedule for the '{item_name}' {type.lower()} has been created."
+    )
+
+
+def create_item_schedule_daily(
+    item: str | UUID,
+    type: str,
+    start_date_time: str,
+    end_date_time: str,
+    local_time_zone: str,
+    times: List[str],
+    job_type: str = "DefaultJob",
+    enabled: bool = True,
+    workspace: Optional[str | UUID] = None,
+):
+    """
+    Create a new daily schedule for an item.
+
+    This is a wrapper function for the following API: `Job Scheduler - Create Item Schedule <https://learn.microsoft.com/rest/api/fabric/core/job-scheduler/create-item-schedule>`_.
+
+    Parameters
+    ----------
+    item : str | uuid.UUID
+        The item name or ID.
+    type : str
+        The item `type <https://learn.microsoft.com/rest/api/fabric/core/items/list-items?tabs=HTTP#itemtype>`_. If specifying the item name as the item, the item type is required.
+    start_date_time: str
+        The start date and time of the schedule. Example: "2024-04-28T00:00:00".
+    end_date_time: str
+        The end date and time of the schedule. Must be later than the start_date_time. Example: "2024-04-30T23:59:00".
+    local_time_zone: str
+        The `time zone <https://learn.microsoft.com/windows-hardware/manufacture/desktop/default-time-zones?view=windows-11>`_ of the schedule. Example: "Central Standard Time".
+    times : List[str]
+        A list of time slots in hh:mm format, at most 100 elements are allowed. Example: ["00:00", "12:00"].
+    job_type : str, default="DefaultJob"
+        The job type.
+    enabled: bool, default=True
+        Whether the schedule is enabled.
+    workspace : str | uuid.UUID, default=None
+        The workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    (item_name, item_id) = resolve_item_name_and_id(
+        item=item, type=type, workspace=workspace
+    )
+
+    payload = {
+        "enabled": enabled,
+        "configuration": {
+            "startDateTime": start_date_time,
+            "endDateTime": end_date_time,
+            "localTimeZoneId": local_time_zone,
+            "type": "Daily",
+            "times": times,
+        },
+    }
+
+    _base_api(
+        request=f"v1/workspaces/{workspace_id}/items/{item_id}/jobs/{job_type}/schedules",
+        method="post",
+        payload=payload,
+        status_codes=201,
+    )
+
+    print(
+        f"{icons.green_dot} The schedule for the '{item_name}' {type.lower()} has been created."
+    )
+
+
+def create_item_schedule_weekly(
+    item: str | UUID,
+    type: str,
+    start_date_time: str,
+    end_date_time: str,
+    local_time_zone: str,
+    times: List[str],
+    weekdays: List[str],
+    job_type: str = "DefaultJob",
+    enabled: bool = True,
+    workspace: Optional[str | UUID] = None,
+):
+    """
+    Create a new daily schedule for an item.
+
+    This is a wrapper function for the following API: `Job Scheduler - Create Item Schedule <https://learn.microsoft.com/rest/api/fabric/core/job-scheduler/create-item-schedule>`_.
+
+    Parameters
+    ----------
+    item : str | uuid.UUID
+        The item name or ID.
+    type : str
+        The item `type <https://learn.microsoft.com/rest/api/fabric/core/items/list-items?tabs=HTTP#itemtype>`_. If specifying the item name as the item, the item type is required.
+    start_date_time: str
+        The start date and time of the schedule. Example: "2024-04-28T00:00:00".
+    end_date_time: str
+        The end date and time of the schedule. Must be later than the start_date_time. Example: "2024-04-30T23:59:00".
+    local_time_zone: str
+        The `time zone <https://learn.microsoft.com/windows-hardware/manufacture/desktop/default-time-zones?view=windows-11>`_ of the schedule. Example: "Central Standard Time".
+    times : List[str]
+        A list of time slots in hh:mm format, at most 100 elements are allowed. Example: ["00:00", "12:00"].
+    weekdays : List[str]
+        A list of weekdays. Example: ["Monday", "Tuesday"].
+    job_type : str, default="DefaultJob"
+        The job type.
+    enabled: bool, default=True
+        Whether the schedule is enabled.
+    workspace : str | uuid.UUID, default=None
+        The workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    (item_name, item_id) = resolve_item_name_and_id(
+        item=item, type=type, workspace=workspace
+    )
+
+    weekdays = [w.capitalize() for w in weekdays]
+    weekday_list = [
+        "Sunday",
+        "Monday",
+        "Tuesday",
+        "Wednesday",
+        "Thursday",
+        "Friday",
+        "Saturday",
+    ]
+    for weekday in weekdays:
+        if weekday not in weekday_list:
+            raise ValueError(
+                f"{icons.red_dot} Invalid weekday: {weekday}. Must be one of {weekday_list}."
+            )
+
+    payload = {
+        "enabled": enabled,
+        "configuration": {
+            "startDateTime": start_date_time,
+            "endDateTime": end_date_time,
+            "localTimeZoneId": local_time_zone,
+            "type": "Weekly",
+            "times": times,
+            "weekdays": weekdays,
+        },
+    }
+
+    _base_api(
+        request=f"v1/workspaces/{workspace_id}/items/{item_id}/jobs/{job_type}/schedules",
+        method="post",
+        payload=payload,
+        status_codes=201,
+    )
+
+    print(
+        f"{icons.green_dot} The schedule for the '{item_name}' {type.lower()} has been created."
+    )

sempy_labs/_list_functions.py

@@ -9,6 +9,7 @@ from sempy_labs._helper_functions import (
     _update_dataframe_datatypes,
     _base_api,
     _create_dataframe,
+    _run_spark_sql_query,
 )
 from sempy._utils._log import log
 import pandas as pd
@@ -584,14 +585,12 @@ def list_columns(
             query = f"{query} FROM {lakehouse}.{lakeTName}"
             sql_statements.append((table_name, query))
 
-            spark = SparkSession.builder.getOrCreate()
-
         for o in sql_statements:
             tName = o[0]
             query = o[1]
 
             # Run the query
-            df = spark.sql(query)
+            df = _run_spark_sql_query(query)
 
             for column in df.columns:
                 x = df.collect()[0][column]
@@ -1240,22 +1239,46 @@ def list_shortcuts(
         uses_pagination=True,
     )
 
+    sources = [
+        "s3Compatible",
+        "googleCloudStorage",
+        "externalDataShare",
+        "amazonS3",
+        "adlsGen2",
+        "dataverse",
+    ]
+    sources_locpath = ["s3Compatible", "googleCloudStorage", "amazonS3", "adlsGen2"]
+
     for r in responses:
         for i in r.get("value", []):
             tgt = i.get("target", {})
-            s3_compat = tgt.get("s3Compatible", {})
-            gcs = tgt.get("googleCloudStorage", {})
-            eds = tgt.get("externalDataShare", {})
-            connection_id = (
-                s3_compat.get("connectionId")
-                or gcs.get("connectionId")
-                or eds.get("connectionId")
-                or None
+            one_lake = tgt.get("oneLake", {})
+            connection_id = next(
+                (
+                    tgt.get(source, {}).get("connectionId")
+                    for source in sources
+                    if tgt.get(source)
+                ),
+                None,
+            )
+            location = next(
+                (
+                    tgt.get(source, {}).get("location")
+                    for source in sources_locpath
+                    if tgt.get(source)
+                ),
+                None,
+            )
+            sub_path = next(
+                (
+                    tgt.get(source, {}).get("subpath")
+                    for source in sources_locpath
+                    if tgt.get(source)
+                ),
+                None,
             )
-            location = s3_compat.get("location") or gcs.get("location") or None
-            sub_path = s3_compat.get("subpath") or gcs.get("subpath") or None
-            source_workspace_id = tgt.get("oneLake", {}).get("workspaceId")
-            source_item_id = tgt.get("oneLake", {}).get("itemId")
+            source_workspace_id = one_lake.get("workspaceId")
+            source_item_id = one_lake.get("itemId")
             source_workspace_name = (
                 fabric.resolve_workspace_name(source_workspace_id)
                 if source_workspace_id is not None
@@ -1281,10 +1304,10 @@ def list_shortcuts(
                     if source_item_id is not None
                     else None
                 ),
-                "OneLake Path": tgt.get("oneLake", {}).get("path"),
+                "OneLake Path": one_lake.get("path"),
                 "Connection Id": connection_id,
                 "Location": location,
-                "Bucket": s3_compat.get("bucket"),
+                "Bucket": tgt.get("s3Compatible", {}).get("bucket"),
                 "SubPath": sub_path,
             }
             df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
@@ -1367,7 +1390,7 @@ def list_reports_using_semantic_model(
         & (dfR["Dataset Workspace Id"] == workspace_id)
     ][["Name", "Id"]]
     dfR_filt.rename(columns={"Name": "Report Name", "Id": "Report Id"}, inplace=True)
-    dfR_filt["Report Worskpace Name"] = workspace_name
+    dfR_filt["Report Workspace Name"] = workspace_name
     dfR_filt["Report Workspace Id"] = workspace_id
 
     return dfR_filt
@@ -1775,7 +1798,7 @@ def list_synonyms(dataset: str | UUID, workspace: Optional[str] = None):
         "State": "string",
         "Source": "string",
         "Weight": "float_fillna",
-        "Last Modified": "datetime",
+        "Last Modified": "string",
     }
 
     df = _create_dataframe(columns=columns)

sempy_labs/_ml_experiments.py

@@ -91,7 +91,7 @@ def create_ml_experiment(
     _base_api(
         request=f"/v1/workspaces/{workspace_id}/mlExperiments",
         method="post",
-        json=payload,
+        payload=payload,
         status_codes=[201, 202],
         lro_return_status_code=True,
     )

sempy_labs/_model_bpa.py

@@ -14,6 +14,7 @@ from sempy_labs._helper_functions import (
     get_language_codes,
     _get_column_aggregate,
     resolve_workspace_name_and_id,
+    _create_spark_session,
 )
 from sempy_labs.lakehouse import get_lakehouse_tables, lakehouse_attached
 from sempy_labs.tom import connect_semantic_model
@@ -181,7 +182,6 @@ def run_model_bpa(
             def translate_using_spark(rule_file):
 
                 from synapse.ml.services import Translate
-                from pyspark.sql import SparkSession
 
                 rules_temp = rule_file.copy()
                 rules_temp = rules_temp.drop(["Expression", "URL", "Severity"], axis=1)
@@ -195,7 +195,7 @@ def run_model_bpa(
                     ]
                 )
 
-                spark = SparkSession.builder.getOrCreate()
+                spark = _create_spark_session()
                 dfRules = spark.createDataFrame(rules_temp, schema)
 
                 columns = ["Category", "Rule Name", "Description"]
@@ -274,12 +274,17 @@ def run_model_bpa(
                 tom.all_columns(),
                 lambda obj: format_dax_object_name(obj.Parent.Name, obj.Name),
             ),
+            "Calculated Column": (
+                tom.all_calculated_columns(),
+                lambda obj: format_dax_object_name(obj.Parent.Name, obj.Name),
+            ),
             "Measure": (tom.all_measures(), lambda obj: obj.Name),
             "Hierarchy": (
                 tom.all_hierarchies(),
                 lambda obj: format_dax_object_name(obj.Parent.Name, obj.Name),
             ),
             "Table": (tom.model.Tables, lambda obj: obj.Name),
+            "Calculated Table": (tom.all_calculated_tables(), lambda obj: obj.Name),
             "Role": (tom.model.Roles, lambda obj: obj.Name),
             "Model": (tom.model, lambda obj: obj.Model.Name),
             "Calculation Item": (
@@ -322,6 +327,10 @@ def run_model_bpa(
                     x = [nm(obj) for obj in tom.all_hierarchies() if expr(obj, tom)]
                 elif scope == "Table":
                     x = [nm(obj) for obj in tom.model.Tables if expr(obj, tom)]
+                elif scope == "Calculated Table":
+                    x = [
+                        nm(obj) for obj in tom.all_calculated_tables() if expr(obj, tom)
+                    ]
                 elif scope == "Relationship":
                     x = [nm(obj) for obj in tom.model.Relationships if expr(obj, tom)]
                 elif scope == "Role":
@@ -332,6 +341,12 @@ def run_model_bpa(
                     x = [
                         nm(obj) for obj in tom.all_calculation_items() if expr(obj, tom)
                     ]
+                elif scope == "Calculated Column":
+                    x = [
+                        nm(obj)
+                        for obj in tom.all_calculated_columns()
+                        if expr(obj, tom)
+                    ]
 
                 if len(x) > 0:
                     new_data = {

sempy_labs/_model_bpa_rules.py

@@ -416,7 +416,7 @@ def model_bpa_rules(
                 lambda obj, tom: any(
                     re.search(
                         r"USERELATIONSHIP\s*\(\s*.+?(?=])\]\s*,\s*'*"
-                        + obj.Name
+                        + re.escape(obj.Name)
                         + r"'*\[",
                         m.Expression,
                         flags=re.IGNORECASE,
@@ -455,7 +455,9 @@ def model_bpa_rules(
                 "Warning",
                 "The EVALUATEANDLOG function should not be used in production models",
                 lambda obj, tom: re.search(
-                    r"evaluateandlog\s*\(", obj.Expression, flags=re.IGNORECASE
+                    r"evaluateandlog\s*\(",
+                    obj.Expression,
+                    flags=re.IGNORECASE,
                 ),
                 "The EVALUATEANDLOG function is meant to be used only in development/test environments and should not be used in production models.",
                 "https://pbidax.wordpress.com/2022/08/16/introduce-the-dax-evaluateandlog-function",
@@ -563,7 +565,12 @@ def model_bpa_rules(
             ),
             (
                 "DAX Expressions",
-                "Measure",
+                [
+                    "Measure",
+                    "Calculated Table",
+                    "Calculated Column",
+                    "Calculation Item",
+                ],
                 "Error",
                 "Column references should be fully qualified",
                 lambda obj, tom: any(
@@ -574,7 +581,12 @@ def model_bpa_rules(
             ),
             (
                 "DAX Expressions",
-                "Measure",
+                [
+                    "Measure",
+                    "Calculated Table",
+                    "Calculated Column",
+                    "Calculation Item",
+                ],
                 "Error",
                 "Measure references should be unqualified",
                 lambda obj, tom: any(
@@ -592,13 +604,13 @@ def model_bpa_rules(
                 and not any(
                     re.search(
                         r"USERELATIONSHIP\s*\(\s*\'*"
-                        + obj.FromTable.Name
+                        + re.escape(obj.FromTable.Name)
                         + r"'*\["
-                        + obj.FromColumn.Name
+                        + re.escape(obj.FromColumn.Name)
                         + r"\]\s*,\s*'*"
-                        + obj.ToTable.Name
+                        + re.escape(obj.ToTable.Name)
                         + r"'*\["
-                        + obj.ToColumn.Name
+                        + re.escape(obj.ToColumn.Name)
                         + r"\]",
                         m.Expression,
                         flags=re.IGNORECASE,

sempy_labs/_semantic_models.py

@@ -0,0 +1,117 @@
+from uuid import UUID
+from typing import Optional
+import pandas as pd
+from sempy_labs._helper_functions import (
+    _create_dataframe,
+    _base_api,
+    _update_dataframe_datatypes,
+    resolve_workspace_name_and_id,
+    resolve_dataset_name_and_id,
+)
+import sempy_labs._icons as icons
+
+
+def get_semantic_model_refresh_schedule(
+    dataset: str | UUID, workspace: Optional[str | UUID] = None
+) -> pd.DataFrame:
+    """
+    Gets the refresh schedule for the specified dataset from the specified workspace.
+
+    Parameters
+    ----------
+    dataset : str | uuid.UUID
+        Name or ID of the semantic model.
+    workspace : str | uuid.UUID, default=None
+        The workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    pandas.DataFrame
+        Shows the refresh schedule for the specified dataset from the specified workspace.
+    """
+
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    (dataset_name, dataset_id) = resolve_dataset_name_and_id(dataset, workspace)
+
+    columns = {
+        "Days": "str",
+        "Times": "str",
+        "Enabled": "bool",
+        "Local Time Zone Id": "str",
+        "Notify Option": "str",
+    }
+
+    column_map = {
+        "days": "Days",
+        "times": "Times",
+        "enabled": "Enabled",
+        "localTimeZoneId": "Local Time Zone Id",
+        "notifyOption": "Notify Option",
+    }
+
+    df = _create_dataframe(columns)
+
+    result = _base_api(
+        request=f"/v1.0/myorg/groups/{workspace_id}/datasets/{dataset_id}/refreshSchedule"
+    ).json()
+
+    df = (
+        pd.json_normalize(result)
+        .drop(columns=["@odata.context"], errors="ignore")
+        .rename(columns=column_map)
+    )
+
+    _update_dataframe_datatypes(dataframe=df, column_map=columns)
+
+    return df
+
+
+def enable_semantic_model_scheduled_refresh(
+    dataset: str | UUID,
+    workspace: Optional[str | UUID] = None,
+    enable: bool = True,
+):
+    """
+    Enables the scheduled refresh for the specified dataset from the specified workspace.
+
+    Parameters
+    ----------
+    dataset : str | uuid.UUID
+        Name or ID of the semantic model.
+    workspace : str | uuid.UUID, default=None
+        The workspace name or ID.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    enable : bool, default=True
+        If True, enables the scheduled refresh.
+        If False, disables the scheduled refresh.
+    """
+
+    (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
+    (dataset_name, dataset_id) = resolve_dataset_name_and_id(dataset, workspace)
+
+    df = get_semantic_model_refresh_schedule(dataset=dataset, workspace=workspace)
+    status = df["Enabled"].iloc[0]
+
+    if enable and status:
+        print(
+            f"{icons.info} Scheduled refresh for the '{dataset_name}' within the '{workspace_name}' workspace is already enabled."
+        )
+    elif not enable and not status:
+        print(
+            f"{icons.info} Scheduled refresh for the '{dataset_name}' within the '{workspace_name}' workspace is already disabled."
+        )
+    else:
+        payload = {"value": {"enabled": enable}}
+
+        _base_api(
+            request=f"/v1.0/myorg/groups/{workspace_id}/datasets/{dataset_id}/refreshSchedule",
+            method="patch",
+            payload=payload,
+        )
+
+        print(
+            f"{icons.green_dot} Scheduled refresh for the '{dataset_name}' within the '{workspace_name}' workspace has been enabled."
+        )