semantic-link-labs 0.8.3__py3-none-any.whl → 0.8.5__py3-none-any.whl

sempy_labs/_eventstreams.py

@@ -14,6 +14,8 @@ def list_eventstreams(workspace: Optional[str] = None) -> pd.DataFrame:
     """
     Shows the eventstreams within a workspace.
 
+    This is a wrapper function for the following API: `Items - List Eventstreams <https://learn.microsoft.com/rest/api/fabric/environment/items/list-eventstreams>`_.
+
     Parameters
     ----------
     workspace : str, default=None
@@ -56,6 +58,8 @@ def create_eventstream(
     """
     Creates a Fabric eventstream.
 
+    This is a wrapper function for the following API: `Items - Create Eventstream <https://learn.microsoft.com/rest/api/fabric/environment/items/create-eventstream>`_.
+
     Parameters
     ----------
     name: str
@@ -91,6 +95,8 @@ def delete_eventstream(name: str, workspace: Optional[str] = None):
     """
     Deletes a Fabric eventstream.
 
+    This is a wrapper function for the following API: `Items - Delete Eventstream <https://learn.microsoft.com/rest/api/fabric/environment/items/delete-eventstream>`_.
+
     Parameters
     ----------
     name: str

sempy_labs/_external_data_shares.py

@@ -20,6 +20,8 @@ def create_external_data_share(
     """
     Creates an external data share for a given path or list of paths in the specified item.
 
+    This is a wrapper function for the following API: `External Data Shares - Create External Data Share <https://learn.microsoft.com/rest/api/fabric/core/external-data-shares/create-external-data-share>`_.
+
     Parameters
     ----------
     item_name : str
@@ -71,6 +73,8 @@ def revoke_external_data_share(
     """
     Revokes the specified external data share. Note: This action cannot be undone.
 
+    This is a wrapper function for the following API: `External Data Shares - Revoke External Data Share <https://learn.microsoft.com/rest/api/fabric/core/external-data-shares/revoke-external-data-share`_.
+
     Parameters
     ----------
     external_data_share_id : UUID
@@ -85,8 +89,6 @@ def revoke_external_data_share(
         or if no lakehouse attached, resolves to the workspace of the notebook.
     """
 
-    # https://learn.microsoft.com/en-us/rest/api/fabric/core/external-data-shares/revoke-external-data-share?tabs=HTTP
-
     (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
     item_id = fabric.resolve_item_id(
         item_name=item_name, type=item_type, workspace=workspace
@@ -111,6 +113,8 @@ def list_external_data_shares_in_item(
     """
     Returns a list of the external data shares that exist for the specified item.
 
+    This is a wrapper function for the following API: `External Data Shares - List External Data Shares In Item <https://learn.microsoft.com/rest/api/fabric/core/external-data-shares/list-external-data-shares-in-item`_.
+
     Parameters
     ----------
     item_name : str
@@ -128,8 +132,6 @@ def list_external_data_shares_in_item(
         A pandas dataframe showing a list of the external data shares that exist for the specified item.
     """
 
-    # https://learn.microsoft.com/en-us/rest/api/fabric/core/external-data-shares/list-external-data-shares-in-item?tabs=HTTP
-
     (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
     item_id = fabric.resolve_item_id(
         item_name=item_name, type=item_type, workspace=workspace

sempy_labs/_generate_semantic_model.py

@@ -64,11 +64,28 @@ def create_blank_semantic_model(
             "name": '{dataset}',
             "compatibilityLevel": {compatibility_level},
             "model": {{
-            "culture": "en-US",
-            "defaultPowerBIDataSourceVersion": "powerBI_V3"
+                "cultures": [
+                    {{
+                        "name": "en-US",
+                        "linguisticMetadata": {{
+                            "content": {{
+                                "Version": "1.0.0",
+                                "Language": "en-US"
+                            }},
+                            "contentType": "json"
+                        }}
+                    }}
+                ],
+                "collation": "Latin1_General_100_BIN2_UTF8",
+                "dataAccessOptions": {{
+                    "legacyRedirects": true,
+                    "returnErrorValuesAsNull": true,
+                }},
+                "defaultPowerBIDataSourceVersion": "powerBI_V3",
+                "sourceQueryCulture": "en-US",
+                }}
             }}
         }}
-        }}
     }}
     """
 
@@ -85,6 +102,8 @@ def create_semantic_model_from_bim(
     """
     Creates a new semantic model based on a Model.bim file.
 
+    This is a wrapper function for the following API: `Items - Create Semantic Model <https://learn.microsoft.com/rest/api/fabric/semanticmodel/items/create-semantic-model>`_.
+
     Parameters
     ----------
     dataset : str
@@ -149,6 +168,8 @@ def update_semantic_model_from_bim(
     """
     Updates a semantic model definition based on a Model.bim file.
 
+    This is a wrapper function for the following API: `Items - Update Semantic Model Definition <https://learn.microsoft.com/rest/api/fabric/semanticmodel/items/update-semantic-model-definition>`_.
+
     Parameters
     ----------
     dataset : str
@@ -287,6 +308,8 @@ def get_semantic_model_bim(
     """
     Extracts the Model.bim file for a given semantic model.
 
+    This is a wrapper function for the following API: `Items - Get Semantic Model Definition <https://learn.microsoft.com/rest/api/fabric/semanticmodel/items/get-semantic-model-definition>`_.
+
     Parameters
     ----------
     dataset : str

sempy_labs/_git.py

@@ -21,6 +21,8 @@ def connect_workspace_to_git(
     """
     Connects a workspace to a git repository.
 
+    This is a wrapper function for the following API: `Git - Connect <https://learn.microsoft.com/rest/api/fabric/core/git/connect>`_.
+
     Parameters
     ----------
     organization_name : str
@@ -41,8 +43,6 @@ def connect_workspace_to_git(
         or if no lakehouse attached, resolves to the workspace of the notebook.
     """
 
-    # https://learn.microsoft.com/en-us/rest/api/fabric/core/git/connect?tabs=HTTP
-
     workspace, workspace_id = resolve_workspace_name_and_id(workspace)
 
     request_body = {
@@ -72,6 +72,8 @@ def disconnect_workspace_from_git(workspace: Optional[str] = None):
     """
     Disconnects a workpsace from a git repository.
 
+    This is a wrapper function for the following API: `Git - Disconnect <https://learn.microsoft.com/rest/api/fabric/core/git/disconnect>`_.
+
     Parameters
     ----------
     workspace : str, default=None
@@ -80,8 +82,6 @@ def disconnect_workspace_from_git(workspace: Optional[str] = None):
         or if no lakehouse attached, resolves to the workspace of the notebook.
     """
 
-    # https://learn.microsoft.com/en-us/rest/api/fabric/core/git/disconnect?tabs=HTTP
-
     workspace, workspace_id = resolve_workspace_name_and_id(workspace)
 
     client = fabric.FabricRestClient()
@@ -98,6 +98,8 @@ def get_git_status(workspace: Optional[str] = None) -> pd.DataFrame:
     """
     Obtains the Git status of items in the workspace, that can be committed to Git.
 
+    This is a wrapper function for the following API: `Git - Get Status <https://learn.microsoft.com/rest/api/fabric/core/git/get-status>.
+
     Parameters
     ----------
     workspace : str, default=None
@@ -111,8 +113,6 @@ def get_git_status(workspace: Optional[str] = None) -> pd.DataFrame:
         A pandas dataframe showing the Git status of items in the workspace.
     """
 
-    # https://learn.microsoft.com/en-us/rest/api/fabric/core/git/get-status?tabs=HTTP
-
     workspace, workspace_id = resolve_workspace_name_and_id(workspace)
 
     df = pd.DataFrame(
@@ -161,6 +161,8 @@ def get_git_connection(workspace: Optional[str] = None) -> pd.DataFrame:
     """
     Obtains the Git status of items in the workspace, that can be committed to Git.
 
+    This is a wrapper function for the following API: `Git - Get Connection <https://learn.microsoft.com/rest/api/fabric/core/git/get-connection>`_.
+
     Parameters
     ----------
     workspace : str, default=None
@@ -174,8 +176,6 @@ def get_git_connection(workspace: Optional[str] = None) -> pd.DataFrame:
         A pandas dataframe showing the Git status of items in the workspace.
     """
 
-    # https://learn.microsoft.com/en-us/rest/api/fabric/core/git/get-status?tabs=HTTP
-
     workspace, workspace_id = resolve_workspace_name_and_id(workspace)
 
     df = pd.DataFrame(
@@ -221,6 +221,8 @@ def initialize_git_connection(workspace: Optional[str] = None):
     """
     Initializes a connection for a workspace that is connected to Git.
 
+    This is a wrapper function for the following API: `Git - Initialize Connection <https://learn.microsoft.com/rest/api/fabric/core/git/initialize-connection>`_.
+
     Parameters
     ----------
     workspace : str, default=None
@@ -229,8 +231,6 @@ def initialize_git_connection(workspace: Optional[str] = None):
         or if no lakehouse attached, resolves to the workspace of the notebook.
     """
 
-    # https://learn.microsoft.com/en-us/rest/api/fabric/core/git/initialize-connection?tabs=HTTP
-
     workspace, workspace_id = resolve_workspace_name_and_id(workspace)
 
     client = fabric.FabricRestClient()
@@ -252,6 +252,8 @@ def commit_to_git(
     """
     Commits all or a selection of items within a workspace to Git.
 
+    This is a wrapper function for the following API: `Git - Commit to Git <https://learn.microsoft.com/rest/api/fabric/core/git/commit-to-git>`_.
+
     Parameters
     ----------
     comment : str
@@ -265,8 +267,6 @@ def commit_to_git(
         or if no lakehouse attached, resolves to the workspace of the notebook.
     """
 
-    # https://learn.microsoft.com/en-us/rest/api/fabric/core/git/commit-to-git?tabs=HTTP
-
     workspace, workspace_id = resolve_workspace_name_and_id(workspace)
 
     gs = get_git_status(workspace=workspace)
@@ -320,6 +320,8 @@ def update_from_git(
     """
     Updates the workspace with commits pushed to the connected branch.
 
+    This is a wrapper function for the following API: `Git - Update From Git <https://learn.microsoft.com/rest/api/fabric/core/git/update-from-git>`_.
+
     Parameters
     ----------
     workspace_head : str
@@ -336,8 +338,6 @@ def update_from_git(
         or if no lakehouse attached, resolves to the workspace of the notebook.
     """
 
-    # https://learn.microsoft.com/en-us/rest/api/fabric/core/git/update-from-git?tabs=HTTP
-
     workspace, workspace_id = resolve_workspace_name_and_id(workspace)
 
     conflict_resolution_policies = ["PreferWorkspace", "PreferRemote"]

sempy_labs/_helper_functions.py

@@ -13,6 +13,8 @@ from uuid import UUID
 import sempy_labs._icons as icons
 import urllib.parse
 from azure.core.credentials import TokenCredential, AccessToken
+import numpy as np
+from IPython.display import display, HTML
 
 
 def create_abfss_path(
@@ -1142,6 +1144,200 @@ def _get_max_run_id(lakehouse: str, table_name: str) -> int:
     spark = SparkSession.builder.getOrCreate()
     query = f"SELECT MAX(RunId) FROM {lakehouse}.{table_name}"
     dfSpark = spark.sql(query)
-    max_run_id = dfSpark.collect()[0][0]
+    max_run_id = dfSpark.collect()[0][0] or 0
 
     return max_run_id
+
+
+def _make_list_unique(my_list):
+
+    return list(set(my_list))
+
+
+def _get_partition_map(dataset: str, workspace: Optional[str] = None) -> pd.DataFrame:
+
+    if workspace is None:
+        workspace = fabric.resolve_workspace_name()
+
+    partitions = fabric.evaluate_dax(
+        dataset=dataset,
+        workspace=workspace,
+        dax_string="""
+    select [ID] AS [PartitionID], [TableID], [Name] AS [PartitionName] from $system.tmschema_partitions
+    """,
+    )
+
+    tables = fabric.evaluate_dax(
+        dataset=dataset,
+        workspace=workspace,
+        dax_string="""
+    select [ID] AS [TableID], [Name] AS [TableName] from $system.tmschema_tables
+    """,
+    )
+
+    partition_map = pd.merge(partitions, tables, on="TableID", how="left")
+    partition_map["PartitionID"] = partition_map["PartitionID"].astype(str)
+    partition_counts = partition_map.groupby("TableID")["PartitionID"].transform(
+        "count"
+    )
+    partition_map["Object Name"] = partition_map.apply(
+        lambda row: (
+            f"'{row['TableName']}'[{row['PartitionName']}]"
+            if partition_counts[row.name] > 1
+            else row["TableName"]
+        ),
+        axis=1,
+    )
+    return partition_map
+
+
+def _show_chart(spec, title):
+
+    h = f"""
+    <!DOCTYPE html>
+    <html>
+        <head>
+            <script src="https://cdn.jsdelivr.net/npm/vega@5"></script>
+            <script src="https://cdn.jsdelivr.net/npm/vega-lite@5"></script>
+            <script src="https://cdn.jsdelivr.net/npm/vega-embed@6"></script>
+            <style>
+                table, th, td {{
+                border: 10px solid #e7e9eb;
+                border-collapse: collapse;
+                }}
+            </style>
+        </head>
+        <body>
+            <table>
+                <tr>
+                    <td style="text-align: center;">
+                        <h1>{title}</h1>
+                    </td>
+                </tr>
+                <tr>
+                    <td>
+                        <div id="vis"></div>
+                    </td>
+                </tr>
+            </table>    
+            <script type="text/javascript">
+                var spec = {spec};
+                var opt = {{"renderer": "canvas", "actions": false}};
+                vegaEmbed("#vis", spec, opt);
+            </script>
+        </body>
+    </html>"""
+
+    display(HTML(h))
+
+
+def _process_and_display_chart(df, title, widget):
+
+    # Convert time columns to milliseconds
+    df["Start"] = df["Start Time"].astype(np.int64) / int(1e6)
+    df["End"] = df["End Time"].astype(np.int64) / int(1e6)
+
+    # Calculate the time offset for proper Gantt chart rendering
+    Offset = min(df["Start"])
+    df["Start"] = df["Start"] - Offset
+    df["End"] = df["End"] - Offset
+
+    # Vega-Lite spec for Gantt chart
+    spec = (
+        """{
+        "$schema": "https://vega.github.io/schema/vega-lite/v5.json",
+        "description": "A simple bar chart with ranged data (aka Gantt Chart).",
+        "data": { "values": """
+        + df.to_json(orient="records")
+        + """ },
+        "width": 700,
+        "height": 400,
+        "mark": "bar",
+        "encoding": {
+            "y": {
+                "field": "Object Name",
+                "type": "ordinal",
+                "axis": {
+                    "labelFontSize": 15,
+                    "titleFontSize": 20,
+                    "title": "Object"
+                }
+            },
+            "x": {
+                "field": "Start", 
+                "type": "quantitative", 
+                "title": "milliseconds",
+                "axis": {
+                    "titleFontSize": 20
+                }
+            },
+            "x2": {"field": "End"},
+            "color": {
+                "field": "Event Subclass",
+                "scale": {
+                    "domain": ["Process", "ExecuteSql"],
+                    "range": ["#FFC000","#0070C0"]
+                },
+                "legend": {
+                    "labelFontSize": 20,
+                    "titleFontSize": 20,
+                    "title": "Event Type"
+                }
+            },
+            "tooltip": [
+                {"field": "Duration", "type": "quantitative", "format": ","},
+                {"field": "Cpu Time", "type": "quantitative", "format": ","},
+                {"field": "Event Subclass", "type": "nominal"}
+            ]
+        }
+    }"""
+    )
+
+    with widget:
+        widget.clear_output(wait=True)
+    _show_chart(spec, title=title)
+
+
+def _convert_data_type(input_data_type: str) -> str:
+
+    data_type_mapping = {
+        "string": "String",
+        "int": "Int64",
+        "tinyint": "Int64",
+        "smallint": "Int64",
+        "bigint": "Int64",
+        "boolean": "Boolean",
+        "timestamp": "DateTime",
+        "date": "DateTime",
+        "double": "Double",
+        "float": "Double",
+    }
+
+    if "decimal" in input_data_type:
+        return "Decimal"
+    else:
+        return data_type_mapping.get(input_data_type)
+
+
+def _is_valid_uuid(
+    guid: str,
+):
+    """
+    Validates if a string is a valid GUID in version 4
+
+    Parameters
+    ----------
+    guid : str
+        GUID to be validated.
+
+    Returns
+    -------
+    bool
+        Boolean that indicates if the string is a GUID or not.
+    """
+
+    try:
+        UUID(str(guid), version=4)
+        return True
+    except ValueError:
+        return False

sempy_labs/_icons.py

@@ -10,20 +10,6 @@ bullet = "\u2022"
 warning = "⚠️"
 error = "\u274C"
 info = "ℹ️"
-data_type_mapping = {
-    "string": "String",
-    "int": "Int64",
-    "tinyint": "Int64",
-    "smallint": "Int64",
-    "bigint": "Int64",
-    "boolean": "Boolean",
-    "timestamp": "DateTime",
-    "date": "DateTime",
-    "decimal(38,18)": "Decimal",
-    "decimal(19,4)": "Decimal",
-    "double": "Double",
-    "float": "Double",
-}
 measure_icon = "\u2211"
 table_icon = "\u229E"
 column_icon = "\u229F"
@@ -93,14 +79,15 @@ sku_mapping = {
     "P4": "F512",
     "P5": "F1024",
 }
-refreshTypes = [
-    "full",
-    "automatic",
-    "dataOnly",
-    "calculate",
-    "clearValues",
-    "defragment",
-]
+
+refresh_type_mapping = {
+    "full": "full",
+    "auto": "automatic",
+    "data": "dataOnly",
+    "calc": "calculate",
+    "clear": "clearValues",
+    "defrag": "defragment",
+}
 
 itemTypes = {
     "DataPipeline": "dataPipelines",
@@ -114,3 +101,49 @@ itemTypes = {
     "Warehouse": "warehouses",
 }
 default_schema = "dbo"
+
+data_type_string = "string"
+data_type_long = "long"
+data_type_timestamp = "timestamp"
+data_type_double = "double"
+data_type_bool = "bool"
+int_format = "int"
+pct_format = "pct"
+no_format = ""
+
+bpa_schema = {
+    "Capacity Name": data_type_string,
+    "Capacity Id": data_type_string,
+    "Workspace Name": data_type_string,
+    "Workspace Id": data_type_string,
+    "Dataset Name": data_type_string,
+    "Dataset Id": data_type_string,
+    "Configured By": data_type_string,
+    "Rule Name": data_type_string,
+    "Category": data_type_string,
+    "Severity": data_type_string,
+    "Object Type": data_type_string,
+    "Object Name": data_type_string,
+    "Description": data_type_string,
+    "URL": data_type_string,
+    "RunId": data_type_long,
+    "Timestamp": data_type_timestamp,
+}
+
+sll_ann_name = "PBI_ProTooling"
+sll_prefix = "SLL_"
+sll_tags = []
+base_cols = ["EventClass", "EventSubclass", "CurrentTime", "TextData"]
+end_cols = base_cols + [
+    "StartTime",
+    "EndTime",
+    "Duration",
+    "CpuTime",
+    "Success",
+    "IntegerData",
+    "ObjectID",
+]
+refresh_event_schema = {
+    "JobGraph": base_cols,
+    "ProgressReportEnd": end_cols,
+}

sempy_labs/_kql_databases.py

@@ -14,6 +14,8 @@ def list_kql_databases(workspace: Optional[str] = None) -> pd.DataFrame:
     """
     Shows the KQL databases within a workspace.
 
+    This is a wrapper function for the following API: `Items - List KQL Databases <https://learn.microsoft.com/rest/api/fabric/kqldatabase/items/list-kql-databases>`_.
+
     Parameters
     ----------
     workspace : str, default=None
@@ -72,6 +74,8 @@ def create_kql_database(
     """
     Creates a KQL database.
 
+    This is a wrapper function for the following API: `Items - Create KQL Database <https://learn.microsoft.com/rest/api/fabric/kqldatabase/items/create-kql-database>`_.
+
     Parameters
     ----------
     name: str
@@ -107,6 +111,8 @@ def delete_kql_database(name: str, workspace: Optional[str] = None):
     """
     Deletes a KQL database.
 
+    This is a wrapper function for the following API: `Items - Delete KQL Database <https://learn.microsoft.com/rest/api/fabric/kqldatabase/items/delete-kql-database>`_.
+
     Parameters
     ----------
     name: str

sempy_labs/_kql_querysets.py

@@ -14,6 +14,8 @@ def list_kql_querysets(workspace: Optional[str] = None) -> pd.DataFrame:
     """
     Shows the KQL querysets within a workspace.
 
+    This is a wrapper function for the following API: `Items - List KQL Querysets <https://learn.microsoft.com/rest/api/fabric/kqlqueryset/items/list-kql-querysets>`_.
+
     Parameters
     ----------
     workspace : str, default=None
@@ -62,6 +64,8 @@ def create_kql_queryset(
     """
     Creates a KQL queryset.
 
+    This is a wrapper function for the following API: `Items - Create KQL Queryset <https://learn.microsoft.com/rest/api/fabric/kqlqueryset/items/create-kql-queryset>`_.
+
     Parameters
     ----------
     name: str
@@ -97,6 +101,8 @@ def delete_kql_queryset(name: str, workspace: Optional[str] = None):
     """
     Deletes a KQL queryset.
 
+    This is a wrapper function for the following API: `Items - Delete KQL Queryset <https://learn.microsoft.com/rest/api/fabric/kqlqueryset/items/delete-kql-queryset>`_.
+
     Parameters
     ----------
     name: str

sempy_labs/_list_functions.py

@@ -1515,7 +1515,7 @@ def list_semantic_model_object_report_usage(
                 "Object Type": object_type,
             }
             df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
-            df["Referenced Object Type"] = df["Referenced Object Type"].replace(
+            df["Object Type"] = df["Object Type"].replace(
                 "Attribute Hierarchy", "Column"
             )
             if object_type in ["Measure", "Calc Column", "Calc Table", "Hierarchy"]: