semantic-link-labs 0.7.2__py3-none-any.whl → 0.7.3__py3-none-any.whl

sempy_labs/_sql.py

@@ -0,0 +1,96 @@
+import sempy.fabric as fabric
+import pandas as pd
+from typing import Optional, Union
+from sempy._utils._log import log
+import struct
+import uuid
+from itertools import chain, repeat
+from sempy.fabric.exceptions import FabricHTTPException
+from sempy_labs._helper_functions import resolve_warehouse_id
+
+
+def bytes2mswin_bstr(value: bytes) -> bytes:
+    """Convert a sequence of bytes into a (MS-Windows) BSTR (as bytes).
+
+    See https://github.com/mkleehammer/pyodbc/issues/228#issuecomment-319190980
+    for the original code.  It appears the input is converted to an
+    MS-Windows BSTR (in 'Little-endian' format).
+
+    See https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-dtyp\
+       /692a42a9-06ce-4394-b9bc-5d2a50440168
+    for more info on BSTR.
+
+    :param value: the sequence of bytes to convert
+    :return: the converted value (as a sequence of bytes)
+    """
+
+    encoded_bytes = bytes(chain.from_iterable(zip(value, repeat(0))))
+    return struct.pack("<i", len(encoded_bytes)) + encoded_bytes
+
+
+class ConnectWarehouse:
+    def __init__(
+        self,
+        warehouse: str,
+        workspace: Optional[Union[str, uuid.UUID]] = None,
+        timeout: Optional[int] = None,
+    ):
+        from sempy.fabric._token_provider import SynapseTokenProvider
+        import pyodbc
+
+        workspace = fabric.resolve_workspace_name(workspace)
+        workspace_id = fabric.resolve_workspace_id(workspace)
+        warehouse_id = resolve_warehouse_id(warehouse=warehouse, workspace=workspace)
+
+        # get the TDS endpoint
+        client = fabric.FabricRestClient()
+        response = client.get(f"v1/workspaces/{workspace_id}/warehouses/{warehouse_id}")
+        if response.status_code != 200:
+            raise FabricHTTPException(response)
+        tds_endpoint = response.json().get("properties", {}).get("connectionString")
+
+        access_token = SynapseTokenProvider()()
+        tokenstruct = bytes2mswin_bstr(access_token.encode())
+        conn_str = f"DRIVER={{ODBC Driver 18 for SQL Server}};SERVER={tds_endpoint};DATABASE={warehouse};Encrypt=Yes;"
+
+        if timeout is not None:
+            conn_str += f"Connect Timeout={timeout};"
+
+        self.connection = pyodbc.connect(conn_str, attrs_before={1256: tokenstruct})
+
+    @log
+    def query(self, sql: str) -> pd.DataFrame:
+        """
+        Runs a SQL query against a Fabric Warehouse.
+
+        Parameters
+        ----------
+        sql : str
+            The SQL query.
+
+        Returns
+        -------
+        pandas.DataFrame
+            A pandas dataframe with the result of the SQL query.
+        """
+        cursor = None
+
+        try:
+            cursor = self.connection.cursor()
+            cursor.execute(sql)
+
+            return pd.DataFrame.from_records(
+                cursor.fetchall(), columns=[col[0] for col in cursor.description]
+            )
+        finally:
+            if cursor:
+                cursor.close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, type, value, traceback):
+        self.close()
+
+    def close(self):
+        self.connection.close()

sempy_labs/_translations.py

@@ -32,7 +32,6 @@ def translate_semantic_model(
     -------
     pandas.DataFrame
         Shows a pandas dataframe which displays all of the translations in the semantic model.
-
     """
 
     from synapse.ml.services import Translate

sempy_labs/_workspace_identity.py

@@ -0,0 +1,66 @@
+import sempy.fabric as fabric
+from sempy_labs._helper_functions import (
+    resolve_workspace_name_and_id,
+    lro,
+)
+from typing import Optional
+import sempy_labs._icons as icons
+from sempy.fabric.exceptions import FabricHTTPException
+
+
+def provision_workspace_identity(workspace: Optional[str] = None):
+    """
+    Provisions a workspace identity for a workspace.
+
+    Parameters
+    ----------
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    # https://learn.microsoft.com/en-us/rest/api/fabric/core/workspaces/provision-identity?tabs=HTTP
+
+    workspace, workspace_id = resolve_workspace_name_and_id(workspace)
+
+    client = fabric.FabricRestClient()
+    response = client.post(f"/v1/workspaces/{workspace_id}/provisionIdentity")
+
+    if response.status_code not in [200, 202]:
+        raise FabricHTTPException(response)
+
+    lro(client, response)
+
+    print(
+        f"{icons.green_dot} A workspace identity has been provisioned for the '{workspace}' workspace."
+    )
+
+
+def deprovision_workspace_identity(workspace: Optional[str] = None):
+    """
+    Deprovisions a workspace identity for a workspace.
+
+    Parameters
+    ----------
+    workspace : str, default=None
+        The Fabric workspace name.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+    """
+
+    # https://learn.microsoft.com/en-us/rest/api/fabric/core/workspaces/deprovision-identity?tabs=HTTP
+
+    workspace, workspace_id = resolve_workspace_name_and_id(workspace)
+
+    client = fabric.FabricRestClient()
+    response = client.post(f"/v1/workspaces/{workspace_id}/deprovisionIdentity")
+
+    if response.status_code not in [200, 202]:
+        raise FabricHTTPException(response)
+
+    lro(client, response)
+
+    print(
+        f"{icons.green_dot} The workspace identity has been deprovisioned from the '{workspace}' workspace."
+    )

sempy_labs/directlake/__init__.py

@@ -1,3 +1,4 @@
+from sempy_labs.directlake._generate_shared_expression import generate_shared_expression
 from sempy_labs.directlake._directlake_schema_compare import direct_lake_schema_compare
 from sempy_labs.directlake._directlake_schema_sync import direct_lake_schema_sync
 from sempy_labs.directlake._dl_helper import (
@@ -31,6 +32,7 @@ from sempy_labs.directlake._warm_cache import (
 )
 
 __all__ = [
+    "generate_shared_expression",
     "direct_lake_schema_compare",
     "direct_lake_schema_sync",
     "check_fallback_reason",

sempy_labs/directlake/_directlake_schema_compare.py

@@ -6,7 +6,6 @@ from sempy_labs._helper_functions import (
 from IPython.display import display
 from sempy_labs.lakehouse import get_lakehouse_columns
 from sempy_labs.directlake._dl_helper import get_direct_lake_source
-from sempy_labs._list_functions import list_tables
 from typing import Optional
 import sempy_labs._icons as icons
 from sempy._utils._log import log
@@ -61,7 +60,7 @@ def direct_lake_schema_compare(
             f"{icons.red_dot} The '{dataset}' semantic model is not in Direct Lake mode."
         )
 
-    dfT = list_tables(dataset, workspace)
+    dfT = fabric.list_tables(dataset=dataset, workspace=workspace)
     dfC = fabric.list_columns(dataset=dataset, workspace=workspace)
     lc = get_lakehouse_columns(lakehouse_name, lakehouse_workspace)
 

sempy_labs/directlake/_dl_helper.py

@@ -126,18 +126,15 @@ def generate_direct_lake_semantic_model(
     expr = get_shared_expression(lakehouse=lakehouse, workspace=lakehouse_workspace)
     dfD = fabric.list_datasets(workspace=workspace)
     dfD_filt = dfD[dfD["Dataset Name"] == dataset]
-    dfD_filt_len = len(dfD_filt)
 
-    if dfD_filt_len > 0 and overwrite is False:
+    if len(dfD_filt) > 0 and not overwrite:
         raise ValueError(
             f"{icons.red_dot} The '{dataset}' semantic model within the '{workspace}' workspace already exists. Overwrite is set to False so the new semantic model has not been created."
         )
-    if dfD_filt_len > 0 and overwrite:
-        print(
-            f"{icons.warning} Overwriting the existing '{dataset}' semantic model within the '{workspace}' workspace."
-        )
 
-    create_blank_semantic_model(dataset=dataset, workspace=workspace)
+    create_blank_semantic_model(
+        dataset=dataset, workspace=workspace, overwrite=overwrite
+    )
 
     @retry(
         sleep_time=1,

sempy_labs/directlake/_generate_shared_expression.py

@@ -0,0 +1,85 @@
+import sempy.fabric as fabric
+from sempy_labs._helper_functions import (
+    resolve_lakehouse_name,
+    resolve_lakehouse_id,
+    resolve_warehouse_id,
+)
+from typing import Optional
+import sempy_labs._icons as icons
+from sempy.fabric.exceptions import FabricHTTPException
+
+
+def generate_shared_expression(
+    item_name: Optional[str] = None,
+    item_type: Optional[str] = "Lakehouse",
+    workspace: Optional[str] = None,
+) -> str:
+    """
+    Dynamically generates the M expression used by a Direct Lake model for a given lakehouse/warehouse.
+
+    Parameters
+    ----------
+    item_name : str, default=None
+        The Fabric lakehouse or warehouse name.
+        Defaults to None which resolves to the lakehouse attached to the notebook.
+    item_type : str, default="Lakehouse"
+        The Fabric item name. Valid options: 'Lakehouse', 'Warehouse'.
+    workspace : str, default=None
+        The Fabric workspace used by the item.
+        Defaults to None which resolves to the workspace of the attached lakehouse
+        or if no lakehouse attached, resolves to the workspace of the notebook.
+
+    Returns
+    -------
+    str
+        Shows the expression which can be used to connect a Direct Lake semantic model to its SQL Endpoint.
+    """
+
+    workspace = fabric.resolve_workspace_name(workspace)
+    workspace_id = fabric.resolve_workspace_id(workspace)
+    item_types = ["Lakehouse", "Warehouse"]
+    item_type = item_type.capitalize()
+    if item_type not in item_types:
+        raise ValueError(
+            f"{icons.red_dot} Invalid item type. Valid options: {item_types}."
+        )
+
+    if item_name is None:
+        item_id = fabric.get_lakehouse_id()
+        item_name = resolve_lakehouse_name(item_id, workspace)
+    elif item_name is not None and item_type == "Lakehouse":
+        item_id = resolve_lakehouse_id(lakehouse=item_name, workspace=workspace)
+    elif item_type == "Warehouse":
+        item_id = resolve_warehouse_id(warehouse=item_name, workspace=workspace)
+
+    client = fabric.FabricRestClient()
+    item_type_rest = f"{item_type.lower()}s"
+    response = client.get(f"/v1/workspaces/{workspace_id}/{item_type_rest}/{item_id}")
+    if response.status_code != 200:
+        raise FabricHTTPException(response)
+
+    if item_type == "Lakehouse":
+        prop = response.json()["properties"]["sqlEndpointProperties"]
+        sqlEPCS = prop["connectionString"]
+        sqlepid = prop["id"]
+        provStatus = prop["provisioningStatus"]
+    elif item_type == "Warehouse":
+        prop = response.json()["properties"]
+        sqlEPCS = prop["connectionString"]
+        sqlepid = item_id
+        provStatus = None
+
+    if provStatus == "InProgress":
+        raise ValueError(
+            f"{icons.red_dot} The SQL Endpoint for the '{item_name}' lakehouse within the '{workspace}' workspace has not yet been provisioned. Please wait until it has been provisioned."
+        )
+
+    sh = (
+        'let\n\tdatabase = Sql.Database("'
+        + sqlEPCS
+        + '", "'
+        + sqlepid
+        + '")\nin\n\tdatabase'
+    )
+
+    return sh

sempy_labs/directlake/_show_unsupported_directlake_objects.py

@@ -1,6 +1,5 @@
 import sempy.fabric as fabric
 import pandas as pd
-from sempy_labs._list_functions import list_tables
 from sempy_labs._helper_functions import format_dax_object_name
 from typing import Optional, Tuple
 from sempy._utils._log import log
@@ -33,7 +32,7 @@ def show_unsupported_direct_lake_objects(
 
     workspace = fabric.resolve_workspace_name(workspace)
 
-    dfT = list_tables(dataset, workspace)
+    dfT = fabric.list_tables(dataset=dataset, workspace=workspace)
     dfC = fabric.list_columns(dataset=dataset, workspace=workspace)
     dfR = fabric.list_relationships(dataset=dataset, workspace=workspace)
 

sempy_labs/lakehouse/_get_lakehouse_tables.py

@@ -7,6 +7,7 @@ from sempy_labs._helper_functions import (
     resolve_lakehouse_id,
     resolve_lakehouse_name,
     resolve_workspace_name_and_id,
+    pagination,
 )
 from sempy_labs.directlake._guardrails import (
     get_sku_size,
@@ -52,8 +53,6 @@ def get_lakehouse_tables(
         Shows the tables/columns within a lakehouse and their properties.
     """
 
-    from sempy_labs._helper_functions import pagination
-
     df = pd.DataFrame(
         columns=[
             "Workspace Name",
@@ -96,6 +95,9 @@ def get_lakehouse_tables(
 
     responses = pagination(client, response)
 
+    if not responses[0].get("data"):
+        return df
+
     dfs = []
     for r in responses:
         for i in r.get("data", []):
@@ -108,7 +110,9 @@ def get_lakehouse_tables(
                 "Location": i.get("location"),
             }
             dfs.append(pd.DataFrame(new_data, index=[0]))
-    df = pd.concat(dfs, ignore_index=True)
+
+    if dfs:
+        df = pd.concat(dfs, ignore_index=True)
 
     if extended:
         sku_value = get_sku_size(workspace)

sempy_labs/migration/_migrate_calctables_to_lakehouse.py

@@ -52,6 +52,11 @@ def migrate_calc_tables_to_lakehouse(
         or if no lakehouse attached, resolves to the workspace of the notebook.
     """
 
+    if dataset == new_dataset:
+        raise ValueError(
+            f"{icons.red_dot} The 'dataset' and 'new_dataset' parameters are both set to '{dataset}'. These parameters must be set to different values."
+        )
+
     workspace = fabric.resolve_workspace_name(workspace)
 
     if new_dataset_workspace is None:

sempy_labs/migration/_migrate_calctables_to_semantic_model.py

@@ -48,6 +48,11 @@ def migrate_calc_tables_to_semantic_model(
         or if no lakehouse attached, resolves to the workspace of the notebook.
     """
 
+    if dataset == new_dataset:
+        raise ValueError(
+            f"{icons.red_dot} The 'dataset' and 'new_dataset' parameters are both set to '{dataset}'. These parameters must be set to different values."
+        )
+
     workspace = fabric.resolve_workspace_name(workspace)
 
     if new_dataset_workspace is None:

sempy_labs/migration/_migrate_model_objects_to_semantic_model.py

@@ -1,7 +1,6 @@
 import sempy
 import sempy.fabric as fabric
 import re
-from sempy_labs._list_functions import list_tables
 from sempy_labs._helper_functions import (
     create_relationship_name,
     retry,
@@ -43,12 +42,17 @@ def migrate_model_objects_to_semantic_model(
     import Microsoft.AnalysisServices.Tabular as TOM
     import System
 
+    if dataset == new_dataset:
+        raise ValueError(
+            f"{icons.red_dot} The 'dataset' and 'new_dataset' parameters are both set to '{dataset}'. These parameters must be set to different values."
+        )
+
     workspace = fabric.resolve_workspace_name(workspace)
 
     if new_dataset_workspace is None:
         new_dataset_workspace = workspace
 
-    dfT = list_tables(dataset, workspace)
+    dfT = fabric.list_tables(dataset=dataset, workspace=workspace)
     dfC = fabric.list_columns(dataset=dataset, workspace=workspace)
     dfM = fabric.list_measures(dataset=dataset, workspace=workspace)
     dfRole = fabric.get_roles(dataset=dataset, workspace=workspace)

sempy_labs/migration/_migrate_tables_columns_to_semantic_model.py

@@ -1,8 +1,5 @@
 import sempy.fabric as fabric
 import pandas as pd
-import datetime
-import time
-from sempy_labs._list_functions import list_tables
 from sempy_labs.directlake._get_shared_expression import get_shared_expression
 from sempy_labs._helper_functions import resolve_lakehouse_name, retry
 from sempy_labs.lakehouse._lakehouse import lakehouse_attached
@@ -45,9 +42,13 @@ def migrate_tables_columns_to_semantic_model(
         The Fabric workspace used by the lakehouse.
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
-
     """
 
+    if dataset == new_dataset:
+        raise ValueError(
+            f"{icons.red_dot} The 'dataset' and 'new_dataset' parameters are both set to '{dataset}'. These parameters must be set to different values."
+        )
+
     workspace = fabric.resolve_workspace_name(workspace)
 
     if new_dataset_workspace is None:
@@ -71,7 +72,7 @@ def migrate_tables_columns_to_semantic_model(
     shEx = get_shared_expression(lakehouse, lakehouse_workspace)
 
     dfC = fabric.list_columns(dataset=dataset, workspace=workspace)
-    dfT = list_tables(dataset, workspace)
+    dfT = fabric.list_tables(dataset=dataset, workspace=workspace)
     dfT.rename(columns={"Type": "Table Type"}, inplace=True)
     dfC = pd.merge(
         dfC,

sempy_labs/migration/_migration_validation.py

@@ -3,6 +3,7 @@ import pandas as pd
 from typing import Optional
 from sempy_labs._list_functions import list_semantic_model_objects
 from sempy._utils._log import log
+import sempy_labs._icons as icons
 
 
 @log
@@ -36,6 +37,11 @@ def migration_validation(
        A pandas dataframe showing a list of objects and whether they were successfully migrated. Also shows the % of objects which were migrated successfully.
     """
 
+    if dataset == new_dataset:
+        raise ValueError(
+            f"{icons.red_dot} The 'dataset' and 'new_dataset' parameters are both set to '{dataset}'. These parameters must be set to different values."
+        )
+
     workspace = fabric.resolve_workspace_name(workspace)
     if new_dataset_workspace is None:
         new_dataset_workspace = workspace

sempy_labs/report/_report_functions.py

@@ -3,24 +3,21 @@ import pandas as pd
 import json
 import os
 import time
-import base64
 import copy
 from anytree import Node, RenderTree
 from powerbiclient import Report
-from synapse.ml.services import Translate
 from pyspark.sql.functions import col, flatten
-from pyspark.sql import SparkSession
 from sempy_labs.report._generate_report import update_report_from_reportjson
 from sempy_labs.lakehouse._lakehouse import lakehouse_attached
 from sempy_labs._helper_functions import (
     generate_embedded_filter,
-    resolve_dataset_name,
     resolve_report_id,
     resolve_lakehouse_name,
     language_validate,
     resolve_workspace_name_and_id,
     lro,
     _decode_b64,
+    resolve_dataset_id,
 )
 from typing import List, Optional, Union
 from sempy._utils._log import log
@@ -405,6 +402,7 @@ def clone_report(
     workspace: Optional[str] = None,
     target_workspace: Optional[str] = None,
     target_dataset: Optional[str] = None,
+    target_dataset_workspace: Optional[str] = None,
 ):
     """
     Clones a Power BI report.
@@ -426,6 +424,9 @@ def clone_report(
     target_dataset : str, default=None
         The name of the semantic model to be used by the cloned report.
         Defaults to None which resolves to the semantic model used by the initial report.
+    target_dataset_workspace : str, default=None
+        The workspace in which the semantic model to be used by the report resides.
+        Defaults to None which resolves to the semantic model used by the initial report.
     """
 
     # https://learn.microsoft.com/rest/api/power-bi/reports/clone-report-in-group
@@ -446,48 +447,25 @@ def clone_report(
         target_workspace = workspace
         target_workspace_id = workspace_id
     else:
-        dfW = fabric.list_workspaces()
-        dfW_filt = dfW[dfW["Name"] == target_workspace]
-
-        if len(dfW_filt) == 0:
-            raise ValueError(
-                f"{icons.red_dot} The '{workspace}' is not a valid workspace."
-            )
+        target_workspace_id = fabric.resolve_workspace_id(target_workspace)
 
-        target_workspace_id = dfW_filt["Id"].iloc[0]
+    if target_dataset is not None:
+        if target_dataset_workspace is None:
+            target_dataset_workspace = workspace
+        target_dataset_id = resolve_dataset_id(target_dataset, target_dataset_workspace)
 
-    if target_dataset is None:
-        dfR = fabric.list_reports(workspace=target_workspace)
-        dfR_filt = dfR[dfR["Name"] == report]
-        target_dataset_id = dfR_filt["Dataset Id"].iloc[0]
-        target_dataset = resolve_dataset_name(
-            dataset_id=target_dataset_id, workspace=target_workspace
+    if report == cloned_report and workspace == target_workspace:
+        raise ValueError(
+            f"{icons.warning} The 'report' and 'cloned_report' parameters have the same value of '{report}. The 'workspace' and 'target_workspace' have the same value of '{workspace}'. Either the 'cloned_report' or the 'target_workspace' must be different from the original report."
         )
-    else:
-        dfD = fabric.list_datasets(workspace=target_workspace)
-        dfD_filt = dfD[dfD["Dataset Name"] == target_dataset]
-
-        if len(dfD_filt) == 0:
-            raise ValueError(
-                f"{icons.red_dot} The '{target_dataset}' target dataset does not exist in the '{target_workspace}' workspace."
-            )
-
-        target_dataset_id = dfD_filt["Dataset Id"].iloc[0]
 
     client = fabric.PowerBIRestClient()
 
-    if target_workspace is None and target_dataset is None:
-        request_body = {"name": cloned_report}
-    elif target_workspace is not None and target_dataset is None:
-        request_body = {"name": cloned_report, "targetWorkspaceId": target_workspace_id}
-    elif target_workspace is not None and target_dataset is not None:
-        request_body = {
-            "name": cloned_report,
-            "targetModelId": target_dataset_id,
-            "targetWorkspaceId": target_workspace_id,
-        }
-    elif target_workspace is None and target_dataset is not None:
-        request_body = {"name": cloned_report, "targetModelId": target_dataset_id}
+    request_body = {"name": cloned_report}
+    if target_dataset is not None:
+        request_body["targetModelId"] = target_dataset_id
+    if target_workspace != workspace:
+        request_body["targetWorkspaceId"] = target_workspace_id
 
     response = client.post(
         f"/v1.0/myorg/groups/{workspace_id}/reports/{reportId}/Clone", json=request_body
@@ -496,8 +474,7 @@ def clone_report(
     if response.status_code != 200:
         raise FabricHTTPException(response)
     print(
-        f"{icons.green_dot} The '{report}' report has been successfully cloned as the '{cloned_report}' report within the"
-        f" '{target_workspace}' workspace using the '{target_dataset}' semantic model."
+        f"{icons.green_dot} The '{report}' report has been successfully cloned as the '{cloned_report}' report within the '{target_workspace}' workspace."
     )
 
 
@@ -756,6 +733,8 @@ def translate_report_titles(
         Defaults to None which resolves to the workspace of the attached lakehouse
         or if no lakehouse attached, resolves to the workspace of the notebook.
     """
+    from synapse.ml.services import Translate
+    from pyspark.sql import SparkSession
 
     if isinstance(languages, str):
         languages = [languages]

sempy_labs/report/_report_rebind.py

@@ -101,6 +101,11 @@ def report_rebind_all(
 
     from sempy_labs._list_functions import list_reports_using_semantic_model
 
+    if dataset == new_dataset:
+        raise ValueError(
+            f"{icons.red_dot} The 'dataset' and 'new_dataset' parameters are both set to '{dataset}'. These parameters must be set to different values."
+        )
+
     dataset_workspace = fabric.resolve_workspace_name(dataset_workspace)
 
     if new_dataset_workpace is None: