semantic-link-labs 0.9.6__py3-none-any.whl → 0.9.8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of semantic-link-labs might be problematic. Click here for more details.
- {semantic_link_labs-0.9.6.dist-info → semantic_link_labs-0.9.8.dist-info}/METADATA +8 -5
- {semantic_link_labs-0.9.6.dist-info → semantic_link_labs-0.9.8.dist-info}/RECORD +35 -32
- {semantic_link_labs-0.9.6.dist-info → semantic_link_labs-0.9.8.dist-info}/WHEEL +1 -1
- sempy_labs/__init__.py +4 -0
- sempy_labs/_ai.py +3 -1
- sempy_labs/_capacities.py +0 -1
- sempy_labs/_dax_query_view.py +2 -0
- sempy_labs/_delta_analyzer_history.py +298 -0
- sempy_labs/_helper_functions.py +65 -16
- sempy_labs/_icons.py +6 -6
- sempy_labs/_list_functions.py +3 -1
- sempy_labs/_model_bpa_bulk.py +10 -11
- sempy_labs/_model_bpa_rules.py +1 -1
- sempy_labs/admin/_basic_functions.py +28 -2
- sempy_labs/admin/_reports.py +1 -1
- sempy_labs/admin/_scanner.py +0 -2
- sempy_labs/admin/_tenant.py +8 -3
- sempy_labs/directlake/_generate_shared_expression.py +9 -1
- sempy_labs/directlake/_update_directlake_model_lakehouse_connection.py +82 -36
- sempy_labs/directlake/_update_directlake_partition_entity.py +3 -0
- sempy_labs/graph/_groups.py +6 -0
- sempy_labs/graph/_teams.py +2 -0
- sempy_labs/graph/_users.py +4 -0
- sempy_labs/lakehouse/__init__.py +12 -3
- sempy_labs/lakehouse/_blobs.py +231 -0
- sempy_labs/lakehouse/_shortcuts.py +22 -3
- sempy_labs/migration/_direct_lake_to_import.py +47 -10
- sempy_labs/report/__init__.py +4 -0
- sempy_labs/report/_report_functions.py +3 -3
- sempy_labs/report/_report_helper.py +17 -5
- sempy_labs/report/_reportwrapper.py +17 -8
- sempy_labs/report/_save_report.py +147 -0
- sempy_labs/tom/_model.py +156 -23
- {semantic_link_labs-0.9.6.dist-info → semantic_link_labs-0.9.8.dist-info/licenses}/LICENSE +0 -0
- {semantic_link_labs-0.9.6.dist-info → semantic_link_labs-0.9.8.dist-info}/top_level.txt +0 -0
sempy_labs/_helper_functions.py
CHANGED
|
@@ -183,7 +183,7 @@ def resolve_report_name(report_id: UUID, workspace: Optional[str | UUID] = None)
|
|
|
183
183
|
The name of the Power BI report.
|
|
184
184
|
"""
|
|
185
185
|
|
|
186
|
-
return resolve_item_name(item_id=report_id,
|
|
186
|
+
return resolve_item_name(item_id=report_id, workspace=workspace)
|
|
187
187
|
|
|
188
188
|
|
|
189
189
|
def delete_item(
|
|
@@ -392,7 +392,7 @@ def resolve_lakehouse_name_and_id(
|
|
|
392
392
|
lakehouse: Optional[str | UUID] = None, workspace: Optional[str | UUID] = None
|
|
393
393
|
) -> Tuple[str, UUID]:
|
|
394
394
|
|
|
395
|
-
|
|
395
|
+
workspace_id = resolve_workspace_id(workspace)
|
|
396
396
|
type = "Lakehouse"
|
|
397
397
|
|
|
398
398
|
if lakehouse is None:
|
|
@@ -469,9 +469,7 @@ def resolve_dataset_name(
|
|
|
469
469
|
The name of the semantic model.
|
|
470
470
|
"""
|
|
471
471
|
|
|
472
|
-
return resolve_item_name(
|
|
473
|
-
item_id=dataset_id, type="SemanticModel", workspace=workspace
|
|
474
|
-
)
|
|
472
|
+
return resolve_item_name(item_id=dataset_id, workspace=workspace)
|
|
475
473
|
|
|
476
474
|
|
|
477
475
|
def resolve_lakehouse_name(
|
|
@@ -503,9 +501,7 @@ def resolve_lakehouse_name(
|
|
|
503
501
|
f"{icons.red_dot} Cannot resolve a lakehouse. Please enter a valid lakehouse or make sure a lakehouse is attached to the notebook."
|
|
504
502
|
)
|
|
505
503
|
|
|
506
|
-
return resolve_item_name(
|
|
507
|
-
item_id=lakehouse_id, type="Lakehouse", workspace=workspace
|
|
508
|
-
)
|
|
504
|
+
return resolve_item_name(item_id=lakehouse_id, workspace=workspace)
|
|
509
505
|
|
|
510
506
|
|
|
511
507
|
def resolve_lakehouse_id(
|
|
@@ -1308,10 +1304,8 @@ class FabricTokenCredential(TokenCredential):
|
|
|
1308
1304
|
|
|
1309
1305
|
import notebookutils
|
|
1310
1306
|
|
|
1311
|
-
token = notebookutils.credentials.getToken(
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
return access_token
|
|
1307
|
+
token = notebookutils.credentials.getToken("storage")
|
|
1308
|
+
return AccessToken(token, 0)
|
|
1315
1309
|
|
|
1316
1310
|
|
|
1317
1311
|
def _get_adls_client(account_name):
|
|
@@ -1320,11 +1314,21 @@ def _get_adls_client(account_name):
|
|
|
1320
1314
|
|
|
1321
1315
|
account_url = f"https://{account_name}.dfs.core.windows.net"
|
|
1322
1316
|
|
|
1323
|
-
|
|
1324
|
-
|
|
1317
|
+
return DataLakeServiceClient(account_url, credential=FabricTokenCredential())
|
|
1318
|
+
|
|
1319
|
+
|
|
1320
|
+
def _get_blob_client(workspace_id: UUID, item_id: UUID):
|
|
1321
|
+
|
|
1322
|
+
from azure.storage.blob import BlobServiceClient
|
|
1323
|
+
|
|
1324
|
+
endpoint = _get_fabric_context_setting(name="trident.onelake.endpoint").replace(
|
|
1325
|
+
".dfs.", ".blob."
|
|
1325
1326
|
)
|
|
1327
|
+
url = f"https://{endpoint}/{workspace_id}/{item_id}"
|
|
1328
|
+
|
|
1329
|
+
# account_url = f"https://{account_name}.blob.core.windows.net"
|
|
1326
1330
|
|
|
1327
|
-
return
|
|
1331
|
+
return BlobServiceClient(url, credential=FabricTokenCredential())
|
|
1328
1332
|
|
|
1329
1333
|
|
|
1330
1334
|
def resolve_warehouse_id(
|
|
@@ -1889,7 +1893,9 @@ def _run_spark_sql_query(query):
|
|
|
1889
1893
|
return spark.sql(query)
|
|
1890
1894
|
|
|
1891
1895
|
|
|
1892
|
-
def _mount(
|
|
1896
|
+
def _mount(
|
|
1897
|
+
lakehouse: Optional[str | UUID] = None, workspace: Optional[str | UUID] = None
|
|
1898
|
+
) -> str:
|
|
1893
1899
|
"""
|
|
1894
1900
|
Mounts a lakehouse to a notebook if it is not already mounted. Returns the local path to the lakehouse.
|
|
1895
1901
|
"""
|
|
@@ -1901,6 +1907,16 @@ def _mount(lakehouse, workspace) -> str:
|
|
|
1901
1907
|
lakehouse=lakehouse, workspace=workspace
|
|
1902
1908
|
)
|
|
1903
1909
|
|
|
1910
|
+
# Hide display mounts
|
|
1911
|
+
current_setting = ""
|
|
1912
|
+
try:
|
|
1913
|
+
current_setting = notebookutils.conf.get(
|
|
1914
|
+
"spark.notebookutils.displaymountpoint.enabled"
|
|
1915
|
+
)
|
|
1916
|
+
notebookutils.conf.set("spark.notebookutils.displaymountpoint.enabled", "false")
|
|
1917
|
+
except Exception:
|
|
1918
|
+
pass
|
|
1919
|
+
|
|
1904
1920
|
lake_path = create_abfss_path(lakehouse_id, workspace_id)
|
|
1905
1921
|
mounts = notebookutils.fs.mounts()
|
|
1906
1922
|
mount_point = f"/{workspace_name.replace(' ', '')}{lakehouse_name.replace(' ', '')}"
|
|
@@ -1912,6 +1928,16 @@ def _mount(lakehouse, workspace) -> str:
|
|
|
1912
1928
|
)
|
|
1913
1929
|
|
|
1914
1930
|
mounts = notebookutils.fs.mounts()
|
|
1931
|
+
|
|
1932
|
+
# Set display mounts to original setting
|
|
1933
|
+
try:
|
|
1934
|
+
if current_setting != "false":
|
|
1935
|
+
notebookutils.conf.set(
|
|
1936
|
+
"spark.notebookutils.displaymountpoint.enabled", "true"
|
|
1937
|
+
)
|
|
1938
|
+
except Exception:
|
|
1939
|
+
pass
|
|
1940
|
+
|
|
1915
1941
|
local_path = next(
|
|
1916
1942
|
i.get("localPath") for i in mounts if i.get("source") == lake_path
|
|
1917
1943
|
)
|
|
@@ -2015,3 +2041,26 @@ def _get_or_create_warehouse(
|
|
|
2015
2041
|
)
|
|
2016
2042
|
|
|
2017
2043
|
return (warehouse, warehouse_id)
|
|
2044
|
+
|
|
2045
|
+
|
|
2046
|
+
def _xml_to_dict(element):
|
|
2047
|
+
data = {element.tag: {} if element.attrib else None}
|
|
2048
|
+
children = list(element)
|
|
2049
|
+
if children:
|
|
2050
|
+
temp_dict = {}
|
|
2051
|
+
for child in children:
|
|
2052
|
+
child_dict = _xml_to_dict(child)
|
|
2053
|
+
for key, value in child_dict.items():
|
|
2054
|
+
if key in temp_dict:
|
|
2055
|
+
if isinstance(temp_dict[key], list):
|
|
2056
|
+
temp_dict[key].append(value)
|
|
2057
|
+
else:
|
|
2058
|
+
temp_dict[key] = [temp_dict[key], value]
|
|
2059
|
+
else:
|
|
2060
|
+
temp_dict[key] = value
|
|
2061
|
+
data[element.tag] = temp_dict
|
|
2062
|
+
else:
|
|
2063
|
+
data[element.tag] = (
|
|
2064
|
+
element.text.strip() if element.text and element.text.strip() else None
|
|
2065
|
+
)
|
|
2066
|
+
return data
|
sempy_labs/_icons.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
green_dot = "\
|
|
2
|
-
yellow_dot = "\
|
|
3
|
-
red_dot = "\
|
|
1
|
+
green_dot = "\U0001f7e2"
|
|
2
|
+
yellow_dot = "\U0001f7e1"
|
|
3
|
+
red_dot = "\U0001f534"
|
|
4
4
|
in_progress = "⌛"
|
|
5
5
|
checked = "\u2611"
|
|
6
6
|
unchecked = "\u2610"
|
|
@@ -8,11 +8,11 @@ start_bold = "\033[1m"
|
|
|
8
8
|
end_bold = "\033[0m"
|
|
9
9
|
bullet = "\u2022"
|
|
10
10
|
warning = "⚠️"
|
|
11
|
-
error = "\
|
|
11
|
+
error = "\u274c"
|
|
12
12
|
info = "ℹ️"
|
|
13
13
|
measure_icon = "\u2211"
|
|
14
|
-
table_icon = "\
|
|
15
|
-
column_icon = "\
|
|
14
|
+
table_icon = "\u229e"
|
|
15
|
+
column_icon = "\u229f"
|
|
16
16
|
model_bpa_name = "ModelBPA"
|
|
17
17
|
report_bpa_name = "ReportBPA"
|
|
18
18
|
severity_mapping = {warning: "Warning", error: "Error", info: "Info"}
|
sempy_labs/_list_functions.py
CHANGED
|
@@ -1476,7 +1476,9 @@ def list_server_properties(workspace: Optional[str | UUID] = None) -> pd.DataFra
|
|
|
1476
1476
|
A pandas dataframe showing a list of the server properties.
|
|
1477
1477
|
"""
|
|
1478
1478
|
|
|
1479
|
-
tom_server = fabric.create_tom_server(
|
|
1479
|
+
tom_server = fabric.create_tom_server(
|
|
1480
|
+
dataset=None, readonly=True, workspace=workspace
|
|
1481
|
+
)
|
|
1480
1482
|
|
|
1481
1483
|
rows = [
|
|
1482
1484
|
{
|
sempy_labs/_model_bpa_bulk.py
CHANGED
|
@@ -2,7 +2,6 @@ import sempy.fabric as fabric
|
|
|
2
2
|
import pandas as pd
|
|
3
3
|
import datetime
|
|
4
4
|
from sempy_labs._helper_functions import (
|
|
5
|
-
resolve_lakehouse_name,
|
|
6
5
|
save_as_delta_table,
|
|
7
6
|
resolve_workspace_capacity,
|
|
8
7
|
retry,
|
|
@@ -26,7 +25,7 @@ def run_model_bpa_bulk(
|
|
|
26
25
|
rules: Optional[pd.DataFrame] = None,
|
|
27
26
|
extended: bool = False,
|
|
28
27
|
language: Optional[str] = None,
|
|
29
|
-
workspace: Optional[str | List[str]] = None,
|
|
28
|
+
workspace: Optional[str | UUID | List[str | UUID]] = None,
|
|
30
29
|
skip_models: Optional[str | List[str]] = ["ModelBPA", "Fabric Capacity Metrics"],
|
|
31
30
|
skip_models_in_workspace: Optional[dict] = None,
|
|
32
31
|
):
|
|
@@ -44,8 +43,8 @@ def run_model_bpa_bulk(
|
|
|
44
43
|
language : str, default=None
|
|
45
44
|
The language (code) in which the rules will appear. For example, specifying 'it-IT' will show the Rule Name, Category and Description in Italian.
|
|
46
45
|
Defaults to None which resolves to English.
|
|
47
|
-
workspace : str | List[str], default=None
|
|
48
|
-
The workspace or list of workspaces to scan.
|
|
46
|
+
workspace : str | uuid.UUID | List[str | uuid.UUID], default=None
|
|
47
|
+
The workspace or list of workspaces to scan. Supports both the workspace name and the workspace id.
|
|
49
48
|
Defaults to None which scans all accessible workspaces.
|
|
50
49
|
skip_models : str | List[str], default=['ModelBPA', 'Fabric Capacity Metrics']
|
|
51
50
|
The semantic models to always skip when running this analysis.
|
|
@@ -71,7 +70,7 @@ def run_model_bpa_bulk(
|
|
|
71
70
|
output_table = "modelbparesults"
|
|
72
71
|
lakeT = get_lakehouse_tables()
|
|
73
72
|
lakeT_filt = lakeT[lakeT["Table Name"] == output_table]
|
|
74
|
-
if
|
|
73
|
+
if lakeT_filt.empty:
|
|
75
74
|
runId = 1
|
|
76
75
|
else:
|
|
77
76
|
max_run_id = _get_column_aggregate(table_name=output_table)
|
|
@@ -84,14 +83,14 @@ def run_model_bpa_bulk(
|
|
|
84
83
|
if workspace is None:
|
|
85
84
|
dfW_filt = dfW.copy()
|
|
86
85
|
else:
|
|
87
|
-
dfW_filt = dfW[dfW["Name"].isin(workspace)]
|
|
86
|
+
dfW_filt = dfW[(dfW["Name"].isin(workspace)) | (dfW["Id"].isin(workspace))]
|
|
88
87
|
|
|
89
|
-
if
|
|
88
|
+
if dfW_filt.empty:
|
|
90
89
|
raise ValueError(
|
|
91
90
|
f"{icons.red_dot} There are no valid workspaces to assess. This is likely due to not having proper permissions to the workspace(s) entered in the 'workspace' parameter."
|
|
92
91
|
)
|
|
93
92
|
|
|
94
|
-
for
|
|
93
|
+
for _, r in dfW_filt.iterrows():
|
|
95
94
|
wksp = r["Name"]
|
|
96
95
|
wksp_id = r["Id"]
|
|
97
96
|
capacity_id, capacity_name = resolve_workspace_capacity(workspace=wksp)
|
|
@@ -106,7 +105,7 @@ def run_model_bpa_bulk(
|
|
|
106
105
|
dfD = dfD[~dfD["Dataset Name"].isin(skip_models_wkspc)]
|
|
107
106
|
|
|
108
107
|
# Exclude default semantic models
|
|
109
|
-
if
|
|
108
|
+
if not dfD.empty:
|
|
110
109
|
dfI = fabric.list_items(workspace=wksp)
|
|
111
110
|
filtered_df = dfI.groupby("Display Name").filter(
|
|
112
111
|
lambda x: set(["Warehouse", "SemanticModel"]).issubset(set(x["Type"]))
|
|
@@ -116,7 +115,7 @@ def run_model_bpa_bulk(
|
|
|
116
115
|
skip_models.extend(default_semantic_models)
|
|
117
116
|
dfD_filt = dfD[~dfD["Dataset Name"].isin(skip_models)]
|
|
118
117
|
|
|
119
|
-
if
|
|
118
|
+
if not dfD_filt.empty:
|
|
120
119
|
for _, r2 in dfD_filt.iterrows():
|
|
121
120
|
dataset_id = r2["Dataset Id"]
|
|
122
121
|
dataset_name = r2["Dataset Name"]
|
|
@@ -159,7 +158,7 @@ def run_model_bpa_bulk(
|
|
|
159
158
|
)
|
|
160
159
|
print(e)
|
|
161
160
|
|
|
162
|
-
if
|
|
161
|
+
if df.empty:
|
|
163
162
|
print(
|
|
164
163
|
f"{icons.yellow_dot} No BPA results to save for the '{wksp}' workspace."
|
|
165
164
|
)
|
sempy_labs/_model_bpa_rules.py
CHANGED
|
@@ -556,7 +556,7 @@ def model_bpa_rules(
|
|
|
556
556
|
"Warning",
|
|
557
557
|
"Use the DIVIDE function for division",
|
|
558
558
|
lambda obj, tom: re.search(
|
|
559
|
-
r"\]\s*\/(?!\/)(?!\*)
|
|
559
|
+
r"\]\s*\/(?!\/)(?!\*)|\)\s*\/(?!\/)(?!\*)",
|
|
560
560
|
obj.Expression,
|
|
561
561
|
flags=re.IGNORECASE,
|
|
562
562
|
),
|
|
@@ -350,14 +350,40 @@ def list_workspace_access_details(
|
|
|
350
350
|
return df
|
|
351
351
|
|
|
352
352
|
|
|
353
|
+
def _resolve_workspace_name(workspace_id: Optional[UUID] = None) -> str:
|
|
354
|
+
from sempy_labs._helper_functions import _get_fabric_context_setting
|
|
355
|
+
from sempy.fabric.exceptions import FabricHTTPException
|
|
356
|
+
|
|
357
|
+
if workspace_id is None:
|
|
358
|
+
workspace_id = _get_fabric_context_setting(name="trident.workspace.id")
|
|
359
|
+
|
|
360
|
+
try:
|
|
361
|
+
workspace_name = (
|
|
362
|
+
_base_api(
|
|
363
|
+
request=f"/v1/admin/workspaces/{workspace_id}", client="fabric_sp"
|
|
364
|
+
)
|
|
365
|
+
.json()
|
|
366
|
+
.get("name")
|
|
367
|
+
)
|
|
368
|
+
except FabricHTTPException:
|
|
369
|
+
raise ValueError(
|
|
370
|
+
f"{icons.red_dot} The '{workspace_id}' workspace was not found."
|
|
371
|
+
)
|
|
372
|
+
return workspace_name
|
|
373
|
+
|
|
374
|
+
|
|
353
375
|
def _resolve_workspace_name_and_id(
|
|
354
376
|
workspace: str | UUID,
|
|
355
377
|
) -> Tuple[str, UUID]:
|
|
356
378
|
|
|
357
|
-
from sempy_labs._helper_functions import
|
|
379
|
+
from sempy_labs._helper_functions import _get_fabric_context_setting
|
|
358
380
|
|
|
359
381
|
if workspace is None:
|
|
360
|
-
|
|
382
|
+
workspace_id = _get_fabric_context_setting(name="trident.workspace.id")
|
|
383
|
+
workspace_name = _resolve_workspace_name(workspace_id)
|
|
384
|
+
elif _is_valid_uuid(workspace):
|
|
385
|
+
workspace_id = workspace
|
|
386
|
+
workspace_name = _resolve_workspace_name(workspace_id)
|
|
361
387
|
else:
|
|
362
388
|
dfW = list_workspaces(workspace=workspace)
|
|
363
389
|
if not dfW.empty:
|
sempy_labs/admin/_reports.py
CHANGED
|
@@ -113,7 +113,7 @@ def list_report_users(report: str | UUID) -> pd.DataFrame:
|
|
|
113
113
|
"""
|
|
114
114
|
Shows a list of users that have access to the specified report.
|
|
115
115
|
|
|
116
|
-
This is a wrapper function for the following API: `Admin - Reports GetDatasetUsersAsAdmin <https://learn.microsoft.com/rest/api/power-bi/admin/
|
|
116
|
+
This is a wrapper function for the following API: `Admin - Reports GetDatasetUsersAsAdmin <https://learn.microsoft.com/rest/api/power-bi/admin/reports-get-report-users-as-admin>`_.
|
|
117
117
|
|
|
118
118
|
Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
|
|
119
119
|
|
sempy_labs/admin/_scanner.py
CHANGED
sempy_labs/admin/_tenant.py
CHANGED
|
@@ -126,11 +126,10 @@ def list_capacity_tenant_settings_overrides(
|
|
|
126
126
|
if capacity_id is None:
|
|
127
127
|
# If capacity_id is None, we access 'Overrides' -> 'tenantSettings'
|
|
128
128
|
for override in r.get("overrides", []):
|
|
129
|
+
capacity_id = override.get("id")
|
|
129
130
|
tenant_settings = override.get("tenantSettings", [])
|
|
130
131
|
for setting in tenant_settings:
|
|
131
|
-
data.append(
|
|
132
|
-
create_new_data(setting)
|
|
133
|
-
) # No capacity_id needed here
|
|
132
|
+
data.append(create_new_data(setting, capacity_id))
|
|
134
133
|
else:
|
|
135
134
|
# If capacity_id is provided, we access 'value' directly for tenantSettings
|
|
136
135
|
for setting in r.get("value", []):
|
|
@@ -391,6 +390,7 @@ def list_workspaces_tenant_settings_overrides() -> pd.DataFrame:
|
|
|
391
390
|
"""
|
|
392
391
|
|
|
393
392
|
columns = {
|
|
393
|
+
"Workspace Id": "string",
|
|
394
394
|
"Setting Name": "string",
|
|
395
395
|
"Title": "string",
|
|
396
396
|
"Enabled": "bool",
|
|
@@ -409,8 +409,10 @@ def list_workspaces_tenant_settings_overrides() -> pd.DataFrame:
|
|
|
409
409
|
|
|
410
410
|
for r in responses:
|
|
411
411
|
for v in r.get("value", []):
|
|
412
|
+
workspace_id = v.get("id")
|
|
412
413
|
for setting in v.get("tenantSettings", []):
|
|
413
414
|
new_data = {
|
|
415
|
+
"Workspace Id": workspace_id,
|
|
414
416
|
"Setting Name": setting.get("settingName"),
|
|
415
417
|
"Title": setting.get("title"),
|
|
416
418
|
"Enabled": setting.get("enabled"),
|
|
@@ -447,6 +449,7 @@ def list_domain_tenant_settings_overrides() -> pd.DataFrame:
|
|
|
447
449
|
"""
|
|
448
450
|
|
|
449
451
|
columns = {
|
|
452
|
+
"Domain Id": "string",
|
|
450
453
|
"Setting Name": "string",
|
|
451
454
|
"Title": "string",
|
|
452
455
|
"Enabled": "bool",
|
|
@@ -466,8 +469,10 @@ def list_domain_tenant_settings_overrides() -> pd.DataFrame:
|
|
|
466
469
|
|
|
467
470
|
for r in responses:
|
|
468
471
|
for v in r.get("value", []):
|
|
472
|
+
domain_id = v.get("id")
|
|
469
473
|
for setting in v.get("tenantSettings", []):
|
|
470
474
|
new_data = {
|
|
475
|
+
"Domain Id": domain_id,
|
|
471
476
|
"Setting Name": setting.get("settingName"),
|
|
472
477
|
"Title": setting.get("title"),
|
|
473
478
|
"Enabled": setting.get("enabled"),
|
|
@@ -13,6 +13,7 @@ def generate_shared_expression(
|
|
|
13
13
|
item_name: Optional[str] = None,
|
|
14
14
|
item_type: str = "Lakehouse",
|
|
15
15
|
workspace: Optional[str | UUID] = None,
|
|
16
|
+
use_sql_endpoint: bool = True,
|
|
16
17
|
) -> str:
|
|
17
18
|
"""
|
|
18
19
|
Dynamically generates the M expression used by a Direct Lake model for a given lakehouse/warehouse.
|
|
@@ -28,6 +29,9 @@ def generate_shared_expression(
|
|
|
28
29
|
The Fabric workspace name or ID used by the item.
|
|
29
30
|
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
30
31
|
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
32
|
+
use_sql_endpoint : bool, default=True
|
|
33
|
+
Whether to use the SQL Endpoint for the lakehouse/warehouse.
|
|
34
|
+
If False, the expression will be generated without using the SQL Endpoint.
|
|
31
35
|
|
|
32
36
|
Returns
|
|
33
37
|
-------
|
|
@@ -78,4 +82,8 @@ def generate_shared_expression(
|
|
|
78
82
|
end_expr = "\nin\n\tdatabase"
|
|
79
83
|
mid_expr = f'Sql.Database("{sqlEPCS}", "{sqlepid}")'
|
|
80
84
|
|
|
81
|
-
|
|
85
|
+
# Build DL/OL expression
|
|
86
|
+
if not use_sql_endpoint and item_type == "Lakehouse":
|
|
87
|
+
return f'AzureDataLakeStorage{{"server":"onelake.dfs.fabric.microsoft.com","path":"/{workspace_id}/{item_id}/"}}'
|
|
88
|
+
else:
|
|
89
|
+
return f"{start_expr}{mid_expr}{end_expr}"
|
|
@@ -1,18 +1,64 @@
|
|
|
1
|
-
import sempy.fabric as fabric
|
|
2
1
|
from sempy_labs.directlake._generate_shared_expression import generate_shared_expression
|
|
3
2
|
from sempy_labs._helper_functions import (
|
|
4
|
-
resolve_lakehouse_name,
|
|
5
3
|
resolve_dataset_name_and_id,
|
|
6
4
|
resolve_workspace_name_and_id,
|
|
7
5
|
resolve_item_name_and_id,
|
|
8
6
|
resolve_lakehouse_name_and_id,
|
|
9
7
|
)
|
|
8
|
+
from sempy._utils._log import log
|
|
10
9
|
from sempy_labs.tom import connect_semantic_model
|
|
11
10
|
from typing import Optional
|
|
12
11
|
import sempy_labs._icons as icons
|
|
13
12
|
from uuid import UUID
|
|
13
|
+
import re
|
|
14
14
|
|
|
15
15
|
|
|
16
|
+
def _extract_expression_list(expression):
|
|
17
|
+
"""
|
|
18
|
+
Finds the pattern for DL/SQL & DL/OL expressions in the semantic model.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
pattern_sql = r'Sql\.Database\s*\(\s*"([^"]+)"\s*,\s*"([^"]+)"\s*\)'
|
|
22
|
+
pattern_no_sql = r'AzureDataLakeStorage\s*\{\s*"server".*?:\s*onelake\.dfs\.fabric\.microsoft\.com"\s*,\s*"path"\s*:\s*"/([\da-fA-F-]+)\s*/\s*([\da-fA-F-]+)\s*/"\s*\}'
|
|
23
|
+
|
|
24
|
+
match_sql = re.search(pattern_sql, expression)
|
|
25
|
+
match_no_sql = re.search(pattern_no_sql, expression)
|
|
26
|
+
|
|
27
|
+
result = []
|
|
28
|
+
if match_sql:
|
|
29
|
+
value_1, value_2 = match_sql.groups()
|
|
30
|
+
result = [value_1, value_2, True]
|
|
31
|
+
elif match_no_sql:
|
|
32
|
+
value_1, value_2 = match_no_sql.groups()
|
|
33
|
+
result = [value_1, value_2, False]
|
|
34
|
+
|
|
35
|
+
return result
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _get_direct_lake_expressions(
|
|
39
|
+
dataset: str | UUID, workspace: Optional[str | UUID] = None
|
|
40
|
+
) -> dict:
|
|
41
|
+
"""
|
|
42
|
+
Extracts a dictionary of all Direct Lake expressions from a semantic model.
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
from sempy_labs.tom import connect_semantic_model
|
|
46
|
+
|
|
47
|
+
result = {}
|
|
48
|
+
|
|
49
|
+
with connect_semantic_model(dataset=dataset, workspace=workspace) as tom:
|
|
50
|
+
for e in tom.model.Expressions:
|
|
51
|
+
expr_name = e.Name
|
|
52
|
+
expr = e.Expression
|
|
53
|
+
|
|
54
|
+
list_values = _extract_expression_list(expr)
|
|
55
|
+
if list_values:
|
|
56
|
+
result[expr_name] = list_values
|
|
57
|
+
|
|
58
|
+
return result
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
@log
|
|
16
62
|
def update_direct_lake_model_lakehouse_connection(
|
|
17
63
|
dataset: str | UUID,
|
|
18
64
|
workspace: Optional[str | UUID] = None,
|
|
@@ -39,41 +85,23 @@ def update_direct_lake_model_lakehouse_connection(
|
|
|
39
85
|
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
40
86
|
"""
|
|
41
87
|
|
|
42
|
-
(
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
icons.sll_tags.append("UpdateDLConnection")
|
|
50
|
-
|
|
51
|
-
shEx = generate_shared_expression(
|
|
52
|
-
item_name=lakehouse, item_type="Lakehouse", workspace=lakehouse_workspace
|
|
53
|
-
)
|
|
54
|
-
|
|
55
|
-
with connect_semantic_model(
|
|
56
|
-
dataset=dataset_id, readonly=False, workspace=workspace_id
|
|
57
|
-
) as tom:
|
|
58
|
-
|
|
59
|
-
if not tom.is_direct_lake():
|
|
60
|
-
raise ValueError(
|
|
61
|
-
f"{icons.red_dot} The '{dataset_name}' semantic model is not in Direct Lake. This function is only applicable to Direct Lake semantic models."
|
|
62
|
-
)
|
|
63
|
-
|
|
64
|
-
tom.model.Expressions["DatabaseQuery"].Expression = shEx
|
|
65
|
-
|
|
66
|
-
print(
|
|
67
|
-
f"{icons.green_dot} The expression in the '{dataset_name}' semantic model has been updated to point to the '{lakehouse}' lakehouse in the '{lakehouse_workspace}' workspace."
|
|
88
|
+
update_direct_lake_model_connection(
|
|
89
|
+
dataset=dataset,
|
|
90
|
+
workspace=workspace,
|
|
91
|
+
source=lakehouse,
|
|
92
|
+
source_type="Lakehouse",
|
|
93
|
+
source_workspace=lakehouse_workspace,
|
|
68
94
|
)
|
|
69
95
|
|
|
70
96
|
|
|
97
|
+
@log
|
|
71
98
|
def update_direct_lake_model_connection(
|
|
72
99
|
dataset: str | UUID,
|
|
73
100
|
workspace: Optional[str | UUID] = None,
|
|
74
101
|
source: Optional[str] = None,
|
|
75
102
|
source_type: str = "Lakehouse",
|
|
76
103
|
source_workspace: Optional[str | UUID] = None,
|
|
104
|
+
use_sql_endpoint: bool = True,
|
|
77
105
|
):
|
|
78
106
|
"""
|
|
79
107
|
Remaps a Direct Lake semantic model's SQL Endpoint connection to a new lakehouse/warehouse.
|
|
@@ -95,7 +123,14 @@ def update_direct_lake_model_connection(
|
|
|
95
123
|
The Fabric workspace name or ID used by the lakehouse/warehouse.
|
|
96
124
|
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
97
125
|
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
126
|
+
use_sql_endpoint : bool, default=True
|
|
127
|
+
If True, the SQL Endpoint will be used for the connection.
|
|
128
|
+
If False, Direct Lake over OneLake will be used.
|
|
98
129
|
"""
|
|
130
|
+
if use_sql_endpoint:
|
|
131
|
+
icons.sll_tags.append("UpdateDLConnection_SQL")
|
|
132
|
+
else:
|
|
133
|
+
icons.sll_tags.append("UpdateDLConnection_DLOL")
|
|
99
134
|
|
|
100
135
|
(workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
|
|
101
136
|
(dataset_name, dataset_id) = resolve_dataset_name_and_id(dataset, workspace_id)
|
|
@@ -119,12 +154,16 @@ def update_direct_lake_model_connection(
|
|
|
119
154
|
item=source, type=source_type, workspace=source_workspace
|
|
120
155
|
)
|
|
121
156
|
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
157
|
+
shared_expression = generate_shared_expression(
|
|
158
|
+
item_name=source_name,
|
|
159
|
+
item_type=source_type,
|
|
160
|
+
workspace=source_workspace,
|
|
161
|
+
use_sql_endpoint=use_sql_endpoint,
|
|
126
162
|
)
|
|
127
163
|
|
|
164
|
+
expression_dict = _get_direct_lake_expressions(dataset=dataset, workspace=workspace)
|
|
165
|
+
expressions = list(expression_dict.keys())
|
|
166
|
+
|
|
128
167
|
with connect_semantic_model(
|
|
129
168
|
dataset=dataset_id, readonly=False, workspace=workspace_id
|
|
130
169
|
) as tom:
|
|
@@ -134,8 +173,15 @@ def update_direct_lake_model_connection(
|
|
|
134
173
|
f"{icons.red_dot} The '{dataset_name}' semantic model within the '{workspace_name}' workspace is not in Direct Lake. This function is only applicable to Direct Lake semantic models."
|
|
135
174
|
)
|
|
136
175
|
|
|
137
|
-
|
|
176
|
+
# Update the single connection expression
|
|
177
|
+
if len(expressions) == 1:
|
|
178
|
+
expr = expressions[0]
|
|
179
|
+
tom.model.Expressions[expr].Expression = shared_expression
|
|
138
180
|
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
181
|
+
print(
|
|
182
|
+
f"{icons.green_dot} The expression in the '{dataset_name}' semantic model within the '{workspace_name}' workspace has been updated to point to the '{source}' {source_type.lower()} in the '{source_workspace}' workspace."
|
|
183
|
+
)
|
|
184
|
+
else:
|
|
185
|
+
print(
|
|
186
|
+
f"{icons.info} Multiple expressions found in the model. Please use the update_direct_lake_partition_entity function to update specific tables."
|
|
187
|
+
)
|
|
@@ -8,11 +8,13 @@ from sempy_labs._helper_functions import (
|
|
|
8
8
|
resolve_workspace_name_and_id,
|
|
9
9
|
resolve_workspace_name,
|
|
10
10
|
)
|
|
11
|
+
from sempy._utils._log import log
|
|
11
12
|
from typing import List, Optional, Union
|
|
12
13
|
import sempy_labs._icons as icons
|
|
13
14
|
from uuid import UUID
|
|
14
15
|
|
|
15
16
|
|
|
17
|
+
@log
|
|
16
18
|
def update_direct_lake_partition_entity(
|
|
17
19
|
dataset: str | UUID,
|
|
18
20
|
table_name: Union[str, List[str]],
|
|
@@ -96,6 +98,7 @@ def update_direct_lake_partition_entity(
|
|
|
96
98
|
)
|
|
97
99
|
|
|
98
100
|
|
|
101
|
+
@log
|
|
99
102
|
def add_table_to_direct_lake_semantic_model(
|
|
100
103
|
dataset: str | UUID,
|
|
101
104
|
table_name: str,
|
sempy_labs/graph/_groups.py
CHANGED
|
@@ -6,6 +6,7 @@ from sempy_labs._helper_functions import (
|
|
|
6
6
|
_create_dataframe,
|
|
7
7
|
_update_dataframe_datatypes,
|
|
8
8
|
)
|
|
9
|
+
from sempy._utils._log import log
|
|
9
10
|
import sempy_labs._icons as icons
|
|
10
11
|
from typing import List, Literal
|
|
11
12
|
|
|
@@ -38,6 +39,7 @@ def resolve_group_id(group: str | UUID) -> UUID:
|
|
|
38
39
|
return group_id
|
|
39
40
|
|
|
40
41
|
|
|
42
|
+
@log
|
|
41
43
|
def list_groups() -> pd.DataFrame:
|
|
42
44
|
"""
|
|
43
45
|
Shows a list of groups and their properties.
|
|
@@ -158,6 +160,7 @@ def _get_group(group_id: UUID) -> pd.DataFrame:
|
|
|
158
160
|
return df
|
|
159
161
|
|
|
160
162
|
|
|
163
|
+
@log
|
|
161
164
|
def list_group_members(group: str | UUID) -> pd.DataFrame:
|
|
162
165
|
"""
|
|
163
166
|
Shows a list of the members of a group.
|
|
@@ -217,6 +220,7 @@ def list_group_members(group: str | UUID) -> pd.DataFrame:
|
|
|
217
220
|
return df
|
|
218
221
|
|
|
219
222
|
|
|
223
|
+
@log
|
|
220
224
|
def list_group_owners(group: str | UUID) -> pd.DataFrame:
|
|
221
225
|
"""
|
|
222
226
|
Shows a list of the owners of a group.
|
|
@@ -332,6 +336,7 @@ def _base_add_to_group(
|
|
|
332
336
|
)
|
|
333
337
|
|
|
334
338
|
|
|
339
|
+
@log
|
|
335
340
|
def add_group_members(
|
|
336
341
|
group: str | UUID,
|
|
337
342
|
user: str | UUID | List[str | UUID],
|
|
@@ -376,6 +381,7 @@ def add_group_owners(
|
|
|
376
381
|
_base_add_to_group(group=group, object=user, object_type="owners")
|
|
377
382
|
|
|
378
383
|
|
|
384
|
+
@log
|
|
379
385
|
def renew_group(group: str | UUID):
|
|
380
386
|
"""
|
|
381
387
|
Renews the group.
|
sempy_labs/graph/_teams.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import pandas as pd
|
|
2
2
|
from uuid import UUID
|
|
3
|
+
from sempy._utils._log import log
|
|
3
4
|
from sempy_labs._helper_functions import (
|
|
4
5
|
_base_api,
|
|
5
6
|
_create_dataframe,
|
|
@@ -7,6 +8,7 @@ from sempy_labs._helper_functions import (
|
|
|
7
8
|
)
|
|
8
9
|
|
|
9
10
|
|
|
11
|
+
@log
|
|
10
12
|
def list_teams() -> pd.DataFrame:
|
|
11
13
|
"""
|
|
12
14
|
Shows a list of teams and their properties.
|