semantic-link-labs 0.9.1__py3-none-any.whl → 0.9.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of semantic-link-labs might be problematic. Click here for more details.
- {semantic_link_labs-0.9.1.dist-info → semantic_link_labs-0.9.3.dist-info}/METADATA +67 -8
- {semantic_link_labs-0.9.1.dist-info → semantic_link_labs-0.9.3.dist-info}/RECORD +87 -80
- sempy_labs/__init__.py +14 -12
- sempy_labs/_ai.py +8 -5
- sempy_labs/_capacities.py +120 -142
- sempy_labs/_capacity_migration.py +61 -94
- sempy_labs/_clear_cache.py +9 -8
- sempy_labs/_connections.py +107 -104
- sempy_labs/_data_pipelines.py +47 -49
- sempy_labs/_dataflows.py +45 -51
- sempy_labs/_dax.py +228 -6
- sempy_labs/_delta_analyzer.py +321 -0
- sempy_labs/_deployment_pipelines.py +72 -66
- sempy_labs/_environments.py +39 -36
- sempy_labs/_eventhouses.py +35 -35
- sempy_labs/_eventstreams.py +38 -39
- sempy_labs/_external_data_shares.py +29 -42
- sempy_labs/_gateways.py +103 -99
- sempy_labs/_generate_semantic_model.py +22 -30
- sempy_labs/_git.py +46 -66
- sempy_labs/_graphQL.py +95 -0
- sempy_labs/_helper_functions.py +227 -36
- sempy_labs/_job_scheduler.py +47 -59
- sempy_labs/_kql_databases.py +27 -34
- sempy_labs/_kql_querysets.py +23 -30
- sempy_labs/_list_functions.py +264 -167
- sempy_labs/_managed_private_endpoints.py +52 -47
- sempy_labs/_mirrored_databases.py +110 -134
- sempy_labs/_mirrored_warehouses.py +13 -13
- sempy_labs/_ml_experiments.py +36 -36
- sempy_labs/_ml_models.py +37 -38
- sempy_labs/_model_bpa.py +2 -2
- sempy_labs/_model_bpa_rules.py +8 -6
- sempy_labs/_model_dependencies.py +2 -0
- sempy_labs/_notebooks.py +28 -29
- sempy_labs/_one_lake_integration.py +2 -0
- sempy_labs/_query_scale_out.py +63 -81
- sempy_labs/_refresh_semantic_model.py +12 -14
- sempy_labs/_spark.py +54 -79
- sempy_labs/_sql.py +7 -11
- sempy_labs/_translations.py +2 -2
- sempy_labs/_vertipaq.py +11 -6
- sempy_labs/_warehouses.py +30 -33
- sempy_labs/_workloads.py +15 -20
- sempy_labs/_workspace_identity.py +13 -17
- sempy_labs/_workspaces.py +49 -48
- sempy_labs/admin/__init__.py +2 -0
- sempy_labs/admin/_basic_functions.py +244 -281
- sempy_labs/admin/_domains.py +186 -103
- sempy_labs/admin/_external_data_share.py +26 -31
- sempy_labs/admin/_git.py +17 -22
- sempy_labs/admin/_items.py +34 -48
- sempy_labs/admin/_scanner.py +61 -49
- sempy_labs/directlake/_directlake_schema_compare.py +2 -0
- sempy_labs/directlake/_dl_helper.py +10 -11
- sempy_labs/directlake/_generate_shared_expression.py +4 -5
- sempy_labs/directlake/_get_directlake_lakehouse.py +1 -0
- sempy_labs/directlake/_list_directlake_model_calc_tables.py +1 -0
- sempy_labs/directlake/_show_unsupported_directlake_objects.py +2 -0
- sempy_labs/directlake/_warm_cache.py +2 -0
- sempy_labs/graph/__init__.py +33 -0
- sempy_labs/graph/_groups.py +402 -0
- sempy_labs/graph/_teams.py +113 -0
- sempy_labs/graph/_users.py +191 -0
- sempy_labs/lakehouse/__init__.py +4 -0
- sempy_labs/lakehouse/_get_lakehouse_columns.py +12 -12
- sempy_labs/lakehouse/_get_lakehouse_tables.py +16 -22
- sempy_labs/lakehouse/_lakehouse.py +104 -7
- sempy_labs/lakehouse/_shortcuts.py +42 -20
- sempy_labs/migration/__init__.py +4 -0
- sempy_labs/migration/_direct_lake_to_import.py +66 -0
- sempy_labs/migration/_migrate_calctables_to_lakehouse.py +3 -2
- sempy_labs/migration/_migrate_calctables_to_semantic_model.py +1 -0
- sempy_labs/migration/_migrate_model_objects_to_semantic_model.py +1 -0
- sempy_labs/migration/_migrate_tables_columns_to_semantic_model.py +2 -0
- sempy_labs/migration/_refresh_calc_tables.py +2 -2
- sempy_labs/report/_download_report.py +8 -13
- sempy_labs/report/_generate_report.py +49 -46
- sempy_labs/report/_paginated.py +20 -26
- sempy_labs/report/_report_functions.py +52 -47
- sempy_labs/report/_report_list_functions.py +2 -0
- sempy_labs/report/_report_rebind.py +6 -10
- sempy_labs/report/_reportwrapper.py +187 -220
- sempy_labs/tom/_model.py +12 -6
- {semantic_link_labs-0.9.1.dist-info → semantic_link_labs-0.9.3.dist-info}/LICENSE +0 -0
- {semantic_link_labs-0.9.1.dist-info → semantic_link_labs-0.9.3.dist-info}/WHEEL +0 -0
- {semantic_link_labs-0.9.1.dist-info → semantic_link_labs-0.9.3.dist-info}/top_level.txt +0 -0
|
@@ -5,11 +5,12 @@ from sempy_labs._helper_functions import (
|
|
|
5
5
|
_get_partition_map,
|
|
6
6
|
_process_and_display_chart,
|
|
7
7
|
resolve_dataset_name_and_id,
|
|
8
|
+
_update_dataframe_datatypes,
|
|
9
|
+
_base_api,
|
|
8
10
|
)
|
|
9
11
|
from typing import Any, List, Optional, Union
|
|
10
12
|
from sempy._utils._log import log
|
|
11
13
|
import sempy_labs._icons as icons
|
|
12
|
-
from sempy.fabric.exceptions import FabricHTTPException
|
|
13
14
|
import pandas as pd
|
|
14
15
|
import warnings
|
|
15
16
|
import ipywidgets as widgets
|
|
@@ -311,14 +312,10 @@ def cancel_dataset_refresh(
|
|
|
311
312
|
|
|
312
313
|
request_id = rr_filt["Request Id"].iloc[0]
|
|
313
314
|
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
f"/v1.0/myorg/groups/{workspace_id}/datasets/{dataset_id}/refreshes/{request_id}"
|
|
315
|
+
_base_api(
|
|
316
|
+
request=f"/v1.0/myorg/groups/{workspace_id}/datasets/{dataset_id}/refreshes/{request_id}",
|
|
317
|
+
method="delete",
|
|
318
318
|
)
|
|
319
|
-
|
|
320
|
-
if response.status_code != 200:
|
|
321
|
-
raise FabricHTTPException(response)
|
|
322
319
|
print(
|
|
323
320
|
f"{icons.green_dot} The '{request_id}' refresh request for the '{dataset_name}' semantic model within the '{workspace_name}' workspace has been cancelled."
|
|
324
321
|
)
|
|
@@ -365,9 +362,8 @@ def get_semantic_model_refresh_history(
|
|
|
365
362
|
]
|
|
366
363
|
)
|
|
367
364
|
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
f"/v1.0/myorg/groups/{workspace_id}/datasets/{dataset_id}/refreshes"
|
|
365
|
+
response = _base_api(
|
|
366
|
+
request=f"/v1.0/myorg/groups/{workspace_id}/datasets/{dataset_id}/refreshes"
|
|
371
367
|
)
|
|
372
368
|
data = []
|
|
373
369
|
|
|
@@ -429,8 +425,10 @@ def get_semantic_model_refresh_history(
|
|
|
429
425
|
# df[date_cols] = df[date_cols].apply(pd.to_datetime)
|
|
430
426
|
|
|
431
427
|
if "Attempt Id" in df.columns:
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
428
|
+
column_map = {
|
|
429
|
+
"Attempt Id": "int",
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
_update_dataframe_datatypes(dataframe=df, column_map=column_map)
|
|
435
433
|
|
|
436
434
|
return df
|
sempy_labs/_spark.py
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
|
-
import sempy.fabric as fabric
|
|
2
1
|
import pandas as pd
|
|
3
2
|
import sempy_labs._icons as icons
|
|
4
3
|
from typing import Optional
|
|
5
4
|
from sempy_labs._helper_functions import (
|
|
6
5
|
resolve_workspace_name_and_id,
|
|
6
|
+
_update_dataframe_datatypes,
|
|
7
|
+
_base_api,
|
|
8
|
+
_create_dataframe,
|
|
7
9
|
)
|
|
8
|
-
from sempy.fabric.exceptions import FabricHTTPException
|
|
9
10
|
from uuid import UUID
|
|
10
11
|
|
|
11
12
|
|
|
@@ -30,26 +31,22 @@ def list_custom_pools(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
|
|
|
30
31
|
|
|
31
32
|
(workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
|
|
32
33
|
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
)
|
|
34
|
+
columns = {
|
|
35
|
+
"Custom Pool ID": "string",
|
|
36
|
+
"Custom Pool Name": "string",
|
|
37
|
+
"Type": "string",
|
|
38
|
+
"Node Family": "string",
|
|
39
|
+
"Node Size": "string",
|
|
40
|
+
"Auto Scale Enabled": "bool",
|
|
41
|
+
"Auto Scale Min Node Count": "int",
|
|
42
|
+
"Auto Scale Max Node Count": "int",
|
|
43
|
+
"Dynamic Executor Allocation Enabled": "bool",
|
|
44
|
+
"Dynamic Executor Allocation Min Executors": "int",
|
|
45
|
+
"Dynamic Executor Allocation Max Executors": "int",
|
|
46
|
+
}
|
|
47
|
+
df = _create_dataframe(columns=columns)
|
|
48
48
|
|
|
49
|
-
|
|
50
|
-
response = client.get(f"/v1/workspaces/{workspace_id}/spark/pools")
|
|
51
|
-
if response.status_code != 200:
|
|
52
|
-
raise FabricHTTPException(response)
|
|
49
|
+
response = _base_api(request=f"/v1/workspaces/{workspace_id}/spark/pools")
|
|
53
50
|
|
|
54
51
|
for i in response.json()["value"]:
|
|
55
52
|
|
|
@@ -71,17 +68,7 @@ def list_custom_pools(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
|
|
|
71
68
|
}
|
|
72
69
|
df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
|
|
73
70
|
|
|
74
|
-
|
|
75
|
-
int_cols = [
|
|
76
|
-
"Auto Scale Min Node Count",
|
|
77
|
-
"Auto Scale Max Node Count",
|
|
78
|
-
"Dynamic Executor Allocation Enabled",
|
|
79
|
-
"Dynamic Executor Allocation Min Executors",
|
|
80
|
-
"Dynamic Executor Allocation Max Executors",
|
|
81
|
-
]
|
|
82
|
-
|
|
83
|
-
df[bool_cols] = df[bool_cols].astype(bool)
|
|
84
|
-
df[int_cols] = df[int_cols].astype(int)
|
|
71
|
+
_update_dataframe_datatypes(dataframe=df, column_map=columns)
|
|
85
72
|
|
|
86
73
|
return df
|
|
87
74
|
|
|
@@ -131,7 +118,7 @@ def create_custom_pool(
|
|
|
131
118
|
|
|
132
119
|
(workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
|
|
133
120
|
|
|
134
|
-
|
|
121
|
+
payload = {
|
|
135
122
|
"name": pool_name,
|
|
136
123
|
"nodeFamily": node_family,
|
|
137
124
|
"nodeSize": node_size,
|
|
@@ -147,13 +134,12 @@ def create_custom_pool(
|
|
|
147
134
|
},
|
|
148
135
|
}
|
|
149
136
|
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
137
|
+
_base_api(
|
|
138
|
+
request=f"/v1/workspaces/{workspace_id}/spark/pools",
|
|
139
|
+
payload=payload,
|
|
140
|
+
method="post",
|
|
141
|
+
status_codes=201,
|
|
153
142
|
)
|
|
154
|
-
|
|
155
|
-
if response.status_code != 201:
|
|
156
|
-
raise FabricHTTPException(response)
|
|
157
143
|
print(
|
|
158
144
|
f"{icons.green_dot} The '{pool_name}' spark pool has been created within the '{workspace_name}' workspace."
|
|
159
145
|
)
|
|
@@ -239,7 +225,7 @@ def update_custom_pool(
|
|
|
239
225
|
if max_executors is None:
|
|
240
226
|
max_executors = int(df_pool["Max Executors"].iloc[0])
|
|
241
227
|
|
|
242
|
-
|
|
228
|
+
payload = {
|
|
243
229
|
"name": pool_name,
|
|
244
230
|
"nodeFamily": node_family,
|
|
245
231
|
"nodeSize": node_size,
|
|
@@ -255,13 +241,11 @@ def update_custom_pool(
|
|
|
255
241
|
},
|
|
256
242
|
}
|
|
257
243
|
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
244
|
+
_base_api(
|
|
245
|
+
request=f"/v1/workspaces/{workspace_id}/spark/pools",
|
|
246
|
+
payload=payload,
|
|
247
|
+
method="post",
|
|
261
248
|
)
|
|
262
|
-
|
|
263
|
-
if response.status_code != 200:
|
|
264
|
-
raise FabricHTTPException(response)
|
|
265
249
|
print(
|
|
266
250
|
f"{icons.green_dot} The '{pool_name}' spark pool within the '{workspace_name}' workspace has been updated."
|
|
267
251
|
)
|
|
@@ -288,17 +272,15 @@ def delete_custom_pool(pool_name: str, workspace: Optional[str | UUID] = None):
|
|
|
288
272
|
dfL = list_custom_pools(workspace=workspace_id)
|
|
289
273
|
dfL_filt = dfL[dfL["Custom Pool Name"] == pool_name]
|
|
290
274
|
|
|
291
|
-
if
|
|
275
|
+
if dfL_filt.empty:
|
|
292
276
|
raise ValueError(
|
|
293
277
|
f"{icons.red_dot} The '{pool_name}' custom pool does not exist within the '{workspace_name}' workspace."
|
|
294
278
|
)
|
|
295
279
|
pool_id = dfL_filt["Custom Pool ID"].iloc[0]
|
|
296
280
|
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
if response.status_code != 200:
|
|
301
|
-
raise FabricHTTPException(response)
|
|
281
|
+
_base_api(
|
|
282
|
+
request=f"/v1/workspaces/{workspace_id}/spark/pools/{pool_id}", method="delete"
|
|
283
|
+
)
|
|
302
284
|
print(
|
|
303
285
|
f"{icons.green_dot} The '{pool_name}' spark pool has been deleted from the '{workspace_name}' workspace."
|
|
304
286
|
)
|
|
@@ -343,10 +325,7 @@ def get_spark_settings(
|
|
|
343
325
|
]
|
|
344
326
|
)
|
|
345
327
|
|
|
346
|
-
|
|
347
|
-
response = client.get(f"/v1/workspaces/{workspace_id}/spark/settings")
|
|
348
|
-
if response.status_code != 200:
|
|
349
|
-
raise FabricHTTPException(response)
|
|
328
|
+
response = _base_api(request=f"/v1/workspaces/{workspace_id}/spark/settings")
|
|
350
329
|
|
|
351
330
|
i = response.json()
|
|
352
331
|
p = i.get("pool")
|
|
@@ -369,15 +348,13 @@ def get_spark_settings(
|
|
|
369
348
|
}
|
|
370
349
|
df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
|
|
371
350
|
|
|
372
|
-
|
|
373
|
-
"Automatic Log Enabled",
|
|
374
|
-
"High Concurrency Enabled",
|
|
375
|
-
"Customize Compute Enabled",
|
|
376
|
-
|
|
377
|
-
# int_cols = ["Max Node Count", "Max Executors"]
|
|
351
|
+
column_map = {
|
|
352
|
+
"Automatic Log Enabled": "bool",
|
|
353
|
+
"High Concurrency Enabled": "bool",
|
|
354
|
+
"Customize Compute Enabled": "bool",
|
|
355
|
+
}
|
|
378
356
|
|
|
379
|
-
df
|
|
380
|
-
# df[int_cols] = df[int_cols].astype(int)
|
|
357
|
+
_update_dataframe_datatypes(dataframe=df, column_map=column_map)
|
|
381
358
|
|
|
382
359
|
if return_dataframe:
|
|
383
360
|
return df
|
|
@@ -435,34 +412,32 @@ def update_spark_settings(
|
|
|
435
412
|
|
|
436
413
|
(workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
|
|
437
414
|
|
|
438
|
-
|
|
415
|
+
payload = get_spark_settings(workspace=workspace, return_dataframe=False)
|
|
439
416
|
|
|
440
417
|
if automatic_log_enabled is not None:
|
|
441
|
-
|
|
418
|
+
payload["automaticLog"]["enabled"] = automatic_log_enabled
|
|
442
419
|
if high_concurrency_enabled is not None:
|
|
443
|
-
|
|
420
|
+
payload["highConcurrency"][
|
|
444
421
|
"notebookInteractiveRunEnabled"
|
|
445
422
|
] = high_concurrency_enabled
|
|
446
423
|
if customize_compute_enabled is not None:
|
|
447
|
-
|
|
424
|
+
payload["pool"]["customizeComputeEnabled"] = customize_compute_enabled
|
|
448
425
|
if default_pool_name is not None:
|
|
449
|
-
|
|
426
|
+
payload["pool"]["defaultPool"]["name"] = default_pool_name
|
|
450
427
|
if max_node_count is not None:
|
|
451
|
-
|
|
428
|
+
payload["pool"]["starterPool"]["maxNodeCount"] = max_node_count
|
|
452
429
|
if max_executors is not None:
|
|
453
|
-
|
|
430
|
+
payload["pool"]["starterPool"]["maxExecutors"] = max_executors
|
|
454
431
|
if environment_name is not None:
|
|
455
|
-
|
|
432
|
+
payload["environment"]["name"] = environment_name
|
|
456
433
|
if runtime_version is not None:
|
|
457
|
-
|
|
434
|
+
payload["environment"]["runtimeVersion"] = runtime_version
|
|
458
435
|
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
436
|
+
_base_api(
|
|
437
|
+
request=f"/v1/workspaces/{workspace_id}/spark/settings",
|
|
438
|
+
payload=payload,
|
|
439
|
+
method="patch",
|
|
462
440
|
)
|
|
463
|
-
|
|
464
|
-
if response.status_code != 200:
|
|
465
|
-
raise FabricHTTPException(response)
|
|
466
441
|
print(
|
|
467
442
|
f"{icons.green_dot} The spark settings within the '{workspace_name}' workspace have been updated accordingly."
|
|
468
443
|
)
|
sempy_labs/_sql.py
CHANGED
|
@@ -1,14 +1,13 @@
|
|
|
1
|
-
import sempy.fabric as fabric
|
|
2
1
|
import pandas as pd
|
|
3
2
|
from typing import Optional, Union, List
|
|
4
3
|
from sempy._utils._log import log
|
|
5
4
|
import struct
|
|
6
5
|
from itertools import chain, repeat
|
|
7
|
-
from sempy.fabric.exceptions import FabricHTTPException
|
|
8
6
|
from sempy_labs._helper_functions import (
|
|
9
7
|
resolve_lakehouse_name_and_id,
|
|
10
8
|
resolve_item_name_and_id,
|
|
11
9
|
resolve_workspace_name_and_id,
|
|
10
|
+
_base_api,
|
|
12
11
|
)
|
|
13
12
|
from uuid import UUID
|
|
14
13
|
|
|
@@ -47,21 +46,18 @@ class ConnectBase:
|
|
|
47
46
|
|
|
48
47
|
# Resolve the appropriate ID and name (warehouse or lakehouse)
|
|
49
48
|
if endpoint_type == "warehouse":
|
|
50
|
-
(
|
|
49
|
+
(resource_name, resource_id) = resolve_item_name_and_id(
|
|
51
50
|
item=item, type=endpoint_type.capitalize(), workspace=workspace_id
|
|
52
51
|
)
|
|
53
52
|
else:
|
|
54
|
-
(
|
|
53
|
+
(resource_name, resource_id) = resolve_lakehouse_name_and_id(
|
|
55
54
|
lakehouse=item, workspace=workspace_id
|
|
56
55
|
)
|
|
57
56
|
|
|
58
57
|
# Get the TDS endpoint
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
f"v1/workspaces/{workspace_id}/{endpoint_type}s/{resource_id}"
|
|
58
|
+
response = _base_api(
|
|
59
|
+
request=f"v1/workspaces/{workspace_id}/{endpoint_type}s/{resource_id}"
|
|
62
60
|
)
|
|
63
|
-
if response.status_code != 200:
|
|
64
|
-
raise FabricHTTPException(response)
|
|
65
61
|
|
|
66
62
|
if endpoint_type == "warehouse":
|
|
67
63
|
tds_endpoint = response.json().get("properties", {}).get("connectionString")
|
|
@@ -150,7 +146,7 @@ class ConnectWarehouse(ConnectBase):
|
|
|
150
146
|
timeout: Optional[int] = None,
|
|
151
147
|
):
|
|
152
148
|
super().__init__(
|
|
153
|
-
|
|
149
|
+
item=warehouse,
|
|
154
150
|
workspace=workspace,
|
|
155
151
|
timeout=timeout,
|
|
156
152
|
endpoint_type="warehouse",
|
|
@@ -165,7 +161,7 @@ class ConnectLakehouse(ConnectBase):
|
|
|
165
161
|
timeout: Optional[int] = None,
|
|
166
162
|
):
|
|
167
163
|
super().__init__(
|
|
168
|
-
|
|
164
|
+
item=lakehouse,
|
|
169
165
|
workspace=workspace,
|
|
170
166
|
timeout=timeout,
|
|
171
167
|
endpoint_type="lakehouse",
|
sempy_labs/_translations.py
CHANGED
|
@@ -5,6 +5,7 @@ from sempy._utils._log import log
|
|
|
5
5
|
import sempy_labs._icons as icons
|
|
6
6
|
from sempy_labs._helper_functions import (
|
|
7
7
|
get_language_codes,
|
|
8
|
+
_create_spark_session,
|
|
8
9
|
)
|
|
9
10
|
from uuid import UUID
|
|
10
11
|
|
|
@@ -40,7 +41,6 @@ def translate_semantic_model(
|
|
|
40
41
|
|
|
41
42
|
from synapse.ml.services import Translate
|
|
42
43
|
from pyspark.sql.functions import col, flatten
|
|
43
|
-
from pyspark.sql import SparkSession
|
|
44
44
|
from sempy_labs.tom import connect_semantic_model
|
|
45
45
|
|
|
46
46
|
icons.sll_tags.append("TranslateSemanticModel")
|
|
@@ -145,7 +145,7 @@ def translate_semantic_model(
|
|
|
145
145
|
[df_prep, pd.DataFrame(new_data, index=[0])], ignore_index=True
|
|
146
146
|
)
|
|
147
147
|
|
|
148
|
-
spark =
|
|
148
|
+
spark = _create_spark_session()
|
|
149
149
|
df = spark.createDataFrame(df_prep)
|
|
150
150
|
|
|
151
151
|
columns = ["Name", "Description", "Display Folder"]
|
sempy_labs/_vertipaq.py
CHANGED
|
@@ -6,7 +6,6 @@ import os
|
|
|
6
6
|
import shutil
|
|
7
7
|
import datetime
|
|
8
8
|
import warnings
|
|
9
|
-
from pyspark.sql import SparkSession
|
|
10
9
|
from sempy_labs._helper_functions import (
|
|
11
10
|
format_dax_object_name,
|
|
12
11
|
resolve_lakehouse_name,
|
|
@@ -15,6 +14,7 @@ from sempy_labs._helper_functions import (
|
|
|
15
14
|
_get_column_aggregate,
|
|
16
15
|
resolve_workspace_name_and_id,
|
|
17
16
|
resolve_dataset_name_and_id,
|
|
17
|
+
_create_spark_session,
|
|
18
18
|
)
|
|
19
19
|
from sempy_labs._list_functions import list_relationships, list_tables
|
|
20
20
|
from sempy_labs.lakehouse import lakehouse_attached, get_lakehouse_tables
|
|
@@ -69,6 +69,8 @@ def vertipaq_analyzer(
|
|
|
69
69
|
(workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
|
|
70
70
|
(dataset_name, dataset_id) = resolve_dataset_name_and_id(dataset, workspace_id)
|
|
71
71
|
|
|
72
|
+
fabric.refresh_tom_cache(workspace=workspace)
|
|
73
|
+
|
|
72
74
|
vertipaq_map = {
|
|
73
75
|
"Model": {
|
|
74
76
|
"Dataset Name": [icons.data_type_string, icons.no_format],
|
|
@@ -163,9 +165,12 @@ def vertipaq_analyzer(
|
|
|
163
165
|
dfP = fabric.list_partitions(
|
|
164
166
|
dataset=dataset_id, extended=True, workspace=workspace_id
|
|
165
167
|
)
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
168
|
+
|
|
169
|
+
artifact_type = None
|
|
170
|
+
if is_direct_lake:
|
|
171
|
+
artifact_type, lakehouse_name, lakehouse_id, lakehouse_workspace_id = (
|
|
172
|
+
get_direct_lake_source(dataset=dataset_id, workspace=workspace_id)
|
|
173
|
+
)
|
|
169
174
|
|
|
170
175
|
dfR["Missing Rows"] = 0
|
|
171
176
|
dfR["Missing Rows"] = dfR["Missing Rows"].astype(int)
|
|
@@ -192,7 +197,7 @@ def vertipaq_analyzer(
|
|
|
192
197
|
)
|
|
193
198
|
|
|
194
199
|
sql_statements = []
|
|
195
|
-
spark =
|
|
200
|
+
spark = _create_spark_session()
|
|
196
201
|
# Loop through tables
|
|
197
202
|
for lakeTName in dfC_flt["Query"].unique():
|
|
198
203
|
query = "SELECT "
|
|
@@ -270,7 +275,7 @@ def vertipaq_analyzer(
|
|
|
270
275
|
dfR.rename(columns={"Source": "To Lake Column"}, inplace=True)
|
|
271
276
|
dfR.drop(columns=["Column Object"], inplace=True)
|
|
272
277
|
|
|
273
|
-
spark =
|
|
278
|
+
spark = _create_spark_session()
|
|
274
279
|
for i, r in dfR.iterrows():
|
|
275
280
|
fromTable = r["From Lake Table"]
|
|
276
281
|
fromColumn = r["From Lake Column"]
|
sempy_labs/_warehouses.py
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
import sempy.fabric as fabric
|
|
2
2
|
from sempy_labs._helper_functions import (
|
|
3
3
|
resolve_workspace_name_and_id,
|
|
4
|
-
|
|
5
|
-
|
|
4
|
+
_base_api,
|
|
5
|
+
_create_dataframe,
|
|
6
|
+
_update_dataframe_datatypes,
|
|
6
7
|
)
|
|
7
8
|
import pandas as pd
|
|
8
9
|
from typing import Optional
|
|
9
10
|
import sempy_labs._icons as icons
|
|
10
|
-
from sempy.fabric.exceptions import FabricHTTPException
|
|
11
11
|
from uuid import UUID
|
|
12
12
|
|
|
13
13
|
|
|
@@ -38,23 +38,24 @@ def create_warehouse(
|
|
|
38
38
|
|
|
39
39
|
(workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
|
|
40
40
|
|
|
41
|
-
|
|
41
|
+
payload = {"displayName": warehouse}
|
|
42
42
|
|
|
43
43
|
if description:
|
|
44
|
-
|
|
44
|
+
payload["description"] = description
|
|
45
45
|
if case_insensitive_collation:
|
|
46
|
-
|
|
47
|
-
|
|
46
|
+
payload.setdefault("creationPayload", {})
|
|
47
|
+
payload["creationPayload"][
|
|
48
48
|
"defaultCollation"
|
|
49
49
|
] = "Latin1_General_100_CI_AS_KS_WS_SC_UTF8"
|
|
50
50
|
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
51
|
+
_base_api(
|
|
52
|
+
request=f"/v1/workspaces/{workspace_id}/warehouses",
|
|
53
|
+
payload=payload,
|
|
54
|
+
method="post",
|
|
55
|
+
lro_return_status_code=True,
|
|
56
|
+
status_codes=[201, 202],
|
|
54
57
|
)
|
|
55
58
|
|
|
56
|
-
lro(client, response, status_codes=[201, 202])
|
|
57
|
-
|
|
58
59
|
print(
|
|
59
60
|
f"{icons.green_dot} The '{warehouse}' warehouse has been created within the '{workspace_name}' workspace."
|
|
60
61
|
)
|
|
@@ -79,25 +80,21 @@ def list_warehouses(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
|
|
|
79
80
|
A pandas dataframe showing the warehouses within a workspace.
|
|
80
81
|
"""
|
|
81
82
|
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
)
|
|
83
|
+
columns = {
|
|
84
|
+
"Warehouse Name": "string",
|
|
85
|
+
"Warehouse Id": "string",
|
|
86
|
+
"Description": "string",
|
|
87
|
+
"Connection Info": "string",
|
|
88
|
+
"Created Date": "datetime",
|
|
89
|
+
"Last Updated Time": "datetime",
|
|
90
|
+
}
|
|
91
|
+
df = _create_dataframe(columns=columns)
|
|
92
92
|
|
|
93
93
|
(workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
|
|
94
94
|
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
raise FabricHTTPException(response)
|
|
99
|
-
|
|
100
|
-
responses = pagination(client, response)
|
|
95
|
+
responses = _base_api(
|
|
96
|
+
request=f"/v1/workspaces/{workspace_id}/warehouses", uses_pagination=True
|
|
97
|
+
)
|
|
101
98
|
|
|
102
99
|
for r in responses:
|
|
103
100
|
for v in r.get("value", []):
|
|
@@ -113,6 +110,8 @@ def list_warehouses(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
|
|
|
113
110
|
}
|
|
114
111
|
df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
|
|
115
112
|
|
|
113
|
+
_update_dataframe_datatypes(dataframe=df, column_map=columns)
|
|
114
|
+
|
|
116
115
|
return df
|
|
117
116
|
|
|
118
117
|
|
|
@@ -138,11 +137,9 @@ def delete_warehouse(name: str, workspace: Optional[str | UUID] = None):
|
|
|
138
137
|
item_name=name, type="Warehouse", workspace=workspace_id
|
|
139
138
|
)
|
|
140
139
|
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
if response.status_code != 200:
|
|
145
|
-
raise FabricHTTPException(response)
|
|
140
|
+
_base_api(
|
|
141
|
+
request=f"/v1/workspaces/{workspace_id}/warehouses/{item_id}", method="delete"
|
|
142
|
+
)
|
|
146
143
|
|
|
147
144
|
print(
|
|
148
145
|
f"{icons.green_dot} The '{name}' warehouse within the '{workspace_name}' workspace has been deleted."
|
sempy_labs/_workloads.py
CHANGED
|
@@ -1,8 +1,11 @@
|
|
|
1
|
-
import sempy.fabric as fabric
|
|
2
1
|
import pandas as pd
|
|
3
2
|
from typing import Optional
|
|
4
3
|
import sempy_labs._icons as icons
|
|
5
|
-
from
|
|
4
|
+
from sempy_labs._helper_functions import (
|
|
5
|
+
_update_dataframe_datatypes,
|
|
6
|
+
_base_api,
|
|
7
|
+
_create_dataframe,
|
|
8
|
+
)
|
|
6
9
|
|
|
7
10
|
|
|
8
11
|
def list_workloads(capacity_name: str) -> pd.DataFrame:
|
|
@@ -25,17 +28,16 @@ def list_workloads(capacity_name: str) -> pd.DataFrame:
|
|
|
25
28
|
|
|
26
29
|
from sempy_labs._helper_functions import resolve_capacity_id
|
|
27
30
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
+
columns = {
|
|
32
|
+
"Workload Name": "string",
|
|
33
|
+
"State": "string",
|
|
34
|
+
"Max Memory Percentage Set By User": "int",
|
|
35
|
+
}
|
|
36
|
+
df = _create_dataframe(columns=columns)
|
|
31
37
|
|
|
32
38
|
capacity_id = resolve_capacity_id(capacity_name=capacity_name)
|
|
33
39
|
|
|
34
|
-
|
|
35
|
-
response = client.get(f"/v1.0/myorg/capacities/{capacity_id}/Workloads")
|
|
36
|
-
|
|
37
|
-
if response.status_code != 200:
|
|
38
|
-
raise FabricHTTPException(response)
|
|
40
|
+
response = _base_api(request=f"/v1.0/myorg/capacities/{capacity_id}/Workloads")
|
|
39
41
|
|
|
40
42
|
for v in response.json().get("value", []):
|
|
41
43
|
new_data = {
|
|
@@ -45,8 +47,7 @@ def list_workloads(capacity_name: str) -> pd.DataFrame:
|
|
|
45
47
|
}
|
|
46
48
|
df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
|
|
47
49
|
|
|
48
|
-
|
|
49
|
-
df[int_cols] = df[int_cols].astype(int)
|
|
50
|
+
_update_dataframe_datatypes(dataframe=df, column_map=columns)
|
|
50
51
|
|
|
51
52
|
return df
|
|
52
53
|
|
|
@@ -92,12 +93,9 @@ def patch_workload(
|
|
|
92
93
|
f"{icons.red_dot} Invalid max memory percentage. Must be a value between 0-100."
|
|
93
94
|
)
|
|
94
95
|
|
|
95
|
-
client = fabric.PowerBIRestClient()
|
|
96
96
|
url = f"/v1.0/myorg/capacities/{capacity_id}/Workloads/{workload_name}"
|
|
97
|
-
get_response = client.get(url)
|
|
98
|
-
if get_response.status_code != 200:
|
|
99
|
-
raise FabricHTTPException(get_response)
|
|
100
97
|
|
|
98
|
+
get_response = _base_api(request=url)
|
|
101
99
|
get_json = get_response.json().get("value")
|
|
102
100
|
current_state = get_json.get("state")
|
|
103
101
|
current_max_memory = get_json.get("maxMemoryPercentageSetByUser")
|
|
@@ -118,10 +116,7 @@ def patch_workload(
|
|
|
118
116
|
else:
|
|
119
117
|
payload["maxMemoryPercentageSetByUser"] = current_max_memory
|
|
120
118
|
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
if response.status_code != 200:
|
|
124
|
-
raise FabricHTTPException(response)
|
|
119
|
+
_base_api(request=url, method="patch", payload=payload)
|
|
125
120
|
|
|
126
121
|
print(
|
|
127
122
|
f"The '{workload_name}' workload within the '{capacity_name}' capacity has been updated accordingly."
|
|
@@ -1,11 +1,9 @@
|
|
|
1
|
-
import sempy.fabric as fabric
|
|
2
1
|
from sempy_labs._helper_functions import (
|
|
3
2
|
resolve_workspace_name_and_id,
|
|
4
|
-
|
|
3
|
+
_base_api,
|
|
5
4
|
)
|
|
6
5
|
from typing import Optional
|
|
7
6
|
import sempy_labs._icons as icons
|
|
8
|
-
from sempy.fabric.exceptions import FabricHTTPException
|
|
9
7
|
from uuid import UUID
|
|
10
8
|
|
|
11
9
|
|
|
@@ -25,13 +23,12 @@ def provision_workspace_identity(workspace: Optional[str | UUID] = None):
|
|
|
25
23
|
|
|
26
24
|
(workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
|
|
27
25
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
lro(client, response)
|
|
26
|
+
_base_api(
|
|
27
|
+
request=f"/v1/workspaces/{workspace_id}/provisionIdentity",
|
|
28
|
+
method="post",
|
|
29
|
+
lro_return_status_code=True,
|
|
30
|
+
status_codes=None,
|
|
31
|
+
)
|
|
35
32
|
|
|
36
33
|
print(
|
|
37
34
|
f"{icons.green_dot} A workspace identity has been provisioned for the '{workspace_name}' workspace."
|
|
@@ -54,13 +51,12 @@ def deprovision_workspace_identity(workspace: Optional[str | UUID] = None):
|
|
|
54
51
|
|
|
55
52
|
(workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
|
|
56
53
|
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
lro(client, response)
|
|
54
|
+
_base_api(
|
|
55
|
+
request=f"/v1/workspaces/{workspace_id}/deprovisionIdentity",
|
|
56
|
+
method="post",
|
|
57
|
+
lro_return_status_code=True,
|
|
58
|
+
status_codes=None,
|
|
59
|
+
)
|
|
64
60
|
|
|
65
61
|
print(
|
|
66
62
|
f"{icons.green_dot} The workspace identity has been deprovisioned from the '{workspace_name}' workspace."
|