semantic-link-labs 0.7.2__py3-none-any.whl → 0.7.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of semantic-link-labs might be problematic. Click here for more details.
- {semantic_link_labs-0.7.2.dist-info → semantic_link_labs-0.7.4.dist-info}/METADATA +15 -3
- semantic_link_labs-0.7.4.dist-info/RECORD +134 -0
- {semantic_link_labs-0.7.2.dist-info → semantic_link_labs-0.7.4.dist-info}/WHEEL +1 -1
- sempy_labs/__init__.py +120 -24
- sempy_labs/_bpa_translation/{_translations_am-ET.po → _model/_translations_am-ET.po} +22 -0
- sempy_labs/_bpa_translation/{_translations_ar-AE.po → _model/_translations_ar-AE.po} +24 -0
- sempy_labs/_bpa_translation/_model/_translations_bg-BG.po +938 -0
- sempy_labs/_bpa_translation/_model/_translations_ca-ES.po +934 -0
- sempy_labs/_bpa_translation/{_translations_cs-CZ.po → _model/_translations_cs-CZ.po} +179 -157
- sempy_labs/_bpa_translation/{_translations_da-DK.po → _model/_translations_da-DK.po} +24 -0
- sempy_labs/_bpa_translation/{_translations_de-DE.po → _model/_translations_de-DE.po} +77 -52
- sempy_labs/_bpa_translation/{_translations_el-GR.po → _model/_translations_el-GR.po} +25 -0
- sempy_labs/_bpa_translation/{_translations_es-ES.po → _model/_translations_es-ES.po} +67 -43
- sempy_labs/_bpa_translation/{_translations_fa-IR.po → _model/_translations_fa-IR.po} +24 -0
- sempy_labs/_bpa_translation/_model/_translations_fi-FI.po +915 -0
- sempy_labs/_bpa_translation/{_translations_fr-FR.po → _model/_translations_fr-FR.po} +83 -57
- sempy_labs/_bpa_translation/{_translations_ga-IE.po → _model/_translations_ga-IE.po} +25 -0
- sempy_labs/_bpa_translation/{_translations_he-IL.po → _model/_translations_he-IL.po} +23 -0
- sempy_labs/_bpa_translation/{_translations_hi-IN.po → _model/_translations_hi-IN.po} +24 -0
- sempy_labs/_bpa_translation/{_translations_hu-HU.po → _model/_translations_hu-HU.po} +25 -0
- sempy_labs/_bpa_translation/_model/_translations_id-ID.po +918 -0
- sempy_labs/_bpa_translation/{_translations_is-IS.po → _model/_translations_is-IS.po} +25 -0
- sempy_labs/_bpa_translation/{_translations_it-IT.po → _model/_translations_it-IT.po} +25 -0
- sempy_labs/_bpa_translation/{_translations_ja-JP.po → _model/_translations_ja-JP.po} +21 -0
- sempy_labs/_bpa_translation/_model/_translations_ko-KR.po +823 -0
- sempy_labs/_bpa_translation/_model/_translations_mt-MT.po +937 -0
- sempy_labs/_bpa_translation/{_translations_nl-NL.po → _model/_translations_nl-NL.po} +80 -56
- sempy_labs/_bpa_translation/{_translations_pl-PL.po → _model/_translations_pl-PL.po} +101 -76
- sempy_labs/_bpa_translation/{_translations_pt-BR.po → _model/_translations_pt-BR.po} +25 -0
- sempy_labs/_bpa_translation/{_translations_pt-PT.po → _model/_translations_pt-PT.po} +25 -0
- sempy_labs/_bpa_translation/_model/_translations_ro-RO.po +939 -0
- sempy_labs/_bpa_translation/{_translations_ru-RU.po → _model/_translations_ru-RU.po} +25 -0
- sempy_labs/_bpa_translation/_model/_translations_sk-SK.po +925 -0
- sempy_labs/_bpa_translation/_model/_translations_sl-SL.po +922 -0
- sempy_labs/_bpa_translation/_model/_translations_sv-SE.po +914 -0
- sempy_labs/_bpa_translation/{_translations_ta-IN.po → _model/_translations_ta-IN.po} +26 -0
- sempy_labs/_bpa_translation/{_translations_te-IN.po → _model/_translations_te-IN.po} +24 -0
- sempy_labs/_bpa_translation/{_translations_th-TH.po → _model/_translations_th-TH.po} +24 -0
- sempy_labs/_bpa_translation/_model/_translations_tr-TR.po +925 -0
- sempy_labs/_bpa_translation/_model/_translations_uk-UA.po +933 -0
- sempy_labs/_bpa_translation/{_translations_zh-CN.po → _model/_translations_zh-CN.po} +116 -97
- sempy_labs/_bpa_translation/{_translations_zu-ZA.po → _model/_translations_zu-ZA.po} +25 -0
- sempy_labs/_capacities.py +541 -0
- sempy_labs/_clear_cache.py +298 -3
- sempy_labs/_connections.py +138 -0
- sempy_labs/_dataflows.py +130 -0
- sempy_labs/_deployment_pipelines.py +171 -0
- sempy_labs/_environments.py +156 -0
- sempy_labs/_generate_semantic_model.py +148 -27
- sempy_labs/_git.py +380 -0
- sempy_labs/_helper_functions.py +203 -8
- sempy_labs/_icons.py +43 -0
- sempy_labs/_list_functions.py +170 -1012
- sempy_labs/_model_bpa.py +90 -112
- sempy_labs/_model_bpa_bulk.py +3 -1
- sempy_labs/_model_bpa_rules.py +788 -800
- sempy_labs/_notebooks.py +143 -0
- sempy_labs/_query_scale_out.py +28 -7
- sempy_labs/_spark.py +465 -0
- sempy_labs/_sql.py +120 -0
- sempy_labs/_translations.py +3 -1
- sempy_labs/_vertipaq.py +160 -99
- sempy_labs/_workspace_identity.py +66 -0
- sempy_labs/_workspaces.py +294 -0
- sempy_labs/directlake/__init__.py +2 -0
- sempy_labs/directlake/_directlake_schema_compare.py +1 -2
- sempy_labs/directlake/_directlake_schema_sync.py +1 -2
- sempy_labs/directlake/_dl_helper.py +4 -7
- sempy_labs/directlake/_generate_shared_expression.py +85 -0
- sempy_labs/directlake/_show_unsupported_directlake_objects.py +1 -2
- sempy_labs/lakehouse/_get_lakehouse_tables.py +7 -3
- sempy_labs/migration/_migrate_calctables_to_lakehouse.py +5 -0
- sempy_labs/migration/_migrate_calctables_to_semantic_model.py +5 -0
- sempy_labs/migration/_migrate_model_objects_to_semantic_model.py +6 -2
- sempy_labs/migration/_migrate_tables_columns_to_semantic_model.py +6 -5
- sempy_labs/migration/_migration_validation.py +6 -0
- sempy_labs/report/_report_functions.py +21 -42
- sempy_labs/report/_report_rebind.py +5 -0
- sempy_labs/tom/_model.py +95 -52
- semantic_link_labs-0.7.2.dist-info/RECORD +0 -111
- {semantic_link_labs-0.7.2.dist-info → semantic_link_labs-0.7.4.dist-info}/LICENSE +0 -0
- {semantic_link_labs-0.7.2.dist-info → semantic_link_labs-0.7.4.dist-info}/top_level.txt +0 -0
sempy_labs/_notebooks.py
ADDED
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
import sempy.fabric as fabric
|
|
2
|
+
import pandas as pd
|
|
3
|
+
import sempy_labs._icons as icons
|
|
4
|
+
from typing import Optional
|
|
5
|
+
import base64
|
|
6
|
+
import requests
|
|
7
|
+
from sempy_labs._helper_functions import (
|
|
8
|
+
resolve_workspace_name_and_id,
|
|
9
|
+
lro,
|
|
10
|
+
_decode_b64,
|
|
11
|
+
)
|
|
12
|
+
from sempy.fabric.exceptions import FabricHTTPException
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def get_notebook_definition(
|
|
16
|
+
notebook_name: str, workspace: Optional[str] = None, decode: Optional[bool] = True
|
|
17
|
+
):
|
|
18
|
+
"""
|
|
19
|
+
Obtains the notebook definition.
|
|
20
|
+
|
|
21
|
+
Parameters
|
|
22
|
+
----------
|
|
23
|
+
notebook_name : str
|
|
24
|
+
The name of the notebook.
|
|
25
|
+
workspace : str, default=None
|
|
26
|
+
The name of the workspace.
|
|
27
|
+
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
28
|
+
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
29
|
+
decode : bool, default=True
|
|
30
|
+
If True, decodes the notebook definition file into .ipynb format.
|
|
31
|
+
If False, obtains the notebook definition file in base64 format.
|
|
32
|
+
|
|
33
|
+
Returns
|
|
34
|
+
-------
|
|
35
|
+
ipynb
|
|
36
|
+
The notebook definition.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
(workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
|
|
40
|
+
|
|
41
|
+
dfI = fabric.list_items(workspace=workspace, type="Notebook")
|
|
42
|
+
dfI_filt = dfI[dfI["Display Name"] == notebook_name]
|
|
43
|
+
|
|
44
|
+
if len(dfI_filt) == 0:
|
|
45
|
+
raise ValueError(
|
|
46
|
+
f"{icons.red_dot} The '{notebook_name}' notebook does not exist within the '{workspace}' workspace."
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
notebook_id = dfI_filt["Id"].iloc[0]
|
|
50
|
+
client = fabric.FabricRestClient()
|
|
51
|
+
response = client.post(
|
|
52
|
+
f"v1/workspaces/{workspace_id}/notebooks/{notebook_id}/getDefinition",
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
result = lro(client, response).json()
|
|
56
|
+
df_items = pd.json_normalize(result["definition"]["parts"])
|
|
57
|
+
df_items_filt = df_items[df_items["path"] == "notebook-content.py"]
|
|
58
|
+
payload = df_items_filt["payload"].iloc[0]
|
|
59
|
+
|
|
60
|
+
if decode:
|
|
61
|
+
result = _decode_b64(payload)
|
|
62
|
+
else:
|
|
63
|
+
result = payload
|
|
64
|
+
|
|
65
|
+
return result
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def import_notebook_from_web(
|
|
69
|
+
notebook_name: str,
|
|
70
|
+
url: str,
|
|
71
|
+
description: Optional[str] = None,
|
|
72
|
+
workspace: Optional[str] = None,
|
|
73
|
+
):
|
|
74
|
+
"""
|
|
75
|
+
Creates a new notebook within a workspace based on a Jupyter notebook hosted in the web.
|
|
76
|
+
|
|
77
|
+
Note: When specifying a notebook from GitHub, please use the raw file path. Note that if the non-raw file path is specified, the url will be
|
|
78
|
+
converted to the raw URL as the raw URL is needed to obtain the notebook content.
|
|
79
|
+
|
|
80
|
+
Parameters
|
|
81
|
+
----------
|
|
82
|
+
notebook_name : str
|
|
83
|
+
The name of the notebook to be created.
|
|
84
|
+
url : str
|
|
85
|
+
The url of the Jupyter Notebook (.ipynb)
|
|
86
|
+
description : str, default=None
|
|
87
|
+
The description of the notebook.
|
|
88
|
+
Defaults to None which does not place a description.
|
|
89
|
+
workspace : str, default=None
|
|
90
|
+
The name of the workspace.
|
|
91
|
+
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
92
|
+
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
93
|
+
|
|
94
|
+
Returns
|
|
95
|
+
-------
|
|
96
|
+
"""
|
|
97
|
+
|
|
98
|
+
(workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
|
|
99
|
+
client = fabric.FabricRestClient()
|
|
100
|
+
dfI = fabric.list_items(workspace=workspace, type="Notebook")
|
|
101
|
+
dfI_filt = dfI[dfI["Display Name"] == notebook_name]
|
|
102
|
+
if len(dfI_filt) > 0:
|
|
103
|
+
raise ValueError(
|
|
104
|
+
f"{icons.red_dot} The '{notebook_name}' already exists within the '{workspace}' workspace."
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
# Fix links to go to the raw github file
|
|
108
|
+
starting_text = "https://github.com/"
|
|
109
|
+
starting_text_len = len(starting_text)
|
|
110
|
+
if url.startswith(starting_text):
|
|
111
|
+
url = f"https://raw.githubusercontent.com/{url[starting_text_len:]}".replace(
|
|
112
|
+
"/blob/", "/"
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
response = requests.get(url)
|
|
116
|
+
if response.status_code != 200:
|
|
117
|
+
raise FabricHTTPException(response)
|
|
118
|
+
file_content = response.content
|
|
119
|
+
notebook_payload = base64.b64encode(file_content)
|
|
120
|
+
|
|
121
|
+
request_body = {
|
|
122
|
+
"displayName": notebook_name,
|
|
123
|
+
"definition": {
|
|
124
|
+
"format": "ipynb",
|
|
125
|
+
"parts": [
|
|
126
|
+
{
|
|
127
|
+
"path": "notebook-content.py",
|
|
128
|
+
"payload": notebook_payload,
|
|
129
|
+
"payloadType": "InlineBase64",
|
|
130
|
+
}
|
|
131
|
+
],
|
|
132
|
+
},
|
|
133
|
+
}
|
|
134
|
+
if description is not None:
|
|
135
|
+
request_body["description"] = description
|
|
136
|
+
|
|
137
|
+
response = client.post(f"v1/workspaces/{workspace_id}/notebooks", json=request_body)
|
|
138
|
+
|
|
139
|
+
lro(client, response, status_codes=[201, 202])
|
|
140
|
+
|
|
141
|
+
print(
|
|
142
|
+
f"{icons.green_dot} The '{notebook_name}' notebook was created within the '{workspace}' workspace."
|
|
143
|
+
)
|
sempy_labs/_query_scale_out.py
CHANGED
|
@@ -293,13 +293,24 @@ def set_semantic_model_storage_format(
|
|
|
293
293
|
f"{icons.red_dot} Invalid storage format value. Valid options: {storageFormats}."
|
|
294
294
|
)
|
|
295
295
|
|
|
296
|
+
dfL = list_qso_settings(dataset=dataset, workspace=workspace)
|
|
297
|
+
current_storage_format = dfL["Storage Mode"].iloc[0]
|
|
298
|
+
|
|
299
|
+
if current_storage_format == storage_format:
|
|
300
|
+
print(
|
|
301
|
+
f"{icons.info} The '{dataset}' semantic model within the '{workspace}' workspace is already set to '{storage_format.lower()}' storage format."
|
|
302
|
+
)
|
|
303
|
+
return
|
|
304
|
+
|
|
296
305
|
client = fabric.PowerBIRestClient()
|
|
297
306
|
response = client.patch(
|
|
298
307
|
f"/v1.0/myorg/groups/{workspace_id}/datasets/{dataset_id}", json=request_body
|
|
299
308
|
)
|
|
300
309
|
if response.status_code != 200:
|
|
301
310
|
raise FabricHTTPException(response)
|
|
302
|
-
print(
|
|
311
|
+
print(
|
|
312
|
+
f"{icons.green_dot} The semantic model storage format for the '{dataset}' semantic model within the '{workspace}' workspace has been set to '{storage_format}'."
|
|
313
|
+
)
|
|
303
314
|
|
|
304
315
|
|
|
305
316
|
def list_qso_settings(
|
|
@@ -383,10 +394,6 @@ def set_workspace_default_storage_format(
|
|
|
383
394
|
The Fabric workspace name.
|
|
384
395
|
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
385
396
|
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
386
|
-
|
|
387
|
-
Returns
|
|
388
|
-
-------
|
|
389
|
-
|
|
390
397
|
"""
|
|
391
398
|
|
|
392
399
|
# https://learn.microsoft.com/en-us/rest/api/power-bi/groups/update-group#defaultdatasetstorageformat
|
|
@@ -396,19 +403,33 @@ def set_workspace_default_storage_format(
|
|
|
396
403
|
storage_format = storage_format.capitalize()
|
|
397
404
|
|
|
398
405
|
if storage_format not in storageFormats:
|
|
399
|
-
|
|
406
|
+
raise ValueError(
|
|
400
407
|
f"{icons.red_dot} Invalid storage format. Please choose from these options: {storageFormats}."
|
|
401
408
|
)
|
|
402
409
|
|
|
403
410
|
(workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
|
|
404
411
|
|
|
405
|
-
|
|
412
|
+
# Check current storage format
|
|
413
|
+
dfW = fabric.list_workspaces(filter=f"name eq '{workspace}'")
|
|
414
|
+
if len(dfW) == 0:
|
|
415
|
+
raise ValueError()
|
|
416
|
+
current_storage_format = dfW['Default Dataset Storage Format'].iloc[0]
|
|
417
|
+
|
|
418
|
+
if current_storage_format == storage_format:
|
|
419
|
+
print(f"{icons.info} The '{workspace}' is already set to a default storage format of '{current_storage_format}'.")
|
|
420
|
+
return
|
|
421
|
+
|
|
422
|
+
request_body = {
|
|
423
|
+
"name": workspace,
|
|
424
|
+
"defaultDatasetStorageFormat": storage_format,
|
|
425
|
+
}
|
|
406
426
|
|
|
407
427
|
client = fabric.PowerBIRestClient()
|
|
408
428
|
response = client.patch(f"/v1.0/myorg/groups/{workspace_id}", json=request_body)
|
|
409
429
|
|
|
410
430
|
if response.status_code != 200:
|
|
411
431
|
raise FabricHTTPException(response)
|
|
432
|
+
|
|
412
433
|
print(
|
|
413
434
|
f"{icons.green_dot} The default storage format for the '{workspace}' workspace has been updated to '{storage_format}."
|
|
414
435
|
)
|
sempy_labs/_spark.py
ADDED
|
@@ -0,0 +1,465 @@
|
|
|
1
|
+
import sempy.fabric as fabric
|
|
2
|
+
import pandas as pd
|
|
3
|
+
import sempy_labs._icons as icons
|
|
4
|
+
from typing import Optional
|
|
5
|
+
from sempy_labs._helper_functions import (
|
|
6
|
+
resolve_workspace_name_and_id,
|
|
7
|
+
)
|
|
8
|
+
from sempy.fabric.exceptions import FabricHTTPException
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def list_custom_pools(workspace: Optional[str] = None) -> pd.DataFrame:
|
|
12
|
+
"""
|
|
13
|
+
Lists all `custom pools <https://learn.microsoft.com/fabric/data-engineering/create-custom-spark-pools>`_ within a workspace.
|
|
14
|
+
|
|
15
|
+
Parameters
|
|
16
|
+
----------
|
|
17
|
+
workspace : str, default=None
|
|
18
|
+
The name of the Fabric workspace.
|
|
19
|
+
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
20
|
+
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
21
|
+
|
|
22
|
+
Returns
|
|
23
|
+
-------
|
|
24
|
+
pandas.DataFrame
|
|
25
|
+
A pandas dataframe showing all the custom pools within the Fabric workspace.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
# https://learn.microsoft.com/rest/api/fabric/spark/custom-pools/list-workspace-custom-pools
|
|
29
|
+
(workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
|
|
30
|
+
|
|
31
|
+
df = pd.DataFrame(
|
|
32
|
+
columns=[
|
|
33
|
+
"Custom Pool ID",
|
|
34
|
+
"Custom Pool Name",
|
|
35
|
+
"Type",
|
|
36
|
+
"Node Family",
|
|
37
|
+
"Node Size",
|
|
38
|
+
"Auto Scale Enabled",
|
|
39
|
+
"Auto Scale Min Node Count",
|
|
40
|
+
"Auto Scale Max Node Count",
|
|
41
|
+
"Dynamic Executor Allocation Enabled",
|
|
42
|
+
"Dynamic Executor Allocation Min Executors",
|
|
43
|
+
"Dynamic Executor Allocation Max Executors",
|
|
44
|
+
]
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
client = fabric.FabricRestClient()
|
|
48
|
+
response = client.get(f"/v1/workspaces/{workspace_id}/spark/pools")
|
|
49
|
+
if response.status_code != 200:
|
|
50
|
+
raise FabricHTTPException(response)
|
|
51
|
+
|
|
52
|
+
for i in response.json()["value"]:
|
|
53
|
+
|
|
54
|
+
aScale = i.get("autoScale", {})
|
|
55
|
+
d = i.get("dynamicExecutorAllocation", {})
|
|
56
|
+
|
|
57
|
+
new_data = {
|
|
58
|
+
"Custom Pool ID": i.get("id"),
|
|
59
|
+
"Custom Pool Name": i.get("name"),
|
|
60
|
+
"Type": i.get("type"),
|
|
61
|
+
"Node Family": i.get("nodeFamily"),
|
|
62
|
+
"Node Size": i.get("nodeSize"),
|
|
63
|
+
"Auto Scale Enabled": aScale.get("enabled"),
|
|
64
|
+
"Auto Scale Min Node Count": aScale.get("minNodeCount"),
|
|
65
|
+
"Auto Scale Max Node Count": aScale.get("maxNodeCount"),
|
|
66
|
+
"Dynamic Executor Allocation Enabled": d.get("enabled"),
|
|
67
|
+
"Dynamic Executor Allocation Min Executors": d.get("minExecutors"),
|
|
68
|
+
"Dynamic Executor Allocation Max Executors": d.get("maxExecutors"),
|
|
69
|
+
}
|
|
70
|
+
df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
|
|
71
|
+
|
|
72
|
+
bool_cols = ["Auto Scale Enabled", "Dynamic Executor Allocation Enabled"]
|
|
73
|
+
int_cols = [
|
|
74
|
+
"Auto Scale Min Node Count",
|
|
75
|
+
"Auto Scale Max Node Count",
|
|
76
|
+
"Dynamic Executor Allocation Enabled",
|
|
77
|
+
"Dynamic Executor Allocation Min Executors",
|
|
78
|
+
"Dynamic Executor Allocation Max Executors",
|
|
79
|
+
]
|
|
80
|
+
|
|
81
|
+
df[bool_cols] = df[bool_cols].astype(bool)
|
|
82
|
+
df[int_cols] = df[int_cols].astype(int)
|
|
83
|
+
|
|
84
|
+
return df
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def create_custom_pool(
|
|
88
|
+
pool_name: str,
|
|
89
|
+
node_size: str,
|
|
90
|
+
min_node_count: int,
|
|
91
|
+
max_node_count: int,
|
|
92
|
+
min_executors: int,
|
|
93
|
+
max_executors: int,
|
|
94
|
+
node_family: Optional[str] = "MemoryOptimized",
|
|
95
|
+
auto_scale_enabled: Optional[bool] = True,
|
|
96
|
+
dynamic_executor_allocation_enabled: Optional[bool] = True,
|
|
97
|
+
workspace: Optional[str] = None,
|
|
98
|
+
):
|
|
99
|
+
"""
|
|
100
|
+
Creates a `custom pool <https://learn.microsoft.com/fabric/data-engineering/create-custom-spark-pools>`_ within a workspace.
|
|
101
|
+
|
|
102
|
+
Parameters
|
|
103
|
+
----------
|
|
104
|
+
pool_name : str
|
|
105
|
+
The custom pool name.
|
|
106
|
+
node_size : str
|
|
107
|
+
The `node size <https://learn.microsoft.com/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#nodesize>`_.
|
|
108
|
+
min_node_count : int
|
|
109
|
+
The `minimum node count <https://learn.microsoft.com/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#autoscaleproperties>`_.
|
|
110
|
+
max_node_count : int
|
|
111
|
+
The `maximum node count <https://learn.microsoft.com/en-us/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#autoscaleproperties>`_.
|
|
112
|
+
min_executors : int
|
|
113
|
+
The `minimum executors <https://learn.microsoft.com/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#dynamicexecutorallocationproperties>`_.
|
|
114
|
+
max_executors : int
|
|
115
|
+
The `maximum executors <https://learn.microsoft.com/en-us/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#dynamicexecutorallocationproperties>`_.
|
|
116
|
+
node_family : str, default='MemoryOptimized'
|
|
117
|
+
The `node family <https://learn.microsoft.com/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#nodefamily>`_.
|
|
118
|
+
auto_scale_enabled : bool, default=True
|
|
119
|
+
The status of `auto scale <https://learn.microsoft.com/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#autoscaleproperties>`_.
|
|
120
|
+
dynamic_executor_allocation_enabled : bool, default=True
|
|
121
|
+
The status of the `dynamic executor allocation <https://learn.microsoft.com/en-us/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#dynamicexecutorallocationproperties>`_.
|
|
122
|
+
workspace : str, default=None
|
|
123
|
+
The name of the Fabric workspace.
|
|
124
|
+
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
125
|
+
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
126
|
+
"""
|
|
127
|
+
|
|
128
|
+
# https://learn.microsoft.com/en-us/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool
|
|
129
|
+
(workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
|
|
130
|
+
|
|
131
|
+
request_body = {
|
|
132
|
+
"name": pool_name,
|
|
133
|
+
"nodeFamily": node_family,
|
|
134
|
+
"nodeSize": node_size,
|
|
135
|
+
"autoScale": {
|
|
136
|
+
"enabled": auto_scale_enabled,
|
|
137
|
+
"minNodeCount": min_node_count,
|
|
138
|
+
"maxNodeCount": max_node_count,
|
|
139
|
+
},
|
|
140
|
+
"dynamicExecutorAllocation": {
|
|
141
|
+
"enabled": dynamic_executor_allocation_enabled,
|
|
142
|
+
"minExecutors": min_executors,
|
|
143
|
+
"maxExecutors": max_executors,
|
|
144
|
+
},
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
client = fabric.FabricRestClient()
|
|
148
|
+
response = client.post(
|
|
149
|
+
f"/v1/workspaces/{workspace_id}/spark/pools", json=request_body
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
if response.status_code != 201:
|
|
153
|
+
raise FabricHTTPException(response)
|
|
154
|
+
print(
|
|
155
|
+
f"{icons.green_dot} The '{pool_name}' spark pool has been created within the '{workspace}' workspace."
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def update_custom_pool(
|
|
160
|
+
pool_name: str,
|
|
161
|
+
node_size: Optional[str] = None,
|
|
162
|
+
min_node_count: Optional[int] = None,
|
|
163
|
+
max_node_count: Optional[int] = None,
|
|
164
|
+
min_executors: Optional[int] = None,
|
|
165
|
+
max_executors: Optional[int] = None,
|
|
166
|
+
node_family: Optional[str] = None,
|
|
167
|
+
auto_scale_enabled: Optional[bool] = None,
|
|
168
|
+
dynamic_executor_allocation_enabled: Optional[bool] = None,
|
|
169
|
+
workspace: Optional[str] = None,
|
|
170
|
+
):
|
|
171
|
+
"""
|
|
172
|
+
Updates the properties of a `custom pool <https://learn.microsoft.com/fabric/data-engineering/create-custom-spark-pools>`_ within a workspace.
|
|
173
|
+
|
|
174
|
+
Parameters
|
|
175
|
+
----------
|
|
176
|
+
pool_name : str
|
|
177
|
+
The custom pool name.
|
|
178
|
+
node_size : str, default=None
|
|
179
|
+
The `node size <https://learn.microsoft.com/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#nodesize>`_.
|
|
180
|
+
Defaults to None which keeps the existing property setting.
|
|
181
|
+
min_node_count : int, default=None
|
|
182
|
+
The `minimum node count <https://learn.microsoft.com/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#autoscaleproperties>`_.
|
|
183
|
+
Defaults to None which keeps the existing property setting.
|
|
184
|
+
max_node_count : int, default=None
|
|
185
|
+
The `maximum node count <https://learn.microsoft.com/en-us/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#autoscaleproperties>`_.
|
|
186
|
+
Defaults to None which keeps the existing property setting.
|
|
187
|
+
min_executors : int, default=None
|
|
188
|
+
The `minimum executors <https://learn.microsoft.com/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#dynamicexecutorallocationproperties>`_.
|
|
189
|
+
Defaults to None which keeps the existing property setting.
|
|
190
|
+
max_executors : int, default=None
|
|
191
|
+
The `maximum executors <https://learn.microsoft.com/en-us/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#dynamicexecutorallocationproperties>`_.
|
|
192
|
+
Defaults to None which keeps the existing property setting.
|
|
193
|
+
node_family : str, default=None
|
|
194
|
+
The `node family <https://learn.microsoft.com/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#nodefamily>`_.
|
|
195
|
+
Defaults to None which keeps the existing property setting.
|
|
196
|
+
auto_scale_enabled : bool, default=None
|
|
197
|
+
The status of `auto scale <https://learn.microsoft.com/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#autoscaleproperties>`_.
|
|
198
|
+
Defaults to None which keeps the existing property setting.
|
|
199
|
+
dynamic_executor_allocation_enabled : bool, default=None
|
|
200
|
+
The status of the `dynamic executor allocation <https://learn.microsoft.com/en-us/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#dynamicexecutorallocationproperties>`_.
|
|
201
|
+
Defaults to None which keeps the existing property setting.
|
|
202
|
+
workspace : str, default=None
|
|
203
|
+
The name of the Fabric workspace.
|
|
204
|
+
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
205
|
+
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
206
|
+
"""
|
|
207
|
+
|
|
208
|
+
# https://learn.microsoft.com/en-us/rest/api/fabric/spark/custom-pools/update-workspace-custom-pool?tabs=HTTP
|
|
209
|
+
(workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
|
|
210
|
+
|
|
211
|
+
df = list_custom_pools(workspace=workspace)
|
|
212
|
+
df_pool = df[df["Custom Pool Name"] == pool_name]
|
|
213
|
+
|
|
214
|
+
if len(df_pool) == 0:
|
|
215
|
+
raise ValueError(
|
|
216
|
+
f"{icons.red_dot} The '{pool_name}' custom pool does not exist within the '{workspace}'. Please choose a valid custom pool."
|
|
217
|
+
)
|
|
218
|
+
|
|
219
|
+
if node_family is None:
|
|
220
|
+
node_family = df_pool["Node Family"].iloc[0]
|
|
221
|
+
if node_size is None:
|
|
222
|
+
node_size = df_pool["Node Size"].iloc[0]
|
|
223
|
+
if auto_scale_enabled is None:
|
|
224
|
+
auto_scale_enabled = bool(df_pool["Auto Scale Enabled"].iloc[0])
|
|
225
|
+
if min_node_count is None:
|
|
226
|
+
min_node_count = int(df_pool["Min Node Count"].iloc[0])
|
|
227
|
+
if max_node_count is None:
|
|
228
|
+
max_node_count = int(df_pool["Max Node Count"].iloc[0])
|
|
229
|
+
if dynamic_executor_allocation_enabled is None:
|
|
230
|
+
dynamic_executor_allocation_enabled = bool(
|
|
231
|
+
df_pool["Dynami Executor Allocation Enabled"].iloc[0]
|
|
232
|
+
)
|
|
233
|
+
if min_executors is None:
|
|
234
|
+
min_executors = int(df_pool["Min Executors"].iloc[0])
|
|
235
|
+
if max_executors is None:
|
|
236
|
+
max_executors = int(df_pool["Max Executors"].iloc[0])
|
|
237
|
+
|
|
238
|
+
request_body = {
|
|
239
|
+
"name": pool_name,
|
|
240
|
+
"nodeFamily": node_family,
|
|
241
|
+
"nodeSize": node_size,
|
|
242
|
+
"autoScale": {
|
|
243
|
+
"enabled": auto_scale_enabled,
|
|
244
|
+
"minNodeCount": min_node_count,
|
|
245
|
+
"maxNodeCount": max_node_count,
|
|
246
|
+
},
|
|
247
|
+
"dynamicExecutorAllocation": {
|
|
248
|
+
"enabled": dynamic_executor_allocation_enabled,
|
|
249
|
+
"minExecutors": min_executors,
|
|
250
|
+
"maxExecutors": max_executors,
|
|
251
|
+
},
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
client = fabric.FabricRestClient()
|
|
255
|
+
response = client.post(
|
|
256
|
+
f"/v1/workspaces/{workspace_id}/spark/pools", json=request_body
|
|
257
|
+
)
|
|
258
|
+
|
|
259
|
+
if response.status_code != 200:
|
|
260
|
+
raise FabricHTTPException(response)
|
|
261
|
+
print(
|
|
262
|
+
f"{icons.green_dot} The '{pool_name}' spark pool within the '{workspace}' workspace has been updated."
|
|
263
|
+
)
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def delete_custom_pool(pool_name: str, workspace: Optional[str] = None):
|
|
267
|
+
"""
|
|
268
|
+
Deletes a `custom pool <https://learn.microsoft.com/fabric/data-engineering/create-custom-spark-pools>`_ within a workspace.
|
|
269
|
+
|
|
270
|
+
Parameters
|
|
271
|
+
----------
|
|
272
|
+
pool_name : str
|
|
273
|
+
The custom pool name.
|
|
274
|
+
workspace : str, default=None
|
|
275
|
+
The name of the Fabric workspace.
|
|
276
|
+
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
277
|
+
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
278
|
+
"""
|
|
279
|
+
|
|
280
|
+
(workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
|
|
281
|
+
|
|
282
|
+
dfL = list_custom_pools(workspace=workspace)
|
|
283
|
+
dfL_filt = dfL[dfL["Custom Pool Name"] == pool_name]
|
|
284
|
+
|
|
285
|
+
if len(dfL_filt) == 0:
|
|
286
|
+
raise ValueError(
|
|
287
|
+
f"{icons.red_dot} The '{pool_name}' custom pool does not exist within the '{workspace}' workspace."
|
|
288
|
+
)
|
|
289
|
+
poolId = dfL_filt["Custom Pool ID"].iloc[0]
|
|
290
|
+
|
|
291
|
+
client = fabric.FabricRestClient()
|
|
292
|
+
response = client.delete(f"/v1/workspaces/{workspace_id}/spark/pools/{poolId}")
|
|
293
|
+
|
|
294
|
+
if response.status_code != 200:
|
|
295
|
+
raise FabricHTTPException(response)
|
|
296
|
+
print(
|
|
297
|
+
f"{icons.green_dot} The '{pool_name}' spark pool has been deleted from the '{workspace}' workspace."
|
|
298
|
+
)
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def get_spark_settings(workspace: Optional[str] = None) -> pd.DataFrame:
|
|
302
|
+
"""
|
|
303
|
+
Shows the spark settings for a workspace.
|
|
304
|
+
|
|
305
|
+
Parameters
|
|
306
|
+
----------
|
|
307
|
+
workspace : str, default=None
|
|
308
|
+
The name of the Fabric workspace.
|
|
309
|
+
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
310
|
+
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
311
|
+
|
|
312
|
+
Returns
|
|
313
|
+
-------
|
|
314
|
+
pandas.DataFrame
|
|
315
|
+
A pandas dataframe showing the spark settings for a workspace.
|
|
316
|
+
"""
|
|
317
|
+
|
|
318
|
+
# https://learn.microsoft.com/en-us/rest/api/fabric/spark/workspace-settings/get-spark-settings?tabs=HTTP
|
|
319
|
+
(workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
|
|
320
|
+
|
|
321
|
+
df = pd.DataFrame(
|
|
322
|
+
columns=[
|
|
323
|
+
"Automatic Log Enabled",
|
|
324
|
+
"High Concurrency Enabled",
|
|
325
|
+
"Customize Compute Enabled",
|
|
326
|
+
"Default Pool Name",
|
|
327
|
+
"Default Pool Type",
|
|
328
|
+
"Max Node Count",
|
|
329
|
+
"Max Executors",
|
|
330
|
+
"Environment Name",
|
|
331
|
+
"Runtime Version",
|
|
332
|
+
]
|
|
333
|
+
)
|
|
334
|
+
|
|
335
|
+
client = fabric.FabricRestClient()
|
|
336
|
+
response = client.get(f"/v1/workspaces/{workspace_id}/spark/settings")
|
|
337
|
+
if response.status_code != 200:
|
|
338
|
+
raise FabricHTTPException(response)
|
|
339
|
+
|
|
340
|
+
i = response.json()
|
|
341
|
+
p = i.get("pool")
|
|
342
|
+
dp = i.get("pool", {}).get("defaultPool", {})
|
|
343
|
+
sp = i.get("pool", {}).get("starterPool", {})
|
|
344
|
+
e = i.get("environment", {})
|
|
345
|
+
|
|
346
|
+
new_data = {
|
|
347
|
+
"Automatic Log Enabled": i.get("automaticLog").get("enabled"),
|
|
348
|
+
"High Concurrency Enabled": i.get("highConcurrency").get(
|
|
349
|
+
"notebookInteractiveRunEnabled"
|
|
350
|
+
),
|
|
351
|
+
"Customize Compute Enabled": p.get("customizeComputeEnabled"),
|
|
352
|
+
"Default Pool Name": dp.get("name"),
|
|
353
|
+
"Default Pool Type": dp.get("type"),
|
|
354
|
+
"Max Node Count": sp.get("maxNodeCount"),
|
|
355
|
+
"Max Node Executors": sp.get("maxExecutors"),
|
|
356
|
+
"Environment Name": e.get("name"),
|
|
357
|
+
"Runtime Version": e.get("runtimeVersion"),
|
|
358
|
+
}
|
|
359
|
+
df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
|
|
360
|
+
|
|
361
|
+
bool_cols = [
|
|
362
|
+
"Automatic Log Enabled",
|
|
363
|
+
"High Concurrency Enabled",
|
|
364
|
+
"Customize Compute Enabled",
|
|
365
|
+
]
|
|
366
|
+
int_cols = ["Max Node Count", "Max Executors"]
|
|
367
|
+
|
|
368
|
+
df[bool_cols] = df[bool_cols].astype(bool)
|
|
369
|
+
df[int_cols] = df[int_cols].astype(int)
|
|
370
|
+
|
|
371
|
+
return df
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def update_spark_settings(
|
|
375
|
+
automatic_log_enabled: Optional[bool] = None,
|
|
376
|
+
high_concurrency_enabled: Optional[bool] = None,
|
|
377
|
+
customize_compute_enabled: Optional[bool] = None,
|
|
378
|
+
default_pool_name: Optional[str] = None,
|
|
379
|
+
max_node_count: Optional[int] = None,
|
|
380
|
+
max_executors: Optional[int] = None,
|
|
381
|
+
environment_name: Optional[str] = None,
|
|
382
|
+
runtime_version: Optional[str] = None,
|
|
383
|
+
workspace: Optional[str] = None,
|
|
384
|
+
):
|
|
385
|
+
"""
|
|
386
|
+
Updates the spark settings for a workspace.
|
|
387
|
+
|
|
388
|
+
Parameters
|
|
389
|
+
----------
|
|
390
|
+
automatic_log_enabled : bool, default=None
|
|
391
|
+
The status of the `automatic log <https://learn.microsoft.com/rest/api/fabric/spark/workspace-settings/update-spark-settings?tabs=HTTP#automaticlogproperties>`_.
|
|
392
|
+
Defaults to None which keeps the existing property setting.
|
|
393
|
+
high_concurrency_enabled : bool, default=None
|
|
394
|
+
The status of the `high concurrency <https://learn.microsoft.com/rest/api/fabric/spark/workspace-settings/update-spark-settings?tabs=HTTP#highconcurrencyproperties>`_ for notebook interactive run.
|
|
395
|
+
Defaults to None which keeps the existing property setting.
|
|
396
|
+
customize_compute_enabled : bool, default=None
|
|
397
|
+
`Customize compute <https://learn.microsoft.com/rest/api/fabric/spark/workspace-settings/update-spark-settings?tabs=HTTP#poolproperties>`_ configurations for items.
|
|
398
|
+
Defaults to None which keeps the existing property setting.
|
|
399
|
+
default_pool_name : str, default=None
|
|
400
|
+
`Default pool <https://learn.microsoft.com/rest/api/fabric/spark/workspace-settings/update-spark-settings?tabs=HTTP#poolproperties>`_ for workspace.
|
|
401
|
+
Defaults to None which keeps the existing property setting.
|
|
402
|
+
max_node_count : int, default=None
|
|
403
|
+
The `maximum node count <https://learn.microsoft.com/en-us/rest/api/fabric/spark/workspace-settings/update-spark-settings?tabs=HTTP#starterpoolproperties>`_.
|
|
404
|
+
Defaults to None which keeps the existing property setting.
|
|
405
|
+
max_executors : int, default=None
|
|
406
|
+
The `maximum executors <https://learn.microsoft.com/rest/api/fabric/spark/workspace-settings/update-spark-settings?tabs=HTTP#starterpoolproperties>`_.
|
|
407
|
+
Defaults to None which keeps the existing property setting.
|
|
408
|
+
environment_name : str, default=None
|
|
409
|
+
The name of the `default environment <https://learn.microsoft.com/rest/api/fabric/spark/workspace-settings/update-spark-settings?tabs=HTTP#environmentproperties>`_. Empty string indicated there is no workspace default environment
|
|
410
|
+
Defaults to None which keeps the existing property setting.
|
|
411
|
+
runtime_version : str, default=None
|
|
412
|
+
The `runtime version <https://learn.microsoft.com/rest/api/fabric/spark/workspace-settings/update-spark-settings?tabs=HTTP#environmentproperties>`_.
|
|
413
|
+
Defaults to None which keeps the existing property setting.
|
|
414
|
+
workspace : str, default=None
|
|
415
|
+
The name of the Fabric workspace.
|
|
416
|
+
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
417
|
+
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
418
|
+
"""
|
|
419
|
+
|
|
420
|
+
# https://learn.microsoft.com/en-us/rest/api/fabric/spark/workspace-settings/update-spark-settings?tabs=HTTP
|
|
421
|
+
(workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
|
|
422
|
+
|
|
423
|
+
dfS = get_spark_settings(workspace=workspace)
|
|
424
|
+
|
|
425
|
+
if automatic_log_enabled is None:
|
|
426
|
+
automatic_log_enabled = bool(dfS["Automatic Log Enabled"].iloc[0])
|
|
427
|
+
if high_concurrency_enabled is None:
|
|
428
|
+
high_concurrency_enabled = bool(dfS["High Concurrency Enabled"].iloc[0])
|
|
429
|
+
if customize_compute_enabled is None:
|
|
430
|
+
customize_compute_enabled = bool(dfS["Customize Compute Enabled"].iloc[0])
|
|
431
|
+
if default_pool_name is None:
|
|
432
|
+
default_pool_name = dfS["Default Pool Name"].iloc[0]
|
|
433
|
+
if max_node_count is None:
|
|
434
|
+
max_node_count = int(dfS["Max Node Count"].iloc[0])
|
|
435
|
+
if max_executors is None:
|
|
436
|
+
max_executors = int(dfS["Max Executors"].iloc[0])
|
|
437
|
+
if environment_name is None:
|
|
438
|
+
environment_name = dfS["Environment Name"].iloc[0]
|
|
439
|
+
if runtime_version is None:
|
|
440
|
+
runtime_version = dfS["Runtime Version"].iloc[0]
|
|
441
|
+
|
|
442
|
+
request_body = {
|
|
443
|
+
"automaticLog": {"enabled": automatic_log_enabled},
|
|
444
|
+
"highConcurrency": {"notebookInteractiveRunEnabled": high_concurrency_enabled},
|
|
445
|
+
"pool": {
|
|
446
|
+
"customizeComputeEnabled": customize_compute_enabled,
|
|
447
|
+
"defaultPool": {"name": default_pool_name, "type": "Workspace"},
|
|
448
|
+
"starterPool": {
|
|
449
|
+
"maxNodeCount": max_node_count,
|
|
450
|
+
"maxExecutors": max_executors,
|
|
451
|
+
},
|
|
452
|
+
},
|
|
453
|
+
"environment": {"name": environment_name, "runtimeVersion": runtime_version},
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
client = fabric.FabricRestClient()
|
|
457
|
+
response = client.patch(
|
|
458
|
+
f"/v1/workspaces/{workspace_id}/spark/settings", json=request_body
|
|
459
|
+
)
|
|
460
|
+
|
|
461
|
+
if response.status_code != 200:
|
|
462
|
+
raise FabricHTTPException(response)
|
|
463
|
+
print(
|
|
464
|
+
f"{icons.green_dot} The spark settings within the '{workspace}' workspace have been updated accordingly."
|
|
465
|
+
)
|