semantic-link-labs 0.7.2__py3-none-any.whl → 0.7.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of semantic-link-labs might be problematic. Click here for more details.

Files changed (82) hide show
  1. {semantic_link_labs-0.7.2.dist-info → semantic_link_labs-0.7.4.dist-info}/METADATA +15 -3
  2. semantic_link_labs-0.7.4.dist-info/RECORD +134 -0
  3. {semantic_link_labs-0.7.2.dist-info → semantic_link_labs-0.7.4.dist-info}/WHEEL +1 -1
  4. sempy_labs/__init__.py +120 -24
  5. sempy_labs/_bpa_translation/{_translations_am-ET.po → _model/_translations_am-ET.po} +22 -0
  6. sempy_labs/_bpa_translation/{_translations_ar-AE.po → _model/_translations_ar-AE.po} +24 -0
  7. sempy_labs/_bpa_translation/_model/_translations_bg-BG.po +938 -0
  8. sempy_labs/_bpa_translation/_model/_translations_ca-ES.po +934 -0
  9. sempy_labs/_bpa_translation/{_translations_cs-CZ.po → _model/_translations_cs-CZ.po} +179 -157
  10. sempy_labs/_bpa_translation/{_translations_da-DK.po → _model/_translations_da-DK.po} +24 -0
  11. sempy_labs/_bpa_translation/{_translations_de-DE.po → _model/_translations_de-DE.po} +77 -52
  12. sempy_labs/_bpa_translation/{_translations_el-GR.po → _model/_translations_el-GR.po} +25 -0
  13. sempy_labs/_bpa_translation/{_translations_es-ES.po → _model/_translations_es-ES.po} +67 -43
  14. sempy_labs/_bpa_translation/{_translations_fa-IR.po → _model/_translations_fa-IR.po} +24 -0
  15. sempy_labs/_bpa_translation/_model/_translations_fi-FI.po +915 -0
  16. sempy_labs/_bpa_translation/{_translations_fr-FR.po → _model/_translations_fr-FR.po} +83 -57
  17. sempy_labs/_bpa_translation/{_translations_ga-IE.po → _model/_translations_ga-IE.po} +25 -0
  18. sempy_labs/_bpa_translation/{_translations_he-IL.po → _model/_translations_he-IL.po} +23 -0
  19. sempy_labs/_bpa_translation/{_translations_hi-IN.po → _model/_translations_hi-IN.po} +24 -0
  20. sempy_labs/_bpa_translation/{_translations_hu-HU.po → _model/_translations_hu-HU.po} +25 -0
  21. sempy_labs/_bpa_translation/_model/_translations_id-ID.po +918 -0
  22. sempy_labs/_bpa_translation/{_translations_is-IS.po → _model/_translations_is-IS.po} +25 -0
  23. sempy_labs/_bpa_translation/{_translations_it-IT.po → _model/_translations_it-IT.po} +25 -0
  24. sempy_labs/_bpa_translation/{_translations_ja-JP.po → _model/_translations_ja-JP.po} +21 -0
  25. sempy_labs/_bpa_translation/_model/_translations_ko-KR.po +823 -0
  26. sempy_labs/_bpa_translation/_model/_translations_mt-MT.po +937 -0
  27. sempy_labs/_bpa_translation/{_translations_nl-NL.po → _model/_translations_nl-NL.po} +80 -56
  28. sempy_labs/_bpa_translation/{_translations_pl-PL.po → _model/_translations_pl-PL.po} +101 -76
  29. sempy_labs/_bpa_translation/{_translations_pt-BR.po → _model/_translations_pt-BR.po} +25 -0
  30. sempy_labs/_bpa_translation/{_translations_pt-PT.po → _model/_translations_pt-PT.po} +25 -0
  31. sempy_labs/_bpa_translation/_model/_translations_ro-RO.po +939 -0
  32. sempy_labs/_bpa_translation/{_translations_ru-RU.po → _model/_translations_ru-RU.po} +25 -0
  33. sempy_labs/_bpa_translation/_model/_translations_sk-SK.po +925 -0
  34. sempy_labs/_bpa_translation/_model/_translations_sl-SL.po +922 -0
  35. sempy_labs/_bpa_translation/_model/_translations_sv-SE.po +914 -0
  36. sempy_labs/_bpa_translation/{_translations_ta-IN.po → _model/_translations_ta-IN.po} +26 -0
  37. sempy_labs/_bpa_translation/{_translations_te-IN.po → _model/_translations_te-IN.po} +24 -0
  38. sempy_labs/_bpa_translation/{_translations_th-TH.po → _model/_translations_th-TH.po} +24 -0
  39. sempy_labs/_bpa_translation/_model/_translations_tr-TR.po +925 -0
  40. sempy_labs/_bpa_translation/_model/_translations_uk-UA.po +933 -0
  41. sempy_labs/_bpa_translation/{_translations_zh-CN.po → _model/_translations_zh-CN.po} +116 -97
  42. sempy_labs/_bpa_translation/{_translations_zu-ZA.po → _model/_translations_zu-ZA.po} +25 -0
  43. sempy_labs/_capacities.py +541 -0
  44. sempy_labs/_clear_cache.py +298 -3
  45. sempy_labs/_connections.py +138 -0
  46. sempy_labs/_dataflows.py +130 -0
  47. sempy_labs/_deployment_pipelines.py +171 -0
  48. sempy_labs/_environments.py +156 -0
  49. sempy_labs/_generate_semantic_model.py +148 -27
  50. sempy_labs/_git.py +380 -0
  51. sempy_labs/_helper_functions.py +203 -8
  52. sempy_labs/_icons.py +43 -0
  53. sempy_labs/_list_functions.py +170 -1012
  54. sempy_labs/_model_bpa.py +90 -112
  55. sempy_labs/_model_bpa_bulk.py +3 -1
  56. sempy_labs/_model_bpa_rules.py +788 -800
  57. sempy_labs/_notebooks.py +143 -0
  58. sempy_labs/_query_scale_out.py +28 -7
  59. sempy_labs/_spark.py +465 -0
  60. sempy_labs/_sql.py +120 -0
  61. sempy_labs/_translations.py +3 -1
  62. sempy_labs/_vertipaq.py +160 -99
  63. sempy_labs/_workspace_identity.py +66 -0
  64. sempy_labs/_workspaces.py +294 -0
  65. sempy_labs/directlake/__init__.py +2 -0
  66. sempy_labs/directlake/_directlake_schema_compare.py +1 -2
  67. sempy_labs/directlake/_directlake_schema_sync.py +1 -2
  68. sempy_labs/directlake/_dl_helper.py +4 -7
  69. sempy_labs/directlake/_generate_shared_expression.py +85 -0
  70. sempy_labs/directlake/_show_unsupported_directlake_objects.py +1 -2
  71. sempy_labs/lakehouse/_get_lakehouse_tables.py +7 -3
  72. sempy_labs/migration/_migrate_calctables_to_lakehouse.py +5 -0
  73. sempy_labs/migration/_migrate_calctables_to_semantic_model.py +5 -0
  74. sempy_labs/migration/_migrate_model_objects_to_semantic_model.py +6 -2
  75. sempy_labs/migration/_migrate_tables_columns_to_semantic_model.py +6 -5
  76. sempy_labs/migration/_migration_validation.py +6 -0
  77. sempy_labs/report/_report_functions.py +21 -42
  78. sempy_labs/report/_report_rebind.py +5 -0
  79. sempy_labs/tom/_model.py +95 -52
  80. semantic_link_labs-0.7.2.dist-info/RECORD +0 -111
  81. {semantic_link_labs-0.7.2.dist-info → semantic_link_labs-0.7.4.dist-info}/LICENSE +0 -0
  82. {semantic_link_labs-0.7.2.dist-info → semantic_link_labs-0.7.4.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,143 @@
1
+ import sempy.fabric as fabric
2
+ import pandas as pd
3
+ import sempy_labs._icons as icons
4
+ from typing import Optional
5
+ import base64
6
+ import requests
7
+ from sempy_labs._helper_functions import (
8
+ resolve_workspace_name_and_id,
9
+ lro,
10
+ _decode_b64,
11
+ )
12
+ from sempy.fabric.exceptions import FabricHTTPException
13
+
14
+
15
+ def get_notebook_definition(
16
+ notebook_name: str, workspace: Optional[str] = None, decode: Optional[bool] = True
17
+ ):
18
+ """
19
+ Obtains the notebook definition.
20
+
21
+ Parameters
22
+ ----------
23
+ notebook_name : str
24
+ The name of the notebook.
25
+ workspace : str, default=None
26
+ The name of the workspace.
27
+ Defaults to None which resolves to the workspace of the attached lakehouse
28
+ or if no lakehouse attached, resolves to the workspace of the notebook.
29
+ decode : bool, default=True
30
+ If True, decodes the notebook definition file into .ipynb format.
31
+ If False, obtains the notebook definition file in base64 format.
32
+
33
+ Returns
34
+ -------
35
+ ipynb
36
+ The notebook definition.
37
+ """
38
+
39
+ (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
40
+
41
+ dfI = fabric.list_items(workspace=workspace, type="Notebook")
42
+ dfI_filt = dfI[dfI["Display Name"] == notebook_name]
43
+
44
+ if len(dfI_filt) == 0:
45
+ raise ValueError(
46
+ f"{icons.red_dot} The '{notebook_name}' notebook does not exist within the '{workspace}' workspace."
47
+ )
48
+
49
+ notebook_id = dfI_filt["Id"].iloc[0]
50
+ client = fabric.FabricRestClient()
51
+ response = client.post(
52
+ f"v1/workspaces/{workspace_id}/notebooks/{notebook_id}/getDefinition",
53
+ )
54
+
55
+ result = lro(client, response).json()
56
+ df_items = pd.json_normalize(result["definition"]["parts"])
57
+ df_items_filt = df_items[df_items["path"] == "notebook-content.py"]
58
+ payload = df_items_filt["payload"].iloc[0]
59
+
60
+ if decode:
61
+ result = _decode_b64(payload)
62
+ else:
63
+ result = payload
64
+
65
+ return result
66
+
67
+
68
+ def import_notebook_from_web(
69
+ notebook_name: str,
70
+ url: str,
71
+ description: Optional[str] = None,
72
+ workspace: Optional[str] = None,
73
+ ):
74
+ """
75
+ Creates a new notebook within a workspace based on a Jupyter notebook hosted in the web.
76
+
77
+ Note: When specifying a notebook from GitHub, please use the raw file path. Note that if the non-raw file path is specified, the url will be
78
+ converted to the raw URL as the raw URL is needed to obtain the notebook content.
79
+
80
+ Parameters
81
+ ----------
82
+ notebook_name : str
83
+ The name of the notebook to be created.
84
+ url : str
85
+ The url of the Jupyter Notebook (.ipynb)
86
+ description : str, default=None
87
+ The description of the notebook.
88
+ Defaults to None which does not place a description.
89
+ workspace : str, default=None
90
+ The name of the workspace.
91
+ Defaults to None which resolves to the workspace of the attached lakehouse
92
+ or if no lakehouse attached, resolves to the workspace of the notebook.
93
+
94
+ Returns
95
+ -------
96
+ """
97
+
98
+ (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
99
+ client = fabric.FabricRestClient()
100
+ dfI = fabric.list_items(workspace=workspace, type="Notebook")
101
+ dfI_filt = dfI[dfI["Display Name"] == notebook_name]
102
+ if len(dfI_filt) > 0:
103
+ raise ValueError(
104
+ f"{icons.red_dot} The '{notebook_name}' already exists within the '{workspace}' workspace."
105
+ )
106
+
107
+ # Fix links to go to the raw github file
108
+ starting_text = "https://github.com/"
109
+ starting_text_len = len(starting_text)
110
+ if url.startswith(starting_text):
111
+ url = f"https://raw.githubusercontent.com/{url[starting_text_len:]}".replace(
112
+ "/blob/", "/"
113
+ )
114
+
115
+ response = requests.get(url)
116
+ if response.status_code != 200:
117
+ raise FabricHTTPException(response)
118
+ file_content = response.content
119
+ notebook_payload = base64.b64encode(file_content)
120
+
121
+ request_body = {
122
+ "displayName": notebook_name,
123
+ "definition": {
124
+ "format": "ipynb",
125
+ "parts": [
126
+ {
127
+ "path": "notebook-content.py",
128
+ "payload": notebook_payload,
129
+ "payloadType": "InlineBase64",
130
+ }
131
+ ],
132
+ },
133
+ }
134
+ if description is not None:
135
+ request_body["description"] = description
136
+
137
+ response = client.post(f"v1/workspaces/{workspace_id}/notebooks", json=request_body)
138
+
139
+ lro(client, response, status_codes=[201, 202])
140
+
141
+ print(
142
+ f"{icons.green_dot} The '{notebook_name}' notebook was created within the '{workspace}' workspace."
143
+ )
@@ -293,13 +293,24 @@ def set_semantic_model_storage_format(
293
293
  f"{icons.red_dot} Invalid storage format value. Valid options: {storageFormats}."
294
294
  )
295
295
 
296
+ dfL = list_qso_settings(dataset=dataset, workspace=workspace)
297
+ current_storage_format = dfL["Storage Mode"].iloc[0]
298
+
299
+ if current_storage_format == storage_format:
300
+ print(
301
+ f"{icons.info} The '{dataset}' semantic model within the '{workspace}' workspace is already set to '{storage_format.lower()}' storage format."
302
+ )
303
+ return
304
+
296
305
  client = fabric.PowerBIRestClient()
297
306
  response = client.patch(
298
307
  f"/v1.0/myorg/groups/{workspace_id}/datasets/{dataset_id}", json=request_body
299
308
  )
300
309
  if response.status_code != 200:
301
310
  raise FabricHTTPException(response)
302
- print(f"{icons.green_dot} Semantic model storage format set to '{storage_format}'.")
311
+ print(
312
+ f"{icons.green_dot} The semantic model storage format for the '{dataset}' semantic model within the '{workspace}' workspace has been set to '{storage_format}'."
313
+ )
303
314
 
304
315
 
305
316
  def list_qso_settings(
@@ -383,10 +394,6 @@ def set_workspace_default_storage_format(
383
394
  The Fabric workspace name.
384
395
  Defaults to None which resolves to the workspace of the attached lakehouse
385
396
  or if no lakehouse attached, resolves to the workspace of the notebook.
386
-
387
- Returns
388
- -------
389
-
390
397
  """
391
398
 
392
399
  # https://learn.microsoft.com/en-us/rest/api/power-bi/groups/update-group#defaultdatasetstorageformat
@@ -396,19 +403,33 @@ def set_workspace_default_storage_format(
396
403
  storage_format = storage_format.capitalize()
397
404
 
398
405
  if storage_format not in storageFormats:
399
- print(
406
+ raise ValueError(
400
407
  f"{icons.red_dot} Invalid storage format. Please choose from these options: {storageFormats}."
401
408
  )
402
409
 
403
410
  (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
404
411
 
405
- request_body = {"name": workspace, "defaultDatasetStorageFormat": storage_format}
412
+ # Check current storage format
413
+ dfW = fabric.list_workspaces(filter=f"name eq '{workspace}'")
414
+ if len(dfW) == 0:
415
+ raise ValueError()
416
+ current_storage_format = dfW['Default Dataset Storage Format'].iloc[0]
417
+
418
+ if current_storage_format == storage_format:
419
+ print(f"{icons.info} The '{workspace}' is already set to a default storage format of '{current_storage_format}'.")
420
+ return
421
+
422
+ request_body = {
423
+ "name": workspace,
424
+ "defaultDatasetStorageFormat": storage_format,
425
+ }
406
426
 
407
427
  client = fabric.PowerBIRestClient()
408
428
  response = client.patch(f"/v1.0/myorg/groups/{workspace_id}", json=request_body)
409
429
 
410
430
  if response.status_code != 200:
411
431
  raise FabricHTTPException(response)
432
+
412
433
  print(
413
434
  f"{icons.green_dot} The default storage format for the '{workspace}' workspace has been updated to '{storage_format}."
414
435
  )
sempy_labs/_spark.py ADDED
@@ -0,0 +1,465 @@
1
+ import sempy.fabric as fabric
2
+ import pandas as pd
3
+ import sempy_labs._icons as icons
4
+ from typing import Optional
5
+ from sempy_labs._helper_functions import (
6
+ resolve_workspace_name_and_id,
7
+ )
8
+ from sempy.fabric.exceptions import FabricHTTPException
9
+
10
+
11
+ def list_custom_pools(workspace: Optional[str] = None) -> pd.DataFrame:
12
+ """
13
+ Lists all `custom pools <https://learn.microsoft.com/fabric/data-engineering/create-custom-spark-pools>`_ within a workspace.
14
+
15
+ Parameters
16
+ ----------
17
+ workspace : str, default=None
18
+ The name of the Fabric workspace.
19
+ Defaults to None which resolves to the workspace of the attached lakehouse
20
+ or if no lakehouse attached, resolves to the workspace of the notebook.
21
+
22
+ Returns
23
+ -------
24
+ pandas.DataFrame
25
+ A pandas dataframe showing all the custom pools within the Fabric workspace.
26
+ """
27
+
28
+ # https://learn.microsoft.com/rest/api/fabric/spark/custom-pools/list-workspace-custom-pools
29
+ (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
30
+
31
+ df = pd.DataFrame(
32
+ columns=[
33
+ "Custom Pool ID",
34
+ "Custom Pool Name",
35
+ "Type",
36
+ "Node Family",
37
+ "Node Size",
38
+ "Auto Scale Enabled",
39
+ "Auto Scale Min Node Count",
40
+ "Auto Scale Max Node Count",
41
+ "Dynamic Executor Allocation Enabled",
42
+ "Dynamic Executor Allocation Min Executors",
43
+ "Dynamic Executor Allocation Max Executors",
44
+ ]
45
+ )
46
+
47
+ client = fabric.FabricRestClient()
48
+ response = client.get(f"/v1/workspaces/{workspace_id}/spark/pools")
49
+ if response.status_code != 200:
50
+ raise FabricHTTPException(response)
51
+
52
+ for i in response.json()["value"]:
53
+
54
+ aScale = i.get("autoScale", {})
55
+ d = i.get("dynamicExecutorAllocation", {})
56
+
57
+ new_data = {
58
+ "Custom Pool ID": i.get("id"),
59
+ "Custom Pool Name": i.get("name"),
60
+ "Type": i.get("type"),
61
+ "Node Family": i.get("nodeFamily"),
62
+ "Node Size": i.get("nodeSize"),
63
+ "Auto Scale Enabled": aScale.get("enabled"),
64
+ "Auto Scale Min Node Count": aScale.get("minNodeCount"),
65
+ "Auto Scale Max Node Count": aScale.get("maxNodeCount"),
66
+ "Dynamic Executor Allocation Enabled": d.get("enabled"),
67
+ "Dynamic Executor Allocation Min Executors": d.get("minExecutors"),
68
+ "Dynamic Executor Allocation Max Executors": d.get("maxExecutors"),
69
+ }
70
+ df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
71
+
72
+ bool_cols = ["Auto Scale Enabled", "Dynamic Executor Allocation Enabled"]
73
+ int_cols = [
74
+ "Auto Scale Min Node Count",
75
+ "Auto Scale Max Node Count",
76
+ "Dynamic Executor Allocation Enabled",
77
+ "Dynamic Executor Allocation Min Executors",
78
+ "Dynamic Executor Allocation Max Executors",
79
+ ]
80
+
81
+ df[bool_cols] = df[bool_cols].astype(bool)
82
+ df[int_cols] = df[int_cols].astype(int)
83
+
84
+ return df
85
+
86
+
87
+ def create_custom_pool(
88
+ pool_name: str,
89
+ node_size: str,
90
+ min_node_count: int,
91
+ max_node_count: int,
92
+ min_executors: int,
93
+ max_executors: int,
94
+ node_family: Optional[str] = "MemoryOptimized",
95
+ auto_scale_enabled: Optional[bool] = True,
96
+ dynamic_executor_allocation_enabled: Optional[bool] = True,
97
+ workspace: Optional[str] = None,
98
+ ):
99
+ """
100
+ Creates a `custom pool <https://learn.microsoft.com/fabric/data-engineering/create-custom-spark-pools>`_ within a workspace.
101
+
102
+ Parameters
103
+ ----------
104
+ pool_name : str
105
+ The custom pool name.
106
+ node_size : str
107
+ The `node size <https://learn.microsoft.com/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#nodesize>`_.
108
+ min_node_count : int
109
+ The `minimum node count <https://learn.microsoft.com/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#autoscaleproperties>`_.
110
+ max_node_count : int
111
+ The `maximum node count <https://learn.microsoft.com/en-us/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#autoscaleproperties>`_.
112
+ min_executors : int
113
+ The `minimum executors <https://learn.microsoft.com/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#dynamicexecutorallocationproperties>`_.
114
+ max_executors : int
115
+ The `maximum executors <https://learn.microsoft.com/en-us/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#dynamicexecutorallocationproperties>`_.
116
+ node_family : str, default='MemoryOptimized'
117
+ The `node family <https://learn.microsoft.com/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#nodefamily>`_.
118
+ auto_scale_enabled : bool, default=True
119
+ The status of `auto scale <https://learn.microsoft.com/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#autoscaleproperties>`_.
120
+ dynamic_executor_allocation_enabled : bool, default=True
121
+ The status of the `dynamic executor allocation <https://learn.microsoft.com/en-us/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#dynamicexecutorallocationproperties>`_.
122
+ workspace : str, default=None
123
+ The name of the Fabric workspace.
124
+ Defaults to None which resolves to the workspace of the attached lakehouse
125
+ or if no lakehouse attached, resolves to the workspace of the notebook.
126
+ """
127
+
128
+ # https://learn.microsoft.com/en-us/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool
129
+ (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
130
+
131
+ request_body = {
132
+ "name": pool_name,
133
+ "nodeFamily": node_family,
134
+ "nodeSize": node_size,
135
+ "autoScale": {
136
+ "enabled": auto_scale_enabled,
137
+ "minNodeCount": min_node_count,
138
+ "maxNodeCount": max_node_count,
139
+ },
140
+ "dynamicExecutorAllocation": {
141
+ "enabled": dynamic_executor_allocation_enabled,
142
+ "minExecutors": min_executors,
143
+ "maxExecutors": max_executors,
144
+ },
145
+ }
146
+
147
+ client = fabric.FabricRestClient()
148
+ response = client.post(
149
+ f"/v1/workspaces/{workspace_id}/spark/pools", json=request_body
150
+ )
151
+
152
+ if response.status_code != 201:
153
+ raise FabricHTTPException(response)
154
+ print(
155
+ f"{icons.green_dot} The '{pool_name}' spark pool has been created within the '{workspace}' workspace."
156
+ )
157
+
158
+
159
+ def update_custom_pool(
160
+ pool_name: str,
161
+ node_size: Optional[str] = None,
162
+ min_node_count: Optional[int] = None,
163
+ max_node_count: Optional[int] = None,
164
+ min_executors: Optional[int] = None,
165
+ max_executors: Optional[int] = None,
166
+ node_family: Optional[str] = None,
167
+ auto_scale_enabled: Optional[bool] = None,
168
+ dynamic_executor_allocation_enabled: Optional[bool] = None,
169
+ workspace: Optional[str] = None,
170
+ ):
171
+ """
172
+ Updates the properties of a `custom pool <https://learn.microsoft.com/fabric/data-engineering/create-custom-spark-pools>`_ within a workspace.
173
+
174
+ Parameters
175
+ ----------
176
+ pool_name : str
177
+ The custom pool name.
178
+ node_size : str, default=None
179
+ The `node size <https://learn.microsoft.com/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#nodesize>`_.
180
+ Defaults to None which keeps the existing property setting.
181
+ min_node_count : int, default=None
182
+ The `minimum node count <https://learn.microsoft.com/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#autoscaleproperties>`_.
183
+ Defaults to None which keeps the existing property setting.
184
+ max_node_count : int, default=None
185
+ The `maximum node count <https://learn.microsoft.com/en-us/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#autoscaleproperties>`_.
186
+ Defaults to None which keeps the existing property setting.
187
+ min_executors : int, default=None
188
+ The `minimum executors <https://learn.microsoft.com/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#dynamicexecutorallocationproperties>`_.
189
+ Defaults to None which keeps the existing property setting.
190
+ max_executors : int, default=None
191
+ The `maximum executors <https://learn.microsoft.com/en-us/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#dynamicexecutorallocationproperties>`_.
192
+ Defaults to None which keeps the existing property setting.
193
+ node_family : str, default=None
194
+ The `node family <https://learn.microsoft.com/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#nodefamily>`_.
195
+ Defaults to None which keeps the existing property setting.
196
+ auto_scale_enabled : bool, default=None
197
+ The status of `auto scale <https://learn.microsoft.com/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#autoscaleproperties>`_.
198
+ Defaults to None which keeps the existing property setting.
199
+ dynamic_executor_allocation_enabled : bool, default=None
200
+ The status of the `dynamic executor allocation <https://learn.microsoft.com/en-us/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#dynamicexecutorallocationproperties>`_.
201
+ Defaults to None which keeps the existing property setting.
202
+ workspace : str, default=None
203
+ The name of the Fabric workspace.
204
+ Defaults to None which resolves to the workspace of the attached lakehouse
205
+ or if no lakehouse attached, resolves to the workspace of the notebook.
206
+ """
207
+
208
+ # https://learn.microsoft.com/en-us/rest/api/fabric/spark/custom-pools/update-workspace-custom-pool?tabs=HTTP
209
+ (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
210
+
211
+ df = list_custom_pools(workspace=workspace)
212
+ df_pool = df[df["Custom Pool Name"] == pool_name]
213
+
214
+ if len(df_pool) == 0:
215
+ raise ValueError(
216
+ f"{icons.red_dot} The '{pool_name}' custom pool does not exist within the '{workspace}'. Please choose a valid custom pool."
217
+ )
218
+
219
+ if node_family is None:
220
+ node_family = df_pool["Node Family"].iloc[0]
221
+ if node_size is None:
222
+ node_size = df_pool["Node Size"].iloc[0]
223
+ if auto_scale_enabled is None:
224
+ auto_scale_enabled = bool(df_pool["Auto Scale Enabled"].iloc[0])
225
+ if min_node_count is None:
226
+ min_node_count = int(df_pool["Min Node Count"].iloc[0])
227
+ if max_node_count is None:
228
+ max_node_count = int(df_pool["Max Node Count"].iloc[0])
229
+ if dynamic_executor_allocation_enabled is None:
230
+ dynamic_executor_allocation_enabled = bool(
231
+ df_pool["Dynami Executor Allocation Enabled"].iloc[0]
232
+ )
233
+ if min_executors is None:
234
+ min_executors = int(df_pool["Min Executors"].iloc[0])
235
+ if max_executors is None:
236
+ max_executors = int(df_pool["Max Executors"].iloc[0])
237
+
238
+ request_body = {
239
+ "name": pool_name,
240
+ "nodeFamily": node_family,
241
+ "nodeSize": node_size,
242
+ "autoScale": {
243
+ "enabled": auto_scale_enabled,
244
+ "minNodeCount": min_node_count,
245
+ "maxNodeCount": max_node_count,
246
+ },
247
+ "dynamicExecutorAllocation": {
248
+ "enabled": dynamic_executor_allocation_enabled,
249
+ "minExecutors": min_executors,
250
+ "maxExecutors": max_executors,
251
+ },
252
+ }
253
+
254
+ client = fabric.FabricRestClient()
255
+ response = client.post(
256
+ f"/v1/workspaces/{workspace_id}/spark/pools", json=request_body
257
+ )
258
+
259
+ if response.status_code != 200:
260
+ raise FabricHTTPException(response)
261
+ print(
262
+ f"{icons.green_dot} The '{pool_name}' spark pool within the '{workspace}' workspace has been updated."
263
+ )
264
+
265
+
266
+ def delete_custom_pool(pool_name: str, workspace: Optional[str] = None):
267
+ """
268
+ Deletes a `custom pool <https://learn.microsoft.com/fabric/data-engineering/create-custom-spark-pools>`_ within a workspace.
269
+
270
+ Parameters
271
+ ----------
272
+ pool_name : str
273
+ The custom pool name.
274
+ workspace : str, default=None
275
+ The name of the Fabric workspace.
276
+ Defaults to None which resolves to the workspace of the attached lakehouse
277
+ or if no lakehouse attached, resolves to the workspace of the notebook.
278
+ """
279
+
280
+ (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
281
+
282
+ dfL = list_custom_pools(workspace=workspace)
283
+ dfL_filt = dfL[dfL["Custom Pool Name"] == pool_name]
284
+
285
+ if len(dfL_filt) == 0:
286
+ raise ValueError(
287
+ f"{icons.red_dot} The '{pool_name}' custom pool does not exist within the '{workspace}' workspace."
288
+ )
289
+ poolId = dfL_filt["Custom Pool ID"].iloc[0]
290
+
291
+ client = fabric.FabricRestClient()
292
+ response = client.delete(f"/v1/workspaces/{workspace_id}/spark/pools/{poolId}")
293
+
294
+ if response.status_code != 200:
295
+ raise FabricHTTPException(response)
296
+ print(
297
+ f"{icons.green_dot} The '{pool_name}' spark pool has been deleted from the '{workspace}' workspace."
298
+ )
299
+
300
+
301
+ def get_spark_settings(workspace: Optional[str] = None) -> pd.DataFrame:
302
+ """
303
+ Shows the spark settings for a workspace.
304
+
305
+ Parameters
306
+ ----------
307
+ workspace : str, default=None
308
+ The name of the Fabric workspace.
309
+ Defaults to None which resolves to the workspace of the attached lakehouse
310
+ or if no lakehouse attached, resolves to the workspace of the notebook.
311
+
312
+ Returns
313
+ -------
314
+ pandas.DataFrame
315
+ A pandas dataframe showing the spark settings for a workspace.
316
+ """
317
+
318
+ # https://learn.microsoft.com/en-us/rest/api/fabric/spark/workspace-settings/get-spark-settings?tabs=HTTP
319
+ (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
320
+
321
+ df = pd.DataFrame(
322
+ columns=[
323
+ "Automatic Log Enabled",
324
+ "High Concurrency Enabled",
325
+ "Customize Compute Enabled",
326
+ "Default Pool Name",
327
+ "Default Pool Type",
328
+ "Max Node Count",
329
+ "Max Executors",
330
+ "Environment Name",
331
+ "Runtime Version",
332
+ ]
333
+ )
334
+
335
+ client = fabric.FabricRestClient()
336
+ response = client.get(f"/v1/workspaces/{workspace_id}/spark/settings")
337
+ if response.status_code != 200:
338
+ raise FabricHTTPException(response)
339
+
340
+ i = response.json()
341
+ p = i.get("pool")
342
+ dp = i.get("pool", {}).get("defaultPool", {})
343
+ sp = i.get("pool", {}).get("starterPool", {})
344
+ e = i.get("environment", {})
345
+
346
+ new_data = {
347
+ "Automatic Log Enabled": i.get("automaticLog").get("enabled"),
348
+ "High Concurrency Enabled": i.get("highConcurrency").get(
349
+ "notebookInteractiveRunEnabled"
350
+ ),
351
+ "Customize Compute Enabled": p.get("customizeComputeEnabled"),
352
+ "Default Pool Name": dp.get("name"),
353
+ "Default Pool Type": dp.get("type"),
354
+ "Max Node Count": sp.get("maxNodeCount"),
355
+ "Max Node Executors": sp.get("maxExecutors"),
356
+ "Environment Name": e.get("name"),
357
+ "Runtime Version": e.get("runtimeVersion"),
358
+ }
359
+ df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
360
+
361
+ bool_cols = [
362
+ "Automatic Log Enabled",
363
+ "High Concurrency Enabled",
364
+ "Customize Compute Enabled",
365
+ ]
366
+ int_cols = ["Max Node Count", "Max Executors"]
367
+
368
+ df[bool_cols] = df[bool_cols].astype(bool)
369
+ df[int_cols] = df[int_cols].astype(int)
370
+
371
+ return df
372
+
373
+
374
+ def update_spark_settings(
375
+ automatic_log_enabled: Optional[bool] = None,
376
+ high_concurrency_enabled: Optional[bool] = None,
377
+ customize_compute_enabled: Optional[bool] = None,
378
+ default_pool_name: Optional[str] = None,
379
+ max_node_count: Optional[int] = None,
380
+ max_executors: Optional[int] = None,
381
+ environment_name: Optional[str] = None,
382
+ runtime_version: Optional[str] = None,
383
+ workspace: Optional[str] = None,
384
+ ):
385
+ """
386
+ Updates the spark settings for a workspace.
387
+
388
+ Parameters
389
+ ----------
390
+ automatic_log_enabled : bool, default=None
391
+ The status of the `automatic log <https://learn.microsoft.com/rest/api/fabric/spark/workspace-settings/update-spark-settings?tabs=HTTP#automaticlogproperties>`_.
392
+ Defaults to None which keeps the existing property setting.
393
+ high_concurrency_enabled : bool, default=None
394
+ The status of the `high concurrency <https://learn.microsoft.com/rest/api/fabric/spark/workspace-settings/update-spark-settings?tabs=HTTP#highconcurrencyproperties>`_ for notebook interactive run.
395
+ Defaults to None which keeps the existing property setting.
396
+ customize_compute_enabled : bool, default=None
397
+ `Customize compute <https://learn.microsoft.com/rest/api/fabric/spark/workspace-settings/update-spark-settings?tabs=HTTP#poolproperties>`_ configurations for items.
398
+ Defaults to None which keeps the existing property setting.
399
+ default_pool_name : str, default=None
400
+ `Default pool <https://learn.microsoft.com/rest/api/fabric/spark/workspace-settings/update-spark-settings?tabs=HTTP#poolproperties>`_ for workspace.
401
+ Defaults to None which keeps the existing property setting.
402
+ max_node_count : int, default=None
403
+ The `maximum node count <https://learn.microsoft.com/en-us/rest/api/fabric/spark/workspace-settings/update-spark-settings?tabs=HTTP#starterpoolproperties>`_.
404
+ Defaults to None which keeps the existing property setting.
405
+ max_executors : int, default=None
406
+ The `maximum executors <https://learn.microsoft.com/rest/api/fabric/spark/workspace-settings/update-spark-settings?tabs=HTTP#starterpoolproperties>`_.
407
+ Defaults to None which keeps the existing property setting.
408
+ environment_name : str, default=None
409
+ The name of the `default environment <https://learn.microsoft.com/rest/api/fabric/spark/workspace-settings/update-spark-settings?tabs=HTTP#environmentproperties>`_. Empty string indicated there is no workspace default environment
410
+ Defaults to None which keeps the existing property setting.
411
+ runtime_version : str, default=None
412
+ The `runtime version <https://learn.microsoft.com/rest/api/fabric/spark/workspace-settings/update-spark-settings?tabs=HTTP#environmentproperties>`_.
413
+ Defaults to None which keeps the existing property setting.
414
+ workspace : str, default=None
415
+ The name of the Fabric workspace.
416
+ Defaults to None which resolves to the workspace of the attached lakehouse
417
+ or if no lakehouse attached, resolves to the workspace of the notebook.
418
+ """
419
+
420
+ # https://learn.microsoft.com/en-us/rest/api/fabric/spark/workspace-settings/update-spark-settings?tabs=HTTP
421
+ (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
422
+
423
+ dfS = get_spark_settings(workspace=workspace)
424
+
425
+ if automatic_log_enabled is None:
426
+ automatic_log_enabled = bool(dfS["Automatic Log Enabled"].iloc[0])
427
+ if high_concurrency_enabled is None:
428
+ high_concurrency_enabled = bool(dfS["High Concurrency Enabled"].iloc[0])
429
+ if customize_compute_enabled is None:
430
+ customize_compute_enabled = bool(dfS["Customize Compute Enabled"].iloc[0])
431
+ if default_pool_name is None:
432
+ default_pool_name = dfS["Default Pool Name"].iloc[0]
433
+ if max_node_count is None:
434
+ max_node_count = int(dfS["Max Node Count"].iloc[0])
435
+ if max_executors is None:
436
+ max_executors = int(dfS["Max Executors"].iloc[0])
437
+ if environment_name is None:
438
+ environment_name = dfS["Environment Name"].iloc[0]
439
+ if runtime_version is None:
440
+ runtime_version = dfS["Runtime Version"].iloc[0]
441
+
442
+ request_body = {
443
+ "automaticLog": {"enabled": automatic_log_enabled},
444
+ "highConcurrency": {"notebookInteractiveRunEnabled": high_concurrency_enabled},
445
+ "pool": {
446
+ "customizeComputeEnabled": customize_compute_enabled,
447
+ "defaultPool": {"name": default_pool_name, "type": "Workspace"},
448
+ "starterPool": {
449
+ "maxNodeCount": max_node_count,
450
+ "maxExecutors": max_executors,
451
+ },
452
+ },
453
+ "environment": {"name": environment_name, "runtimeVersion": runtime_version},
454
+ }
455
+
456
+ client = fabric.FabricRestClient()
457
+ response = client.patch(
458
+ f"/v1/workspaces/{workspace_id}/spark/settings", json=request_body
459
+ )
460
+
461
+ if response.status_code != 200:
462
+ raise FabricHTTPException(response)
463
+ print(
464
+ f"{icons.green_dot} The spark settings within the '{workspace}' workspace have been updated accordingly."
465
+ )