semantic-link-labs 0.4.2__py3-none-any.whl → 0.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of semantic-link-labs might be problematic. Click here for more details.
- {semantic_link_labs-0.4.2.dist-info → semantic_link_labs-0.6.0.dist-info}/METADATA +2 -2
- semantic_link_labs-0.6.0.dist-info/RECORD +54 -0
- {semantic_link_labs-0.4.2.dist-info → semantic_link_labs-0.6.0.dist-info}/WHEEL +1 -1
- sempy_labs/__init__.py +44 -14
- sempy_labs/_ai.py +31 -32
- sempy_labs/_clear_cache.py +5 -8
- sempy_labs/_connections.py +80 -72
- sempy_labs/_dax.py +7 -9
- sempy_labs/_generate_semantic_model.py +60 -54
- sempy_labs/_helper_functions.py +8 -10
- sempy_labs/_icons.py +15 -0
- sempy_labs/_list_functions.py +1139 -428
- sempy_labs/_model_auto_build.py +5 -6
- sempy_labs/_model_bpa.py +134 -1125
- sempy_labs/_model_bpa_rules.py +831 -0
- sempy_labs/_model_dependencies.py +21 -25
- sempy_labs/_one_lake_integration.py +10 -7
- sempy_labs/_query_scale_out.py +83 -93
- sempy_labs/_refresh_semantic_model.py +12 -16
- sempy_labs/_translations.py +214 -288
- sempy_labs/_vertipaq.py +51 -42
- sempy_labs/directlake/__init__.py +2 -0
- sempy_labs/directlake/_directlake_schema_compare.py +12 -11
- sempy_labs/directlake/_directlake_schema_sync.py +13 -23
- sempy_labs/directlake/_fallback.py +5 -7
- sempy_labs/directlake/_get_directlake_lakehouse.py +1 -1
- sempy_labs/directlake/_get_shared_expression.py +4 -8
- sempy_labs/directlake/_guardrails.py +6 -8
- sempy_labs/directlake/_list_directlake_model_calc_tables.py +18 -12
- sempy_labs/directlake/_show_unsupported_directlake_objects.py +4 -4
- sempy_labs/directlake/_update_directlake_model_lakehouse_connection.py +9 -8
- sempy_labs/directlake/_update_directlake_partition_entity.py +129 -12
- sempy_labs/directlake/_warm_cache.py +5 -5
- sempy_labs/lakehouse/_get_lakehouse_columns.py +2 -2
- sempy_labs/lakehouse/_get_lakehouse_tables.py +4 -4
- sempy_labs/lakehouse/_lakehouse.py +3 -4
- sempy_labs/lakehouse/_shortcuts.py +17 -13
- sempy_labs/migration/__init__.py +1 -1
- sempy_labs/migration/_create_pqt_file.py +21 -24
- sempy_labs/migration/_migrate_calctables_to_lakehouse.py +16 -13
- sempy_labs/migration/_migrate_calctables_to_semantic_model.py +17 -18
- sempy_labs/migration/_migrate_model_objects_to_semantic_model.py +45 -46
- sempy_labs/migration/_migrate_tables_columns_to_semantic_model.py +14 -14
- sempy_labs/migration/_migration_validation.py +6 -2
- sempy_labs/migration/_refresh_calc_tables.py +10 -5
- sempy_labs/report/__init__.py +2 -2
- sempy_labs/report/_generate_report.py +8 -7
- sempy_labs/report/_report_functions.py +47 -52
- sempy_labs/report/_report_rebind.py +38 -37
- sempy_labs/tom/__init__.py +1 -4
- sempy_labs/tom/_model.py +541 -180
- semantic_link_labs-0.4.2.dist-info/RECORD +0 -53
- {semantic_link_labs-0.4.2.dist-info → semantic_link_labs-0.6.0.dist-info}/LICENSE +0 -0
- {semantic_link_labs-0.4.2.dist-info → semantic_link_labs-0.6.0.dist-info}/top_level.txt +0 -0
sempy_labs/_list_functions.py
CHANGED
|
@@ -1,16 +1,22 @@
|
|
|
1
|
-
import sempy
|
|
2
1
|
import sempy.fabric as fabric
|
|
3
2
|
from sempy_labs._helper_functions import (
|
|
4
|
-
resolve_workspace_name_and_id,
|
|
5
|
-
resolve_lakehouse_name,
|
|
6
|
-
create_relationship_name,
|
|
7
|
-
resolve_lakehouse_id
|
|
3
|
+
resolve_workspace_name_and_id,
|
|
4
|
+
resolve_lakehouse_name,
|
|
5
|
+
create_relationship_name,
|
|
6
|
+
resolve_lakehouse_id,
|
|
7
|
+
)
|
|
8
8
|
import pandas as pd
|
|
9
|
-
import json
|
|
9
|
+
import json
|
|
10
|
+
import time
|
|
10
11
|
from pyspark.sql import SparkSession
|
|
11
12
|
from typing import Optional
|
|
13
|
+
import sempy_labs._icons as icons
|
|
14
|
+
from sempy.fabric.exceptions import FabricHTTPException
|
|
12
15
|
|
|
13
|
-
|
|
16
|
+
|
|
17
|
+
def get_object_level_security(
|
|
18
|
+
dataset: str, workspace: Optional[str] = None
|
|
19
|
+
) -> pd.DataFrame:
|
|
14
20
|
"""
|
|
15
21
|
Shows the object level security for the semantic model.
|
|
16
22
|
|
|
@@ -29,47 +35,54 @@ def get_object_level_security(dataset: str, workspace: Optional[str] = None):
|
|
|
29
35
|
A pandas dataframe showing the object level security for the semantic model.
|
|
30
36
|
"""
|
|
31
37
|
|
|
32
|
-
|
|
33
|
-
workspace_id = fabric.get_workspace_id()
|
|
34
|
-
workspace = fabric.resolve_workspace_name(workspace_id)
|
|
38
|
+
from sempy_labs.tom import connect_semantic_model
|
|
35
39
|
|
|
36
|
-
|
|
37
|
-
|
|
40
|
+
if workspace is None:
|
|
41
|
+
workspace = fabric.resolve_workspace_name()
|
|
38
42
|
|
|
39
43
|
df = pd.DataFrame(columns=["Role Name", "Object Type", "Table Name", "Object Name"])
|
|
40
44
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
)
|
|
56
|
-
else:
|
|
57
|
-
objectType = "Column"
|
|
58
|
-
for cp in tp.ColumnPermissions:
|
|
45
|
+
with connect_semantic_model(
|
|
46
|
+
dataset=dataset, readonly=True, workspace=workspace
|
|
47
|
+
) as tom:
|
|
48
|
+
|
|
49
|
+
for r in tom.model.Roles:
|
|
50
|
+
for tp in r.TablePermissions:
|
|
51
|
+
if len(tp.FilterExpression) == 0:
|
|
52
|
+
columnCount = 0
|
|
53
|
+
try:
|
|
54
|
+
columnCount = len(tp.ColumnPermissions)
|
|
55
|
+
except Exception:
|
|
56
|
+
pass
|
|
57
|
+
objectType = "Table"
|
|
58
|
+
if columnCount == 0:
|
|
59
59
|
new_data = {
|
|
60
60
|
"Role Name": r.Name,
|
|
61
61
|
"Object Type": objectType,
|
|
62
62
|
"Table Name": tp.Name,
|
|
63
|
-
"Object Name":
|
|
63
|
+
"Object Name": tp.Name,
|
|
64
64
|
}
|
|
65
65
|
df = pd.concat(
|
|
66
66
|
[df, pd.DataFrame(new_data, index=[0])], ignore_index=True
|
|
67
67
|
)
|
|
68
|
+
else:
|
|
69
|
+
objectType = "Column"
|
|
70
|
+
for cp in tp.ColumnPermissions:
|
|
71
|
+
new_data = {
|
|
72
|
+
"Role Name": r.Name,
|
|
73
|
+
"Object Type": objectType,
|
|
74
|
+
"Table Name": tp.Name,
|
|
75
|
+
"Object Name": cp.Name,
|
|
76
|
+
}
|
|
77
|
+
df = pd.concat(
|
|
78
|
+
[df, pd.DataFrame(new_data, index=[0])],
|
|
79
|
+
ignore_index=True,
|
|
80
|
+
)
|
|
68
81
|
|
|
69
|
-
|
|
82
|
+
return df
|
|
70
83
|
|
|
71
84
|
|
|
72
|
-
def list_tables(dataset: str, workspace: Optional[str] = None):
|
|
85
|
+
def list_tables(dataset: str, workspace: Optional[str] = None) -> pd.DataFrame:
|
|
73
86
|
"""
|
|
74
87
|
Shows a semantic model's tables and their properties.
|
|
75
88
|
|
|
@@ -88,54 +101,23 @@ def list_tables(dataset: str, workspace: Optional[str] = None):
|
|
|
88
101
|
A pandas dataframe showing the semantic model's tables and their properties.
|
|
89
102
|
"""
|
|
90
103
|
|
|
91
|
-
|
|
92
|
-
workspace_id = fabric.get_workspace_id()
|
|
93
|
-
workspace = fabric.resolve_workspace_name(workspace_id)
|
|
94
|
-
|
|
95
|
-
tom_server = fabric.create_tom_server(readonly=True, workspace=workspace)
|
|
96
|
-
m = tom_server.Databases.GetByName(dataset).Model
|
|
104
|
+
workspace = fabric.resolve_workspace_name()
|
|
97
105
|
|
|
98
|
-
df =
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
"Hidden",
|
|
103
|
-
"Data Category",
|
|
104
|
-
"Description",
|
|
105
|
-
"Refresh Policy",
|
|
106
|
-
"Source Expression",
|
|
107
|
-
]
|
|
106
|
+
df = fabric.list_tables(
|
|
107
|
+
dataset=dataset,
|
|
108
|
+
workspace=workspace,
|
|
109
|
+
additional_xmla_properties=["RefreshPolicy", "RefreshPolicy.SourceExpression"],
|
|
108
110
|
)
|
|
109
111
|
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
if str(t.CalculationGroup) != "None":
|
|
115
|
-
tableType = "Calculation Group"
|
|
116
|
-
else:
|
|
117
|
-
for p in t.Partitions:
|
|
118
|
-
if str(p.SourceType) == "Calculated":
|
|
119
|
-
tableType = "Calculated Table"
|
|
120
|
-
|
|
121
|
-
if rPolicy:
|
|
122
|
-
sourceExpression = t.RefreshPolicy.SourceExpression
|
|
123
|
-
|
|
124
|
-
new_data = {
|
|
125
|
-
"Name": t.Name,
|
|
126
|
-
"Type": tableType,
|
|
127
|
-
"Hidden": t.IsHidden,
|
|
128
|
-
"Data Category": t.DataCategory,
|
|
129
|
-
"Description": t.Description,
|
|
130
|
-
"Refresh Policy": rPolicy,
|
|
131
|
-
"Source Expression": sourceExpression,
|
|
132
|
-
}
|
|
133
|
-
df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
|
|
112
|
+
df["Refresh Policy"] = df["Refresh Policy"].notna()
|
|
113
|
+
df.rename(
|
|
114
|
+
columns={"Refresh Policy Source Expression": "Source Expression"}, inplace=True
|
|
115
|
+
)
|
|
134
116
|
|
|
135
117
|
return df
|
|
136
118
|
|
|
137
119
|
|
|
138
|
-
def list_annotations(dataset: str, workspace: Optional[str] = None):
|
|
120
|
+
def list_annotations(dataset: str, workspace: Optional[str] = None) -> pd.DataFrame:
|
|
139
121
|
"""
|
|
140
122
|
Shows a semantic model's annotations and their properties.
|
|
141
123
|
|
|
@@ -154,12 +136,9 @@ def list_annotations(dataset: str, workspace: Optional[str] = None):
|
|
|
154
136
|
A pandas dataframe showing the semantic model's annotations and their properties.
|
|
155
137
|
"""
|
|
156
138
|
|
|
157
|
-
|
|
158
|
-
workspace_id = fabric.get_workspace_id()
|
|
159
|
-
workspace = fabric.resolve_workspace_name(workspace_id)
|
|
139
|
+
from sempy_labs.tom import connect_semantic_model
|
|
160
140
|
|
|
161
|
-
|
|
162
|
-
m = tom_server.Databases.GetByName(dataset).Model
|
|
141
|
+
workspace = fabric.resolve_workspace_name()
|
|
163
142
|
|
|
164
143
|
df = pd.DataFrame(
|
|
165
144
|
columns=[
|
|
@@ -171,183 +150,201 @@ def list_annotations(dataset: str, workspace: Optional[str] = None):
|
|
|
171
150
|
]
|
|
172
151
|
)
|
|
173
152
|
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
"Annotation Name": aName,
|
|
184
|
-
"Annotation Value": aValue,
|
|
185
|
-
}
|
|
186
|
-
df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
|
|
187
|
-
for t in m.Tables:
|
|
188
|
-
objectType = "Table"
|
|
189
|
-
tName = t.Name
|
|
190
|
-
for ta in t.Annotations:
|
|
191
|
-
taName = ta.Name
|
|
192
|
-
taValue = ta.Value
|
|
153
|
+
with connect_semantic_model(
|
|
154
|
+
dataset=dataset, readonly=True, workspace=workspace
|
|
155
|
+
) as tom:
|
|
156
|
+
|
|
157
|
+
mName = tom.model.Name
|
|
158
|
+
for a in tom.model.Annotations:
|
|
159
|
+
objectType = "Model"
|
|
160
|
+
aName = a.Name
|
|
161
|
+
aValue = a.Value
|
|
193
162
|
new_data = {
|
|
194
|
-
"Object Name":
|
|
195
|
-
"Parent Object Name":
|
|
163
|
+
"Object Name": mName,
|
|
164
|
+
"Parent Object Name": None,
|
|
196
165
|
"Object Type": objectType,
|
|
197
|
-
"Annotation Name":
|
|
198
|
-
"Annotation Value":
|
|
166
|
+
"Annotation Name": aName,
|
|
167
|
+
"Annotation Value": aValue,
|
|
199
168
|
}
|
|
200
169
|
df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
|
|
201
|
-
for
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
for
|
|
205
|
-
|
|
206
|
-
|
|
170
|
+
for t in tom.model.Tables:
|
|
171
|
+
objectType = "Table"
|
|
172
|
+
tName = t.Name
|
|
173
|
+
for ta in t.Annotations:
|
|
174
|
+
taName = ta.Name
|
|
175
|
+
taValue = ta.Value
|
|
176
|
+
new_data = {
|
|
177
|
+
"Object Name": tName,
|
|
178
|
+
"Parent Object Name": mName,
|
|
179
|
+
"Object Type": objectType,
|
|
180
|
+
"Annotation Name": taName,
|
|
181
|
+
"Annotation Value": taValue,
|
|
182
|
+
}
|
|
183
|
+
df = pd.concat(
|
|
184
|
+
[df, pd.DataFrame(new_data, index=[0])], ignore_index=True
|
|
185
|
+
)
|
|
186
|
+
for p in t.Partitions:
|
|
187
|
+
pName = p.Name
|
|
188
|
+
objectType = "Partition"
|
|
189
|
+
for pa in p.Annotations:
|
|
190
|
+
paName = pa.Name
|
|
191
|
+
paValue = pa.Value
|
|
192
|
+
new_data = {
|
|
193
|
+
"Object Name": pName,
|
|
194
|
+
"Parent Object Name": tName,
|
|
195
|
+
"Object Type": objectType,
|
|
196
|
+
"Annotation Name": paName,
|
|
197
|
+
"Annotation Value": paValue,
|
|
198
|
+
}
|
|
199
|
+
df = pd.concat(
|
|
200
|
+
[df, pd.DataFrame(new_data, index=[0])], ignore_index=True
|
|
201
|
+
)
|
|
202
|
+
for c in t.Columns:
|
|
203
|
+
objectType = "Column"
|
|
204
|
+
cName = c.Name
|
|
205
|
+
for ca in c.Annotations:
|
|
206
|
+
caName = ca.Name
|
|
207
|
+
caValue = ca.Value
|
|
208
|
+
new_data = {
|
|
209
|
+
"Object Name": cName,
|
|
210
|
+
"Parent Object Name": tName,
|
|
211
|
+
"Object Type": objectType,
|
|
212
|
+
"Annotation Name": caName,
|
|
213
|
+
"Annotation Value": caValue,
|
|
214
|
+
}
|
|
215
|
+
df = pd.concat(
|
|
216
|
+
[df, pd.DataFrame(new_data, index=[0])], ignore_index=True
|
|
217
|
+
)
|
|
218
|
+
for ms in t.Measures:
|
|
219
|
+
objectType = "Measure"
|
|
220
|
+
measName = ms.Name
|
|
221
|
+
for ma in ms.Annotations:
|
|
222
|
+
maName = ma.Name
|
|
223
|
+
maValue = ma.Value
|
|
224
|
+
new_data = {
|
|
225
|
+
"Object Name": measName,
|
|
226
|
+
"Parent Object Name": tName,
|
|
227
|
+
"Object Type": objectType,
|
|
228
|
+
"Annotation Name": maName,
|
|
229
|
+
"Annotation Value": maValue,
|
|
230
|
+
}
|
|
231
|
+
df = pd.concat(
|
|
232
|
+
[df, pd.DataFrame(new_data, index=[0])], ignore_index=True
|
|
233
|
+
)
|
|
234
|
+
for h in t.Hierarchies:
|
|
235
|
+
objectType = "Hierarchy"
|
|
236
|
+
hName = h.Name
|
|
237
|
+
for ha in h.Annotations:
|
|
238
|
+
haName = ha.Name
|
|
239
|
+
haValue = ha.Value
|
|
240
|
+
new_data = {
|
|
241
|
+
"Object Name": hName,
|
|
242
|
+
"Parent Object Name": tName,
|
|
243
|
+
"Object Type": objectType,
|
|
244
|
+
"Annotation Name": haName,
|
|
245
|
+
"Annotation Value": haValue,
|
|
246
|
+
}
|
|
247
|
+
df = pd.concat(
|
|
248
|
+
[df, pd.DataFrame(new_data, index=[0])], ignore_index=True
|
|
249
|
+
)
|
|
250
|
+
for d in tom.model.DataSources:
|
|
251
|
+
dName = d.Name
|
|
252
|
+
objectType = "Data Source"
|
|
253
|
+
for da in d.Annotations:
|
|
254
|
+
daName = da.Name
|
|
255
|
+
daValue = da.Value
|
|
207
256
|
new_data = {
|
|
208
|
-
"Object Name":
|
|
209
|
-
"Parent Object Name":
|
|
257
|
+
"Object Name": dName,
|
|
258
|
+
"Parent Object Name": mName,
|
|
210
259
|
"Object Type": objectType,
|
|
211
|
-
"Annotation Name":
|
|
212
|
-
"Annotation Value":
|
|
260
|
+
"Annotation Name": daName,
|
|
261
|
+
"Annotation Value": daValue,
|
|
213
262
|
}
|
|
214
263
|
df = pd.concat(
|
|
215
264
|
[df, pd.DataFrame(new_data, index=[0])], ignore_index=True
|
|
216
265
|
)
|
|
217
|
-
for
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
for
|
|
221
|
-
|
|
222
|
-
|
|
266
|
+
for r in tom.model.Relationships:
|
|
267
|
+
rName = r.Name
|
|
268
|
+
objectType = "Relationship"
|
|
269
|
+
for ra in r.Annotations:
|
|
270
|
+
raName = ra.Name
|
|
271
|
+
raValue = ra.Value
|
|
223
272
|
new_data = {
|
|
224
|
-
"Object Name":
|
|
225
|
-
"Parent Object Name":
|
|
273
|
+
"Object Name": rName,
|
|
274
|
+
"Parent Object Name": mName,
|
|
226
275
|
"Object Type": objectType,
|
|
227
|
-
"Annotation Name":
|
|
228
|
-
"Annotation Value":
|
|
276
|
+
"Annotation Name": raName,
|
|
277
|
+
"Annotation Value": raValue,
|
|
229
278
|
}
|
|
230
279
|
df = pd.concat(
|
|
231
280
|
[df, pd.DataFrame(new_data, index=[0])], ignore_index=True
|
|
232
281
|
)
|
|
233
|
-
for
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
for
|
|
237
|
-
|
|
238
|
-
|
|
282
|
+
for cul in tom.model.Cultures:
|
|
283
|
+
culName = cul.Name
|
|
284
|
+
objectType = "Translation"
|
|
285
|
+
for cula in cul.Annotations:
|
|
286
|
+
culaName = cula.Name
|
|
287
|
+
culaValue = cula.Value
|
|
239
288
|
new_data = {
|
|
240
|
-
"Object Name":
|
|
241
|
-
"Parent Object Name":
|
|
289
|
+
"Object Name": culName,
|
|
290
|
+
"Parent Object Name": mName,
|
|
242
291
|
"Object Type": objectType,
|
|
243
|
-
"Annotation Name":
|
|
244
|
-
"Annotation Value":
|
|
292
|
+
"Annotation Name": culaName,
|
|
293
|
+
"Annotation Value": culaValue,
|
|
245
294
|
}
|
|
246
295
|
df = pd.concat(
|
|
247
296
|
[df, pd.DataFrame(new_data, index=[0])], ignore_index=True
|
|
248
297
|
)
|
|
249
|
-
for
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
for
|
|
253
|
-
|
|
254
|
-
|
|
298
|
+
for e in tom.model.Expressions:
|
|
299
|
+
eName = e.Name
|
|
300
|
+
objectType = "Expression"
|
|
301
|
+
for ea in e.Annotations:
|
|
302
|
+
eaName = ea.Name
|
|
303
|
+
eaValue = ea.Value
|
|
255
304
|
new_data = {
|
|
256
|
-
"Object Name":
|
|
257
|
-
"Parent Object Name":
|
|
305
|
+
"Object Name": eName,
|
|
306
|
+
"Parent Object Name": mName,
|
|
258
307
|
"Object Type": objectType,
|
|
259
|
-
"Annotation Name":
|
|
260
|
-
"Annotation Value":
|
|
308
|
+
"Annotation Name": eaName,
|
|
309
|
+
"Annotation Value": eaValue,
|
|
310
|
+
}
|
|
311
|
+
df = pd.concat(
|
|
312
|
+
[df, pd.DataFrame(new_data, index=[0])], ignore_index=True
|
|
313
|
+
)
|
|
314
|
+
for per in tom.model.Perspectives:
|
|
315
|
+
perName = per.Name
|
|
316
|
+
objectType = "Perspective"
|
|
317
|
+
for pera in per.Annotations:
|
|
318
|
+
peraName = pera.Name
|
|
319
|
+
peraValue = pera.Value
|
|
320
|
+
new_data = {
|
|
321
|
+
"Object Name": perName,
|
|
322
|
+
"Parent Object Name": mName,
|
|
323
|
+
"Object Type": objectType,
|
|
324
|
+
"Annotation Name": peraName,
|
|
325
|
+
"Annotation Value": peraValue,
|
|
326
|
+
}
|
|
327
|
+
df = pd.concat(
|
|
328
|
+
[df, pd.DataFrame(new_data, index=[0])], ignore_index=True
|
|
329
|
+
)
|
|
330
|
+
for rol in tom.model.Roles:
|
|
331
|
+
rolName = rol.Name
|
|
332
|
+
objectType = "Role"
|
|
333
|
+
for rola in rol.Annotations:
|
|
334
|
+
rolaName = rola.Name
|
|
335
|
+
rolaValue = rola.Value
|
|
336
|
+
new_data = {
|
|
337
|
+
"Object Name": rolName,
|
|
338
|
+
"Parent Object Name": mName,
|
|
339
|
+
"Object Type": objectType,
|
|
340
|
+
"Annotation Name": rolaName,
|
|
341
|
+
"Annotation Value": rolaValue,
|
|
261
342
|
}
|
|
262
343
|
df = pd.concat(
|
|
263
344
|
[df, pd.DataFrame(new_data, index=[0])], ignore_index=True
|
|
264
345
|
)
|
|
265
|
-
for d in m.DataSources:
|
|
266
|
-
dName = d.Name
|
|
267
|
-
objectType = "Data Source"
|
|
268
|
-
for da in d.Annotations:
|
|
269
|
-
daName = da.Name
|
|
270
|
-
daValue = da.Value
|
|
271
|
-
new_data = {
|
|
272
|
-
"Object Name": dName,
|
|
273
|
-
"Parent Object Name": mName,
|
|
274
|
-
"Object Type": objectType,
|
|
275
|
-
"Annotation Name": daName,
|
|
276
|
-
"Annotation Value": daValue,
|
|
277
|
-
}
|
|
278
|
-
df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
|
|
279
|
-
for r in m.Relationships:
|
|
280
|
-
rName = r.Name
|
|
281
|
-
objectType = "Relationship"
|
|
282
|
-
for ra in r.Annotations:
|
|
283
|
-
raName = ra.Name
|
|
284
|
-
raValue = ra.Value
|
|
285
|
-
new_data = {
|
|
286
|
-
"Object Name": rName,
|
|
287
|
-
"Parent Object Name": mName,
|
|
288
|
-
"Object Type": objectType,
|
|
289
|
-
"Annotation Name": raName,
|
|
290
|
-
"Annotation Value": raValue,
|
|
291
|
-
}
|
|
292
|
-
df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
|
|
293
|
-
for cul in m.Cultures:
|
|
294
|
-
culName = cul.Name
|
|
295
|
-
objectType = "Translation"
|
|
296
|
-
for cula in cul.Annotations:
|
|
297
|
-
culaName = cula.Name
|
|
298
|
-
culaValue = cula.Value
|
|
299
|
-
new_data = {
|
|
300
|
-
"Object Name": culName,
|
|
301
|
-
"Parent Object Name": mName,
|
|
302
|
-
"Object Type": objectType,
|
|
303
|
-
"Annotation Name": culaName,
|
|
304
|
-
"Annotation Value": culaValue,
|
|
305
|
-
}
|
|
306
|
-
df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
|
|
307
|
-
for e in m.Expressions:
|
|
308
|
-
eName = e.Name
|
|
309
|
-
objectType = "Expression"
|
|
310
|
-
for ea in e.Annotations:
|
|
311
|
-
eaName = ea.Name
|
|
312
|
-
eaValue = ea.Value
|
|
313
|
-
new_data = {
|
|
314
|
-
"Object Name": eName,
|
|
315
|
-
"Parent Object Name": mName,
|
|
316
|
-
"Object Type": objectType,
|
|
317
|
-
"Annotation Name": eaName,
|
|
318
|
-
"Annotation Value": eaValue,
|
|
319
|
-
}
|
|
320
|
-
df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
|
|
321
|
-
for per in m.Perspectives:
|
|
322
|
-
perName = per.Name
|
|
323
|
-
objectType = "Perspective"
|
|
324
|
-
for pera in per.Annotations:
|
|
325
|
-
peraName = pera.Name
|
|
326
|
-
peraValue = pera.Value
|
|
327
|
-
new_data = {
|
|
328
|
-
"Object Name": perName,
|
|
329
|
-
"Parent Object Name": mName,
|
|
330
|
-
"Object Type": objectType,
|
|
331
|
-
"Annotation Name": peraName,
|
|
332
|
-
"Annotation Value": peraValue,
|
|
333
|
-
}
|
|
334
|
-
df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
|
|
335
|
-
for rol in m.Roles:
|
|
336
|
-
rolName = rol.Name
|
|
337
|
-
objectType = "Role"
|
|
338
|
-
for rola in rol.Annotations:
|
|
339
|
-
rolaName = rola.Name
|
|
340
|
-
rolaValue = rola.Value
|
|
341
|
-
new_data = {
|
|
342
|
-
"Object Name": rolName,
|
|
343
|
-
"Parent Object Name": mName,
|
|
344
|
-
"Object Type": objectType,
|
|
345
|
-
"Annotation Name": rolaName,
|
|
346
|
-
"Annotation Value": rolaValue,
|
|
347
|
-
}
|
|
348
|
-
df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
|
|
349
346
|
|
|
350
|
-
|
|
347
|
+
return df
|
|
351
348
|
|
|
352
349
|
|
|
353
350
|
def list_columns(
|
|
@@ -355,7 +352,7 @@ def list_columns(
|
|
|
355
352
|
workspace: Optional[str] = None,
|
|
356
353
|
lakehouse: Optional[str] = None,
|
|
357
354
|
lakehouse_workspace: Optional[str] = None,
|
|
358
|
-
):
|
|
355
|
+
) -> pd.DataFrame:
|
|
359
356
|
"""
|
|
360
357
|
Shows a semantic model's columns and their properties.
|
|
361
358
|
|
|
@@ -385,8 +382,7 @@ def list_columns(
|
|
|
385
382
|
)
|
|
386
383
|
|
|
387
384
|
if workspace is None:
|
|
388
|
-
|
|
389
|
-
workspace = fabric.resolve_workspace_name(workspace_id)
|
|
385
|
+
workspace = fabric.resolve_workspace_name()
|
|
390
386
|
|
|
391
387
|
dfP = fabric.list_partitions(dataset=dataset, workspace=workspace)
|
|
392
388
|
|
|
@@ -453,7 +449,7 @@ def list_columns(
|
|
|
453
449
|
return dfC
|
|
454
450
|
|
|
455
451
|
|
|
456
|
-
def list_dashboards(workspace: Optional[str] = None):
|
|
452
|
+
def list_dashboards(workspace: Optional[str] = None) -> pd.DataFrame:
|
|
457
453
|
"""
|
|
458
454
|
Shows a list of the dashboards within a workspace.
|
|
459
455
|
|
|
@@ -493,24 +489,15 @@ def list_dashboards(workspace: Optional[str] = None):
|
|
|
493
489
|
response = client.get(f"/v1.0/myorg/groups/{workspace_id}/dashboards")
|
|
494
490
|
|
|
495
491
|
for v in response.json()["value"]:
|
|
496
|
-
dashboardID = v["id"]
|
|
497
|
-
displayName = v["displayName"]
|
|
498
|
-
isReadOnly = v["isReadOnly"]
|
|
499
|
-
webURL = v["webUrl"]
|
|
500
|
-
embedURL = v["embedUrl"]
|
|
501
|
-
dataClass = v["dataClassification"]
|
|
502
|
-
users = v["users"]
|
|
503
|
-
subs = v["subscriptions"]
|
|
504
|
-
|
|
505
492
|
new_data = {
|
|
506
|
-
"Dashboard ID":
|
|
507
|
-
"Dashboard Name": displayName,
|
|
508
|
-
"Read Only": isReadOnly,
|
|
509
|
-
"Web URL":
|
|
510
|
-
"Embed URL":
|
|
511
|
-
"Data Classification":
|
|
512
|
-
"Users": [users],
|
|
513
|
-
"Subscriptions": [
|
|
493
|
+
"Dashboard ID": v.get("id"),
|
|
494
|
+
"Dashboard Name": v.get("displayName"),
|
|
495
|
+
"Read Only": v.get("isReadOnly"),
|
|
496
|
+
"Web URL": v.get("webUrl"),
|
|
497
|
+
"Embed URL": v.get("embedUrl"),
|
|
498
|
+
"Data Classification": v.get("dataClassification"),
|
|
499
|
+
"Users": [v.get("users")],
|
|
500
|
+
"Subscriptions": [v.get("subscriptions")],
|
|
514
501
|
}
|
|
515
502
|
df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
|
|
516
503
|
|
|
@@ -519,7 +506,7 @@ def list_dashboards(workspace: Optional[str] = None):
|
|
|
519
506
|
return df
|
|
520
507
|
|
|
521
508
|
|
|
522
|
-
def list_lakehouses(workspace: Optional[str] = None):
|
|
509
|
+
def list_lakehouses(workspace: Optional[str] = None) -> pd.DataFrame:
|
|
523
510
|
"""
|
|
524
511
|
Shows the lakehouses within a workspace.
|
|
525
512
|
|
|
@@ -555,33 +542,25 @@ def list_lakehouses(workspace: Optional[str] = None):
|
|
|
555
542
|
response = client.get(f"/v1/workspaces/{workspace_id}/lakehouses/")
|
|
556
543
|
|
|
557
544
|
for v in response.json()["value"]:
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
lakehouseDesc = v["description"]
|
|
561
|
-
prop = v["properties"]
|
|
562
|
-
oneLakeTP = prop["oneLakeTablesPath"]
|
|
563
|
-
oneLakeFP = prop["oneLakeFilesPath"]
|
|
564
|
-
sqlEPProp = prop["sqlEndpointProperties"]
|
|
565
|
-
sqlEPCS = sqlEPProp["connectionString"]
|
|
566
|
-
sqlepid = sqlEPProp["id"]
|
|
567
|
-
sqlepstatus = sqlEPProp["provisioningStatus"]
|
|
545
|
+
prop = v.get("properties", {})
|
|
546
|
+
sqlEPProp = prop.get("sqlEndpointProperties", {})
|
|
568
547
|
|
|
569
548
|
new_data = {
|
|
570
|
-
"Lakehouse Name":
|
|
571
|
-
"Lakehouse ID":
|
|
572
|
-
"Description":
|
|
573
|
-
"OneLake Tables Path":
|
|
574
|
-
"OneLake Files Path":
|
|
575
|
-
"SQL Endpoint Connection String":
|
|
576
|
-
"SQL Endpoint ID":
|
|
577
|
-
"SQL Endpoint Provisioning Status":
|
|
549
|
+
"Lakehouse Name": v.get("displayName"),
|
|
550
|
+
"Lakehouse ID": v.get("id"),
|
|
551
|
+
"Description": v.get("description"),
|
|
552
|
+
"OneLake Tables Path": prop.get("oneLakeTablesPath"),
|
|
553
|
+
"OneLake Files Path": prop.get("oneLakeFilesPath"),
|
|
554
|
+
"SQL Endpoint Connection String": sqlEPProp.get("connectionString"),
|
|
555
|
+
"SQL Endpoint ID": sqlEPProp.get("id"),
|
|
556
|
+
"SQL Endpoint Provisioning Status": sqlEPProp.get("provisioningStatus"),
|
|
578
557
|
}
|
|
579
558
|
df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
|
|
580
559
|
|
|
581
560
|
return df
|
|
582
561
|
|
|
583
562
|
|
|
584
|
-
def list_warehouses(workspace: Optional[str] = None):
|
|
563
|
+
def list_warehouses(workspace: Optional[str] = None) -> pd.DataFrame:
|
|
585
564
|
"""
|
|
586
565
|
Shows the warehouses within a workspace.
|
|
587
566
|
|
|
@@ -615,28 +594,22 @@ def list_warehouses(workspace: Optional[str] = None):
|
|
|
615
594
|
response = client.get(f"/v1/workspaces/{workspace_id}/warehouses/")
|
|
616
595
|
|
|
617
596
|
for v in response.json()["value"]:
|
|
618
|
-
|
|
619
|
-
warehouse_name = v["displayName"]
|
|
620
|
-
desc = v["description"]
|
|
621
|
-
prop = v["properties"]
|
|
622
|
-
connInfo = prop["connectionInfo"]
|
|
623
|
-
createdDate = prop["createdDate"]
|
|
624
|
-
lastUpdate = prop["lastUpdatedTime"]
|
|
597
|
+
prop = v.get("properties", {})
|
|
625
598
|
|
|
626
599
|
new_data = {
|
|
627
|
-
"Warehouse Name":
|
|
628
|
-
"Warehouse ID":
|
|
629
|
-
"Description":
|
|
630
|
-
"Connection Info":
|
|
631
|
-
"Created Date": createdDate,
|
|
632
|
-
"Last Updated Time":
|
|
600
|
+
"Warehouse Name": v.get("displayName"),
|
|
601
|
+
"Warehouse ID": v.get("id"),
|
|
602
|
+
"Description": v.get("description"),
|
|
603
|
+
"Connection Info": prop.get("connectionInfo"),
|
|
604
|
+
"Created Date": prop.get("createdDate"),
|
|
605
|
+
"Last Updated Time": prop.get("lastUpdatedTime"),
|
|
633
606
|
}
|
|
634
607
|
df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
|
|
635
608
|
|
|
636
609
|
return df
|
|
637
610
|
|
|
638
611
|
|
|
639
|
-
def list_sqlendpoints(workspace: Optional[str] = None):
|
|
612
|
+
def list_sqlendpoints(workspace: Optional[str] = None) -> pd.DataFrame:
|
|
640
613
|
"""
|
|
641
614
|
Shows the SQL Endpoints within a workspace.
|
|
642
615
|
|
|
@@ -661,21 +634,18 @@ def list_sqlendpoints(workspace: Optional[str] = None):
|
|
|
661
634
|
response = client.get(f"/v1/workspaces/{workspace_id}/sqlEndpoints/")
|
|
662
635
|
|
|
663
636
|
for v in response.json()["value"]:
|
|
664
|
-
sql_id = v["id"]
|
|
665
|
-
lake_name = v["displayName"]
|
|
666
|
-
desc = v["description"]
|
|
667
637
|
|
|
668
638
|
new_data = {
|
|
669
|
-
"SQL Endpoint ID":
|
|
670
|
-
"SQL Endpoint Name":
|
|
671
|
-
"Description":
|
|
639
|
+
"SQL Endpoint ID": v.get("id"),
|
|
640
|
+
"SQL Endpoint Name": v.get("displayName"),
|
|
641
|
+
"Description": v.get("description"),
|
|
672
642
|
}
|
|
673
643
|
df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
|
|
674
644
|
|
|
675
645
|
return df
|
|
676
646
|
|
|
677
647
|
|
|
678
|
-
def list_mirroredwarehouses(workspace: Optional[str] = None):
|
|
648
|
+
def list_mirroredwarehouses(workspace: Optional[str] = None) -> pd.DataFrame:
|
|
679
649
|
"""
|
|
680
650
|
Shows the mirrored warehouses within a workspace.
|
|
681
651
|
|
|
@@ -702,21 +672,18 @@ def list_mirroredwarehouses(workspace: Optional[str] = None):
|
|
|
702
672
|
response = client.get(f"/v1/workspaces/{workspace_id}/mirroredWarehouses/")
|
|
703
673
|
|
|
704
674
|
for v in response.json()["value"]:
|
|
705
|
-
mirr_id = v["id"]
|
|
706
|
-
dbname = v["displayName"]
|
|
707
|
-
desc = v["description"]
|
|
708
675
|
|
|
709
676
|
new_data = {
|
|
710
|
-
"Mirrored Warehouse":
|
|
711
|
-
"Mirrored Warehouse ID":
|
|
712
|
-
"Description":
|
|
677
|
+
"Mirrored Warehouse": v.get("displayName"),
|
|
678
|
+
"Mirrored Warehouse ID": v.get("id"),
|
|
679
|
+
"Description": v.get("description"),
|
|
713
680
|
}
|
|
714
681
|
df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
|
|
715
682
|
|
|
716
683
|
return df
|
|
717
684
|
|
|
718
685
|
|
|
719
|
-
def list_kqldatabases(workspace: Optional[str] = None):
|
|
686
|
+
def list_kqldatabases(workspace: Optional[str] = None) -> pd.DataFrame:
|
|
720
687
|
"""
|
|
721
688
|
Shows the KQL databases within a workspace.
|
|
722
689
|
|
|
@@ -751,30 +718,23 @@ def list_kqldatabases(workspace: Optional[str] = None):
|
|
|
751
718
|
response = client.get(f"/v1/workspaces/{workspace_id}/kqlDatabases/")
|
|
752
719
|
|
|
753
720
|
for v in response.json()["value"]:
|
|
754
|
-
|
|
755
|
-
kql_name = v["displayName"]
|
|
756
|
-
desc = v["description"]
|
|
757
|
-
prop = v["properties"]
|
|
758
|
-
eventId = prop["parentEventhouseItemId"]
|
|
759
|
-
qsURI = prop["queryServiceUri"]
|
|
760
|
-
isURI = prop["ingestionServiceUri"]
|
|
761
|
-
dbType = prop["kustoDatabaseType"]
|
|
721
|
+
prop = v.get("properties", {})
|
|
762
722
|
|
|
763
723
|
new_data = {
|
|
764
|
-
"KQL Database Name":
|
|
765
|
-
"KQL Database ID":
|
|
766
|
-
"Description":
|
|
767
|
-
"Parent Eventhouse Item ID":
|
|
768
|
-
"Query Service URI":
|
|
769
|
-
"Ingestion Service URI":
|
|
770
|
-
"Kusto Database Type":
|
|
724
|
+
"KQL Database Name": v.get("displayName"),
|
|
725
|
+
"KQL Database ID": v.get("id"),
|
|
726
|
+
"Description": v.get("description"),
|
|
727
|
+
"Parent Eventhouse Item ID": prop.get("parentEventhouseItemId"),
|
|
728
|
+
"Query Service URI": prop.get("queryServiceUri"),
|
|
729
|
+
"Ingestion Service URI": prop.get("ingestionServiceUri"),
|
|
730
|
+
"Kusto Database Type": prop.get("kustoDatabaseType"),
|
|
771
731
|
}
|
|
772
732
|
df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
|
|
773
733
|
|
|
774
734
|
return df
|
|
775
735
|
|
|
776
736
|
|
|
777
|
-
def list_kqlquerysets(workspace: Optional[str] = None):
|
|
737
|
+
def list_kqlquerysets(workspace: Optional[str] = None) -> pd.DataFrame:
|
|
778
738
|
"""
|
|
779
739
|
Shows the KQL Querysets within a workspace.
|
|
780
740
|
|
|
@@ -799,21 +759,18 @@ def list_kqlquerysets(workspace: Optional[str] = None):
|
|
|
799
759
|
response = client.get(f"/v1/workspaces/{workspace_id}/kqlQuerysets/")
|
|
800
760
|
|
|
801
761
|
for v in response.json()["value"]:
|
|
802
|
-
kql_id = v["id"]
|
|
803
|
-
kql_name = v["displayName"]
|
|
804
|
-
desc = v["description"]
|
|
805
762
|
|
|
806
763
|
new_data = {
|
|
807
|
-
"KQL Queryset Name":
|
|
808
|
-
"KQL Queryset ID":
|
|
809
|
-
"Description":
|
|
764
|
+
"KQL Queryset Name": v.get("displayName"),
|
|
765
|
+
"KQL Queryset ID": v.get("id"),
|
|
766
|
+
"Description": v.get("description"),
|
|
810
767
|
}
|
|
811
768
|
df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
|
|
812
769
|
|
|
813
770
|
return df
|
|
814
771
|
|
|
815
772
|
|
|
816
|
-
def list_mlmodels(workspace: Optional[str] = None):
|
|
773
|
+
def list_mlmodels(workspace: Optional[str] = None) -> pd.DataFrame:
|
|
817
774
|
"""
|
|
818
775
|
Shows the ML models within a workspace.
|
|
819
776
|
|
|
@@ -838,9 +795,9 @@ def list_mlmodels(workspace: Optional[str] = None):
|
|
|
838
795
|
response = client.get(f"/v1/workspaces/{workspace_id}/mlModels/")
|
|
839
796
|
|
|
840
797
|
for v in response.json()["value"]:
|
|
841
|
-
model_id = v
|
|
842
|
-
modelName = v
|
|
843
|
-
desc = v
|
|
798
|
+
model_id = v.get("id")
|
|
799
|
+
modelName = v.get("displayName")
|
|
800
|
+
desc = v.get("description")
|
|
844
801
|
|
|
845
802
|
new_data = {
|
|
846
803
|
"ML Model Name": modelName,
|
|
@@ -852,7 +809,7 @@ def list_mlmodels(workspace: Optional[str] = None):
|
|
|
852
809
|
return df
|
|
853
810
|
|
|
854
811
|
|
|
855
|
-
def list_eventstreams(workspace: Optional[str] = None):
|
|
812
|
+
def list_eventstreams(workspace: Optional[str] = None) -> pd.DataFrame:
|
|
856
813
|
"""
|
|
857
814
|
Shows the eventstreams within a workspace.
|
|
858
815
|
|
|
@@ -877,9 +834,9 @@ def list_eventstreams(workspace: Optional[str] = None):
|
|
|
877
834
|
response = client.get(f"/v1/workspaces/{workspace_id}/eventstreams/")
|
|
878
835
|
|
|
879
836
|
for v in response.json()["value"]:
|
|
880
|
-
model_id = v
|
|
881
|
-
modelName = v
|
|
882
|
-
desc = v
|
|
837
|
+
model_id = v.get("id")
|
|
838
|
+
modelName = v.get("displayName")
|
|
839
|
+
desc = v.get("description")
|
|
883
840
|
|
|
884
841
|
new_data = {
|
|
885
842
|
"Eventstream Name": modelName,
|
|
@@ -891,7 +848,7 @@ def list_eventstreams(workspace: Optional[str] = None):
|
|
|
891
848
|
return df
|
|
892
849
|
|
|
893
850
|
|
|
894
|
-
def list_datapipelines(workspace: Optional[str] = None):
|
|
851
|
+
def list_datapipelines(workspace: Optional[str] = None) -> pd.DataFrame:
|
|
895
852
|
"""
|
|
896
853
|
Shows the data pipelines within a workspace.
|
|
897
854
|
|
|
@@ -916,9 +873,9 @@ def list_datapipelines(workspace: Optional[str] = None):
|
|
|
916
873
|
response = client.get(f"/v1/workspaces/{workspace_id}/dataPipelines/")
|
|
917
874
|
|
|
918
875
|
for v in response.json()["value"]:
|
|
919
|
-
model_id = v
|
|
920
|
-
modelName = v
|
|
921
|
-
desc = v
|
|
876
|
+
model_id = v.get("id")
|
|
877
|
+
modelName = v.get("displayName")
|
|
878
|
+
desc = v.get("description")
|
|
922
879
|
|
|
923
880
|
new_data = {
|
|
924
881
|
"Data Pipeline Name": modelName,
|
|
@@ -930,7 +887,7 @@ def list_datapipelines(workspace: Optional[str] = None):
|
|
|
930
887
|
return df
|
|
931
888
|
|
|
932
889
|
|
|
933
|
-
def list_mlexperiments(workspace: Optional[str] = None):
|
|
890
|
+
def list_mlexperiments(workspace: Optional[str] = None) -> pd.DataFrame:
|
|
934
891
|
"""
|
|
935
892
|
Shows the ML experiments within a workspace.
|
|
936
893
|
|
|
@@ -955,21 +912,18 @@ def list_mlexperiments(workspace: Optional[str] = None):
|
|
|
955
912
|
response = client.get(f"/v1/workspaces/{workspace_id}/mlExperiments/")
|
|
956
913
|
|
|
957
914
|
for v in response.json()["value"]:
|
|
958
|
-
model_id = v["id"]
|
|
959
|
-
modelName = v["displayName"]
|
|
960
|
-
desc = v["description"]
|
|
961
915
|
|
|
962
916
|
new_data = {
|
|
963
|
-
"ML Experiment Name":
|
|
964
|
-
"ML Experiment ID":
|
|
965
|
-
"Description":
|
|
917
|
+
"ML Experiment Name": v.get("displayName"),
|
|
918
|
+
"ML Experiment ID": v.get("id"),
|
|
919
|
+
"Description": v.get("description"),
|
|
966
920
|
}
|
|
967
921
|
df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
|
|
968
922
|
|
|
969
923
|
return df
|
|
970
924
|
|
|
971
925
|
|
|
972
|
-
def list_datamarts(workspace: Optional[str] = None):
|
|
926
|
+
def list_datamarts(workspace: Optional[str] = None) -> pd.DataFrame:
|
|
973
927
|
"""
|
|
974
928
|
Shows the datamarts within a workspace.
|
|
975
929
|
|
|
@@ -994,14 +948,11 @@ def list_datamarts(workspace: Optional[str] = None):
|
|
|
994
948
|
response = client.get(f"/v1/workspaces/{workspace_id}/datamarts/")
|
|
995
949
|
|
|
996
950
|
for v in response.json()["value"]:
|
|
997
|
-
model_id = v["id"]
|
|
998
|
-
modelName = v["displayName"]
|
|
999
|
-
desc = v["description"]
|
|
1000
951
|
|
|
1001
952
|
new_data = {
|
|
1002
|
-
"Datamart Name":
|
|
1003
|
-
"Datamart ID":
|
|
1004
|
-
"Description":
|
|
953
|
+
"Datamart Name": v.get("displayName"),
|
|
954
|
+
"Datamart ID": v.get("id"),
|
|
955
|
+
"Description": v.get("description"),
|
|
1005
956
|
}
|
|
1006
957
|
df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
|
|
1007
958
|
|
|
@@ -1044,7 +995,7 @@ def create_warehouse(
|
|
|
1044
995
|
|
|
1045
996
|
if response.status_code == 201:
|
|
1046
997
|
print(
|
|
1047
|
-
f"The '{warehouse}' warehouse has been created within the '{workspace}' workspace."
|
|
998
|
+
f"{icons.green_dot} The '{warehouse}' warehouse has been created within the '{workspace}' workspace."
|
|
1048
999
|
)
|
|
1049
1000
|
elif response.status_code == 202:
|
|
1050
1001
|
operationId = response.headers["x-ms-operation-id"]
|
|
@@ -1056,11 +1007,11 @@ def create_warehouse(
|
|
|
1056
1007
|
response_body = json.loads(response.content)
|
|
1057
1008
|
response = client.get(f"/v1/operations/{operationId}/result")
|
|
1058
1009
|
print(
|
|
1059
|
-
f"The '{warehouse}' warehouse has been created within the '{workspace}' workspace."
|
|
1010
|
+
f"{icons.green_dot} The '{warehouse}' warehouse has been created within the '{workspace}' workspace."
|
|
1060
1011
|
)
|
|
1061
1012
|
else:
|
|
1062
|
-
|
|
1063
|
-
f"
|
|
1013
|
+
raise ValueError(
|
|
1014
|
+
f"{icons.red_dot} Failed to create the '{warehouse}' warehouse within the '{workspace}' workspace."
|
|
1064
1015
|
)
|
|
1065
1016
|
|
|
1066
1017
|
|
|
@@ -1107,8 +1058,9 @@ def update_item(
|
|
|
1107
1058
|
item_type = item_type.replace(" ", "").capitalize()
|
|
1108
1059
|
|
|
1109
1060
|
if item_type not in itemTypes.keys():
|
|
1110
|
-
|
|
1111
|
-
|
|
1061
|
+
raise ValueError(
|
|
1062
|
+
f"{icons.red_dot} The '{item_type}' is not a valid item type. "
|
|
1063
|
+
)
|
|
1112
1064
|
|
|
1113
1065
|
itemType = itemTypes[item_type]
|
|
1114
1066
|
|
|
@@ -1116,10 +1068,9 @@ def update_item(
|
|
|
1116
1068
|
dfI_filt = dfI[(dfI["Display Name"] == current_name)]
|
|
1117
1069
|
|
|
1118
1070
|
if len(dfI_filt) == 0:
|
|
1119
|
-
|
|
1120
|
-
f"The '{current_name}' {item_type} does not exist within the '{workspace}' workspace."
|
|
1071
|
+
raise ValueError(
|
|
1072
|
+
f"{icons.red_dot} The '{current_name}' {item_type} does not exist within the '{workspace}' workspace."
|
|
1121
1073
|
)
|
|
1122
|
-
return
|
|
1123
1074
|
|
|
1124
1075
|
itemId = dfI_filt["Id"].iloc[0]
|
|
1125
1076
|
|
|
@@ -1132,24 +1083,21 @@ def update_item(
|
|
|
1132
1083
|
f"/v1/workspaces/{workspace_id}/{itemType}/{itemId}", json=request_body
|
|
1133
1084
|
)
|
|
1134
1085
|
|
|
1135
|
-
if response.status_code
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
print(
|
|
1142
|
-
f"The '{current_name}' {item_type} within the '{workspace}' workspace has been updated to be named '{new_name}' and have a description of '{description}'"
|
|
1143
|
-
)
|
|
1086
|
+
if response.status_code != 200:
|
|
1087
|
+
raise FabricHTTPException(response)
|
|
1088
|
+
if description is None:
|
|
1089
|
+
print(
|
|
1090
|
+
f"{icons.green_dot} The '{current_name}' {item_type} within the '{workspace}' workspace has been updated to be named '{new_name}'"
|
|
1091
|
+
)
|
|
1144
1092
|
else:
|
|
1145
1093
|
print(
|
|
1146
|
-
f"
|
|
1094
|
+
f"{icons.green_dot} The '{current_name}' {item_type} within the '{workspace}' workspace has been updated to be named '{new_name}' and have a description of '{description}'"
|
|
1147
1095
|
)
|
|
1148
1096
|
|
|
1149
1097
|
|
|
1150
1098
|
def list_relationships(
|
|
1151
1099
|
dataset: str, workspace: Optional[str] = None, extended: Optional[bool] = False
|
|
1152
|
-
):
|
|
1100
|
+
) -> pd.DataFrame:
|
|
1153
1101
|
"""
|
|
1154
1102
|
Shows a semantic model's relationships and their properties.
|
|
1155
1103
|
|
|
@@ -1171,8 +1119,7 @@ def list_relationships(
|
|
|
1171
1119
|
"""
|
|
1172
1120
|
|
|
1173
1121
|
if workspace is None:
|
|
1174
|
-
|
|
1175
|
-
workspace = fabric.resolve_workspace_name(workspace_id)
|
|
1122
|
+
workspace = fabric.resolve_workspace_name()
|
|
1176
1123
|
|
|
1177
1124
|
dfR = fabric.list_relationships(dataset=dataset, workspace=workspace)
|
|
1178
1125
|
|
|
@@ -1184,7 +1131,7 @@ def list_relationships(
|
|
|
1184
1131
|
dax_string="""
|
|
1185
1132
|
SELECT
|
|
1186
1133
|
[ID] AS [RelationshipID]
|
|
1187
|
-
,[Name]
|
|
1134
|
+
,[Name]
|
|
1188
1135
|
FROM $SYSTEM.TMSCHEMA_RELATIONSHIPS
|
|
1189
1136
|
""",
|
|
1190
1137
|
)
|
|
@@ -1194,7 +1141,7 @@ def list_relationships(
|
|
|
1194
1141
|
dataset=dataset,
|
|
1195
1142
|
workspace=workspace,
|
|
1196
1143
|
dax_string="""
|
|
1197
|
-
SELECT
|
|
1144
|
+
SELECT
|
|
1198
1145
|
[TABLE_ID]
|
|
1199
1146
|
,[USED_SIZE]
|
|
1200
1147
|
FROM $SYSTEM.DISCOVER_STORAGE_TABLE_COLUMN_SEGMENTS
|
|
@@ -1230,7 +1177,7 @@ def list_relationships(
|
|
|
1230
1177
|
return dfR
|
|
1231
1178
|
|
|
1232
1179
|
|
|
1233
|
-
def list_dataflow_storage_accounts():
|
|
1180
|
+
def list_dataflow_storage_accounts() -> pd.DataFrame:
|
|
1234
1181
|
"""
|
|
1235
1182
|
Shows the accessible dataflow storage accounts.
|
|
1236
1183
|
|
|
@@ -1251,17 +1198,14 @@ def list_dataflow_storage_accounts():
|
|
|
1251
1198
|
]
|
|
1252
1199
|
)
|
|
1253
1200
|
client = fabric.PowerBIRestClient()
|
|
1254
|
-
response = client.get(
|
|
1201
|
+
response = client.get("/v1.0/myorg/dataflowStorageAccounts")
|
|
1255
1202
|
|
|
1256
1203
|
for v in response.json()["value"]:
|
|
1257
|
-
dfsaId = v["id"]
|
|
1258
|
-
dfsaName = v["name"]
|
|
1259
|
-
isEnabled = v["isEnabled"]
|
|
1260
1204
|
|
|
1261
1205
|
new_data = {
|
|
1262
|
-
"Dataflow Storage Account ID":
|
|
1263
|
-
"Dataflow Storage Account Name":
|
|
1264
|
-
"Enabled": isEnabled,
|
|
1206
|
+
"Dataflow Storage Account ID": v.get("id"),
|
|
1207
|
+
"Dataflow Storage Account Name": v.get("name"),
|
|
1208
|
+
"Enabled": v.get("isEnabled"),
|
|
1265
1209
|
}
|
|
1266
1210
|
df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
|
|
1267
1211
|
|
|
@@ -1270,7 +1214,7 @@ def list_dataflow_storage_accounts():
|
|
|
1270
1214
|
return df
|
|
1271
1215
|
|
|
1272
1216
|
|
|
1273
|
-
def list_kpis(dataset: str, workspace: Optional[str] = None):
|
|
1217
|
+
def list_kpis(dataset: str, workspace: Optional[str] = None) -> pd.DataFrame:
|
|
1274
1218
|
"""
|
|
1275
1219
|
Shows a semantic model's KPIs and their properties.
|
|
1276
1220
|
|
|
@@ -1289,7 +1233,7 @@ def list_kpis(dataset: str, workspace: Optional[str] = None):
|
|
|
1289
1233
|
A pandas dataframe showing the KPIs for the semantic model.
|
|
1290
1234
|
"""
|
|
1291
1235
|
|
|
1292
|
-
from .tom import connect_semantic_model
|
|
1236
|
+
from sempy_labs.tom import connect_semantic_model
|
|
1293
1237
|
|
|
1294
1238
|
with connect_semantic_model(
|
|
1295
1239
|
dataset=dataset, workspace=workspace, readonly=True
|
|
@@ -1334,7 +1278,7 @@ def list_kpis(dataset: str, workspace: Optional[str] = None):
|
|
|
1334
1278
|
return df
|
|
1335
1279
|
|
|
1336
1280
|
|
|
1337
|
-
def list_workspace_role_assignments(workspace: Optional[str] = None):
|
|
1281
|
+
def list_workspace_role_assignments(workspace: Optional[str] = None) -> pd.DataFrame:
|
|
1338
1282
|
"""
|
|
1339
1283
|
Shows the members of a given workspace.
|
|
1340
1284
|
|
|
@@ -1359,10 +1303,12 @@ def list_workspace_role_assignments(workspace: Optional[str] = None):
|
|
|
1359
1303
|
response = client.get(f"/v1/workspaces/{workspace_id}/roleAssignments")
|
|
1360
1304
|
|
|
1361
1305
|
for i in response.json()["value"]:
|
|
1362
|
-
user_name = i
|
|
1363
|
-
role_name = i
|
|
1364
|
-
user_email =
|
|
1365
|
-
|
|
1306
|
+
user_name = i.get("principal", {}).get("displayName")
|
|
1307
|
+
role_name = i.get("role")
|
|
1308
|
+
user_email = (
|
|
1309
|
+
i.get("principal", {}).get("userDetails", {}).get("userPrincipalName")
|
|
1310
|
+
)
|
|
1311
|
+
user_type = i.get("principal", {}).get("type")
|
|
1366
1312
|
|
|
1367
1313
|
new_data = {
|
|
1368
1314
|
"User Name": user_name,
|
|
@@ -1374,7 +1320,10 @@ def list_workspace_role_assignments(workspace: Optional[str] = None):
|
|
|
1374
1320
|
|
|
1375
1321
|
return df
|
|
1376
1322
|
|
|
1377
|
-
|
|
1323
|
+
|
|
1324
|
+
def list_semantic_model_objects(
|
|
1325
|
+
dataset: str, workspace: Optional[str] = None
|
|
1326
|
+
) -> pd.DataFrame:
|
|
1378
1327
|
"""
|
|
1379
1328
|
Shows a list of semantic model objects.
|
|
1380
1329
|
|
|
@@ -1393,7 +1342,7 @@ def list_semantic_model_objects(dataset: str, workspace: Optional[str] = None):
|
|
|
1393
1342
|
pandas.DataFrame
|
|
1394
1343
|
A pandas dataframe showing a list of objects in the semantic model
|
|
1395
1344
|
"""
|
|
1396
|
-
from .tom import connect_semantic_model
|
|
1345
|
+
from sempy_labs.tom import connect_semantic_model
|
|
1397
1346
|
|
|
1398
1347
|
df = pd.DataFrame(columns=["Parent Name", "Object Name", "Object Type"])
|
|
1399
1348
|
with connect_semantic_model(
|
|
@@ -1474,11 +1423,11 @@ def list_semantic_model_objects(dataset: str, workspace: Optional[str] = None):
|
|
|
1474
1423
|
df = pd.concat(
|
|
1475
1424
|
[df, pd.DataFrame(new_data, index=[0])], ignore_index=True
|
|
1476
1425
|
)
|
|
1477
|
-
for
|
|
1426
|
+
for lev in h.Levels:
|
|
1478
1427
|
new_data = {
|
|
1479
|
-
"Parent Name":
|
|
1480
|
-
"Object Name":
|
|
1481
|
-
"Object Type": str(
|
|
1428
|
+
"Parent Name": lev.Parent.Name,
|
|
1429
|
+
"Object Name": lev.Name,
|
|
1430
|
+
"Object Type": str(lev.ObjectType),
|
|
1482
1431
|
}
|
|
1483
1432
|
df = pd.concat(
|
|
1484
1433
|
[df, pd.DataFrame(new_data, index=[0])], ignore_index=True
|
|
@@ -1535,6 +1484,7 @@ def list_semantic_model_objects(dataset: str, workspace: Optional[str] = None):
|
|
|
1535
1484
|
|
|
1536
1485
|
return df
|
|
1537
1486
|
|
|
1487
|
+
|
|
1538
1488
|
def list_shortcuts(
|
|
1539
1489
|
lakehouse: Optional[str] = None, workspace: Optional[str] = None
|
|
1540
1490
|
) -> pd.DataFrame:
|
|
@@ -1583,46 +1533,807 @@ def list_shortcuts(
|
|
|
1583
1533
|
response = client.get(
|
|
1584
1534
|
f"/v1/workspaces/{workspace_id}/items/{lakehouse_id}/shortcuts"
|
|
1585
1535
|
)
|
|
1586
|
-
if response.status_code == 200:
|
|
1587
|
-
for s in response.json()["value"]:
|
|
1588
|
-
shortcutName = s["name"]
|
|
1589
|
-
shortcutPath = s["path"]
|
|
1590
|
-
source = list(s["target"].keys())[0]
|
|
1591
|
-
(
|
|
1592
|
-
sourceLakehouseName,
|
|
1593
|
-
sourceWorkspaceName,
|
|
1594
|
-
sourcePath,
|
|
1595
|
-
connectionId,
|
|
1596
|
-
location,
|
|
1597
|
-
subpath,
|
|
1598
|
-
) = (None, None, None, None, None, None)
|
|
1599
|
-
if source == "oneLake":
|
|
1600
|
-
sourceLakehouseId = s["target"][source]["itemId"]
|
|
1601
|
-
sourcePath = s["target"][source]["path"]
|
|
1602
|
-
sourceWorkspaceId = s["target"][source]["workspaceId"]
|
|
1603
|
-
sourceWorkspaceName = fabric.resolve_workspace_name(sourceWorkspaceId)
|
|
1604
|
-
sourceLakehouseName = resolve_lakehouse_name(
|
|
1605
|
-
sourceLakehouseId, sourceWorkspaceName
|
|
1606
|
-
)
|
|
1607
|
-
else:
|
|
1608
|
-
connectionId = s["target"][source]["connectionId"]
|
|
1609
|
-
location = s["target"][source]["location"]
|
|
1610
|
-
subpath = s["target"][source]["subpath"]
|
|
1611
1536
|
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1537
|
+
if response.status_code != 200:
|
|
1538
|
+
raise FabricHTTPException(response)
|
|
1539
|
+
for s in response.json()["value"]:
|
|
1540
|
+
shortcutName = s.get("name")
|
|
1541
|
+
shortcutPath = s.get("path")
|
|
1542
|
+
source = list(s["target"].keys())[0]
|
|
1543
|
+
(
|
|
1544
|
+
sourceLakehouseName,
|
|
1545
|
+
sourceWorkspaceName,
|
|
1546
|
+
sourcePath,
|
|
1547
|
+
connectionId,
|
|
1548
|
+
location,
|
|
1549
|
+
subpath,
|
|
1550
|
+
) = (None, None, None, None, None, None)
|
|
1551
|
+
if source == "oneLake":
|
|
1552
|
+
sourceLakehouseId = s.get("target", {}).get(source, {}).get("itemId")
|
|
1553
|
+
sourcePath = s.get("target", {}).get(source, {}).get("path")
|
|
1554
|
+
sourceWorkspaceId = s.get("target", {}).get(source, {}).get("workspaceId")
|
|
1555
|
+
sourceWorkspaceName = fabric.resolve_workspace_name(sourceWorkspaceId)
|
|
1556
|
+
sourceLakehouseName = resolve_lakehouse_name(
|
|
1557
|
+
sourceLakehouseId, sourceWorkspaceName
|
|
1558
|
+
)
|
|
1559
|
+
else:
|
|
1560
|
+
connectionId = s.get("target", {}).get(source, {}).get("connectionId")
|
|
1561
|
+
location = s.get("target", {}).get(source, {}).get("location")
|
|
1562
|
+
subpath = s.get("target", {}).get(source, {}).get("subpath")
|
|
1563
|
+
|
|
1564
|
+
new_data = {
|
|
1565
|
+
"Shortcut Name": shortcutName,
|
|
1566
|
+
"Shortcut Path": shortcutPath,
|
|
1567
|
+
"Source": source,
|
|
1568
|
+
"Source Lakehouse Name": sourceLakehouseName,
|
|
1569
|
+
"Source Workspace Name": sourceWorkspaceName,
|
|
1570
|
+
"Source Path": sourcePath,
|
|
1571
|
+
"Source Connection ID": connectionId,
|
|
1572
|
+
"Source Location": location,
|
|
1573
|
+
"Source SubPath": subpath,
|
|
1574
|
+
}
|
|
1575
|
+
df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
|
|
1576
|
+
|
|
1577
|
+
return df
|
|
1578
|
+
|
|
1579
|
+
|
|
1580
|
+
def list_custom_pools(workspace: Optional[str] = None) -> pd.DataFrame:
|
|
1581
|
+
"""
|
|
1582
|
+
Lists all `custom pools <https://learn.microsoft.com/fabric/data-engineering/create-custom-spark-pools>`_ within a workspace.
|
|
1583
|
+
|
|
1584
|
+
Parameters
|
|
1585
|
+
----------
|
|
1586
|
+
workspace : str, default=None
|
|
1587
|
+
The name of the Fabric workspace.
|
|
1588
|
+
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
1589
|
+
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
1590
|
+
|
|
1591
|
+
Returns
|
|
1592
|
+
-------
|
|
1593
|
+
pandas.DataFrame
|
|
1594
|
+
A pandas dataframe showing all the custom pools within the Fabric workspace.
|
|
1595
|
+
"""
|
|
1596
|
+
|
|
1597
|
+
# https://learn.microsoft.com/rest/api/fabric/spark/custom-pools/list-workspace-custom-pools
|
|
1598
|
+
(workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
|
|
1599
|
+
|
|
1600
|
+
df = pd.DataFrame(
|
|
1601
|
+
columns=[
|
|
1602
|
+
"Custom Pool ID",
|
|
1603
|
+
"Custom Pool Name",
|
|
1604
|
+
"Type",
|
|
1605
|
+
"Node Family",
|
|
1606
|
+
"Node Size",
|
|
1607
|
+
"Auto Scale Enabled",
|
|
1608
|
+
"Auto Scale Min Node Count",
|
|
1609
|
+
"Auto Scale Max Node Count",
|
|
1610
|
+
"Dynamic Executor Allocation Enabled",
|
|
1611
|
+
"Dynamic Executor Allocation Min Executors",
|
|
1612
|
+
"Dynamic Executor Allocation Max Executors",
|
|
1613
|
+
]
|
|
1614
|
+
)
|
|
1615
|
+
|
|
1616
|
+
client = fabric.FabricRestClient()
|
|
1617
|
+
response = client.get(f"/v1/workspaces/{workspace_id}/spark/pools")
|
|
1618
|
+
|
|
1619
|
+
for i in response.json()["value"]:
|
|
1620
|
+
|
|
1621
|
+
aScale = i.get("autoScale", {})
|
|
1622
|
+
d = i.get("dynamicExecutorAllocation", {})
|
|
1623
|
+
|
|
1624
|
+
new_data = {
|
|
1625
|
+
"Custom Pool ID": i.get("id"),
|
|
1626
|
+
"Custom Pool Name": i.get("name"),
|
|
1627
|
+
"Type": i.get("type"),
|
|
1628
|
+
"Node Family": i.get("nodeFamily"),
|
|
1629
|
+
"Node Size": i.get("nodeSize"),
|
|
1630
|
+
"Auto Scale Enabled": aScale.get("enabled"),
|
|
1631
|
+
"Auto Scale Min Node Count": aScale.get("minNodeCount"),
|
|
1632
|
+
"Auto Scale Max Node Count": aScale.get("maxNodeCount"),
|
|
1633
|
+
"Dynamic Executor Allocation Enabled": d.get("enabled"),
|
|
1634
|
+
"Dynamic Executor Allocation Min Executors": d.get("minExecutors"),
|
|
1635
|
+
"Dynamic Executor Allocation Max Executors": d.get("maxExecutors"),
|
|
1636
|
+
}
|
|
1637
|
+
df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
|
|
1638
|
+
|
|
1639
|
+
bool_cols = ["Auto Scale Enabled", "Dynamic Executor Allocation Enabled"]
|
|
1640
|
+
int_cols = [
|
|
1641
|
+
"Auto Scale Min Node Count",
|
|
1642
|
+
"Auto Scale Max Node Count",
|
|
1643
|
+
"Dynamic Executor Allocation Enabled",
|
|
1644
|
+
"Dynamic Executor Allocation Min Executors",
|
|
1645
|
+
"Dynamic Executor Allocation Max Executors",
|
|
1646
|
+
]
|
|
1647
|
+
|
|
1648
|
+
df[bool_cols] = df[bool_cols].astype(bool)
|
|
1649
|
+
df[int_cols] = df[int_cols].astype(int)
|
|
1650
|
+
|
|
1651
|
+
return df
|
|
1652
|
+
|
|
1624
1653
|
|
|
1654
|
+
def create_custom_pool(
|
|
1655
|
+
pool_name: str,
|
|
1656
|
+
node_size: str,
|
|
1657
|
+
min_node_count: int,
|
|
1658
|
+
max_node_count: int,
|
|
1659
|
+
min_executors: int,
|
|
1660
|
+
max_executors: int,
|
|
1661
|
+
node_family: Optional[str] = "MemoryOptimized",
|
|
1662
|
+
auto_scale_enabled: Optional[bool] = True,
|
|
1663
|
+
dynamic_executor_allocation_enabled: Optional[bool] = True,
|
|
1664
|
+
workspace: Optional[str] = None,
|
|
1665
|
+
):
|
|
1666
|
+
"""
|
|
1667
|
+
Creates a `custom pool <https://learn.microsoft.com/fabric/data-engineering/create-custom-spark-pools>`_ within a workspace.
|
|
1668
|
+
|
|
1669
|
+
Parameters
|
|
1670
|
+
----------
|
|
1671
|
+
pool_name : str
|
|
1672
|
+
The custom pool name.
|
|
1673
|
+
node_size : str
|
|
1674
|
+
The `node size <https://learn.microsoft.com/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#nodesize>`_.
|
|
1675
|
+
min_node_count : int
|
|
1676
|
+
The `minimum node count <https://learn.microsoft.com/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#autoscaleproperties>`_.
|
|
1677
|
+
max_node_count : int
|
|
1678
|
+
The `maximum node count <https://learn.microsoft.com/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#autoscaleproperties>`_.
|
|
1679
|
+
min_executors : int
|
|
1680
|
+
The `minimum executors <https://learn.microsoft.com/en-us/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#dynamicexecutorallocationproperties>`_.
|
|
1681
|
+
max_executors : int
|
|
1682
|
+
The `maximum executors <https://learn.microsoft.com/en-us/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#dynamicexecutorallocationproperties>`_.
|
|
1683
|
+
node_family : str, default='MemoryOptimized'
|
|
1684
|
+
The `node family <https://learn.microsoft.com/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#nodefamily>`_.
|
|
1685
|
+
auto_scale_enabled : bool, default=True
|
|
1686
|
+
The status of `auto scale <https://learn.microsoft.com/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#autoscaleproperties>`_.
|
|
1687
|
+
dynamic_executor_allocation_enabled : bool, default=True
|
|
1688
|
+
The status of the `dynamic executor allocation <https://learn.microsoft.com/en-us/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#dynamicexecutorallocationproperties>`_.
|
|
1689
|
+
workspace : str, default=None
|
|
1690
|
+
The name of the Fabric workspace.
|
|
1691
|
+
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
1692
|
+
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
1693
|
+
|
|
1694
|
+
Returns
|
|
1695
|
+
-------
|
|
1696
|
+
"""
|
|
1697
|
+
|
|
1698
|
+
# https://learn.microsoft.com/en-us/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool
|
|
1699
|
+
(workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
|
|
1700
|
+
|
|
1701
|
+
request_body = {
|
|
1702
|
+
"name": pool_name,
|
|
1703
|
+
"nodeFamily": node_family,
|
|
1704
|
+
"nodeSize": node_size,
|
|
1705
|
+
"autoScale": {
|
|
1706
|
+
"enabled": auto_scale_enabled,
|
|
1707
|
+
"minNodeCount": min_node_count,
|
|
1708
|
+
"maxNodeCount": max_node_count,
|
|
1709
|
+
},
|
|
1710
|
+
"dynamicExecutorAllocation": {
|
|
1711
|
+
"enabled": dynamic_executor_allocation_enabled,
|
|
1712
|
+
"minExecutors": min_executors,
|
|
1713
|
+
"maxExecutors": max_executors,
|
|
1714
|
+
},
|
|
1715
|
+
}
|
|
1716
|
+
|
|
1717
|
+
client = fabric.FabricRestClient()
|
|
1718
|
+
response = client.post(
|
|
1719
|
+
f"/v1/workspaces/{workspace_id}/spark/pools", json=request_body
|
|
1720
|
+
)
|
|
1721
|
+
|
|
1722
|
+
if response.status_code == 201:
|
|
1723
|
+
print(
|
|
1724
|
+
f"{icons.green_dot} The '{pool_name}' spark pool has been created within the '{workspace}' workspace."
|
|
1725
|
+
)
|
|
1726
|
+
else:
|
|
1727
|
+
raise ValueError(f"{icons.red_dot} {response.status_code}")
|
|
1728
|
+
|
|
1729
|
+
|
|
1730
|
+
def update_custom_pool(
|
|
1731
|
+
pool_name: str,
|
|
1732
|
+
node_size: Optional[str] = None,
|
|
1733
|
+
min_node_count: Optional[int] = None,
|
|
1734
|
+
max_node_count: Optional[int] = None,
|
|
1735
|
+
min_executors: Optional[int] = None,
|
|
1736
|
+
max_executors: Optional[int] = None,
|
|
1737
|
+
node_family: Optional[str] = None,
|
|
1738
|
+
auto_scale_enabled: Optional[bool] = None,
|
|
1739
|
+
dynamic_executor_allocation_enabled: Optional[bool] = None,
|
|
1740
|
+
workspace: Optional[str] = None,
|
|
1741
|
+
):
|
|
1742
|
+
"""
|
|
1743
|
+
Updates the properties of a `custom pool <https://learn.microsoft.com/fabric/data-engineering/create-custom-spark-pools>`_ within a workspace.
|
|
1744
|
+
|
|
1745
|
+
Parameters
|
|
1746
|
+
----------
|
|
1747
|
+
pool_name : str
|
|
1748
|
+
The custom pool name.
|
|
1749
|
+
node_size : str, default=None
|
|
1750
|
+
The `node size <https://learn.microsoft.com/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#nodesize>`_.
|
|
1751
|
+
Defaults to None which keeps the existing property setting.
|
|
1752
|
+
min_node_count : int, default=None
|
|
1753
|
+
The `minimum node count <https://learn.microsoft.com/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#autoscaleproperties>`_.
|
|
1754
|
+
Defaults to None which keeps the existing property setting.
|
|
1755
|
+
max_node_count : int, default=None
|
|
1756
|
+
The `maximum node count <https://learn.microsoft.com/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#autoscaleproperties>`_.
|
|
1757
|
+
Defaults to None which keeps the existing property setting.
|
|
1758
|
+
min_executors : int, default=None
|
|
1759
|
+
The `minimum executors <https://learn.microsoft.com/en-us/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#dynamicexecutorallocationproperties>`_.
|
|
1760
|
+
Defaults to None which keeps the existing property setting.
|
|
1761
|
+
max_executors : int, default=None
|
|
1762
|
+
The `maximum executors <https://learn.microsoft.com/en-us/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#dynamicexecutorallocationproperties>`_.
|
|
1763
|
+
Defaults to None which keeps the existing property setting.
|
|
1764
|
+
node_family : str, default=None
|
|
1765
|
+
The `node family <https://learn.microsoft.com/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#nodefamily>`_.
|
|
1766
|
+
Defaults to None which keeps the existing property setting.
|
|
1767
|
+
auto_scale_enabled : bool, default=None
|
|
1768
|
+
The status of `auto scale <https://learn.microsoft.com/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#autoscaleproperties>`_.
|
|
1769
|
+
Defaults to None which keeps the existing property setting.
|
|
1770
|
+
dynamic_executor_allocation_enabled : bool, default=None
|
|
1771
|
+
The status of the `dynamic executor allocation <https://learn.microsoft.com/en-us/rest/api/fabric/spark/custom-pools/create-workspace-custom-pool?tabs=HTTP#dynamicexecutorallocationproperties>`_.
|
|
1772
|
+
Defaults to None which keeps the existing property setting.
|
|
1773
|
+
workspace : str, default=None
|
|
1774
|
+
The name of the Fabric workspace.
|
|
1775
|
+
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
1776
|
+
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
1777
|
+
|
|
1778
|
+
Returns
|
|
1779
|
+
-------
|
|
1780
|
+
"""
|
|
1781
|
+
|
|
1782
|
+
# https://learn.microsoft.com/en-us/rest/api/fabric/spark/custom-pools/update-workspace-custom-pool?tabs=HTTP
|
|
1783
|
+
(workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
|
|
1784
|
+
|
|
1785
|
+
df = list_custom_pools(workspace=workspace)
|
|
1786
|
+
df_pool = df[df["Custom Pool Name"] == pool_name]
|
|
1787
|
+
|
|
1788
|
+
if len(df_pool) == 0:
|
|
1789
|
+
raise ValueError(
|
|
1790
|
+
f"{icons.red_dot} The '{pool_name}' custom pool does not exist within the '{workspace}'. Please choose a valid custom pool."
|
|
1791
|
+
)
|
|
1792
|
+
|
|
1793
|
+
if node_family is None:
|
|
1794
|
+
node_family = df_pool["Node Family"].iloc[0]
|
|
1795
|
+
if node_size is None:
|
|
1796
|
+
node_size = df_pool["Node Size"].iloc[0]
|
|
1797
|
+
if auto_scale_enabled is None:
|
|
1798
|
+
auto_scale_enabled = bool(df_pool["Auto Scale Enabled"].iloc[0])
|
|
1799
|
+
if min_node_count is None:
|
|
1800
|
+
min_node_count = int(df_pool["Min Node Count"].iloc[0])
|
|
1801
|
+
if max_node_count is None:
|
|
1802
|
+
max_node_count = int(df_pool["Max Node Count"].iloc[0])
|
|
1803
|
+
if dynamic_executor_allocation_enabled is None:
|
|
1804
|
+
dynamic_executor_allocation_enabled = bool(
|
|
1805
|
+
df_pool["Dynami Executor Allocation Enabled"].iloc[0]
|
|
1806
|
+
)
|
|
1807
|
+
if min_executors is None:
|
|
1808
|
+
min_executors = int(df_pool["Min Executors"].iloc[0])
|
|
1809
|
+
if max_executors is None:
|
|
1810
|
+
max_executors = int(df_pool["Max Executors"].iloc[0])
|
|
1811
|
+
|
|
1812
|
+
request_body = {
|
|
1813
|
+
"name": pool_name,
|
|
1814
|
+
"nodeFamily": node_family,
|
|
1815
|
+
"nodeSize": node_size,
|
|
1816
|
+
"autoScale": {
|
|
1817
|
+
"enabled": auto_scale_enabled,
|
|
1818
|
+
"minNodeCount": min_node_count,
|
|
1819
|
+
"maxNodeCount": max_node_count,
|
|
1820
|
+
},
|
|
1821
|
+
"dynamicExecutorAllocation": {
|
|
1822
|
+
"enabled": dynamic_executor_allocation_enabled,
|
|
1823
|
+
"minExecutors": min_executors,
|
|
1824
|
+
"maxExecutors": max_executors,
|
|
1825
|
+
},
|
|
1826
|
+
}
|
|
1827
|
+
|
|
1828
|
+
client = fabric.FabricRestClient()
|
|
1829
|
+
response = client.post(
|
|
1830
|
+
f"/v1/workspaces/{workspace_id}/spark/pools", json=request_body
|
|
1831
|
+
)
|
|
1832
|
+
|
|
1833
|
+
if response.status_code != 200:
|
|
1834
|
+
raise FabricHTTPException(response)
|
|
1625
1835
|
print(
|
|
1626
|
-
f"
|
|
1836
|
+
f"{icons.green_dot} The '{pool_name}' spark pool within the '{workspace}' workspace has been updated."
|
|
1627
1837
|
)
|
|
1628
|
-
|
|
1838
|
+
|
|
1839
|
+
|
|
1840
|
+
def delete_custom_pool(pool_name: str, workspace: Optional[str | None] = None):
|
|
1841
|
+
"""
|
|
1842
|
+
Deletes a `custom pool <https://learn.microsoft.com/fabric/data-engineering/create-custom-spark-pools>`_ within a workspace.
|
|
1843
|
+
|
|
1844
|
+
Parameters
|
|
1845
|
+
----------
|
|
1846
|
+
pool_name : str
|
|
1847
|
+
The custom pool name.
|
|
1848
|
+
workspace : str, default=None
|
|
1849
|
+
The name of the Fabric workspace.
|
|
1850
|
+
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
1851
|
+
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
1852
|
+
|
|
1853
|
+
Returns
|
|
1854
|
+
-------
|
|
1855
|
+
"""
|
|
1856
|
+
|
|
1857
|
+
(workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
|
|
1858
|
+
|
|
1859
|
+
dfL = list_custom_pools(workspace=workspace)
|
|
1860
|
+
dfL_filt = dfL[dfL["Custom Pool Name"] == pool_name]
|
|
1861
|
+
|
|
1862
|
+
if len(dfL_filt) == 0:
|
|
1863
|
+
raise ValueError(
|
|
1864
|
+
f"{icons.red_dot} The '{pool_name}' custom pool does not exist within the '{workspace}' workspace."
|
|
1865
|
+
)
|
|
1866
|
+
poolId = dfL_filt["Custom Pool ID"].iloc[0]
|
|
1867
|
+
|
|
1868
|
+
client = fabric.FabricRestClient()
|
|
1869
|
+
response = client.delete(f"/v1/workspaces/{workspace_id}/spark/pools/{poolId}")
|
|
1870
|
+
|
|
1871
|
+
if response.status_code != 200:
|
|
1872
|
+
raise FabricHTTPException(response)
|
|
1873
|
+
print(
|
|
1874
|
+
f"{icons.green_dot} The '{pool_name}' spark pool has been deleted from the '{workspace}' workspace."
|
|
1875
|
+
)
|
|
1876
|
+
|
|
1877
|
+
|
|
1878
|
+
def assign_workspace_to_capacity(capacity_name: str, workspace: Optional[str] = None):
|
|
1879
|
+
"""
|
|
1880
|
+
Assigns a workspace to a capacity.
|
|
1881
|
+
|
|
1882
|
+
Parameters
|
|
1883
|
+
----------
|
|
1884
|
+
capacity_name : str
|
|
1885
|
+
The name of the capacity.
|
|
1886
|
+
workspace : str, default=None
|
|
1887
|
+
The name of the Fabric workspace.
|
|
1888
|
+
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
1889
|
+
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
1890
|
+
|
|
1891
|
+
Returns
|
|
1892
|
+
-------
|
|
1893
|
+
"""
|
|
1894
|
+
|
|
1895
|
+
(workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
|
|
1896
|
+
|
|
1897
|
+
dfC = fabric.list_capacities()
|
|
1898
|
+
dfC_filt = dfC[dfC["Display Name"] == capacity_name]
|
|
1899
|
+
capacity_id = dfC_filt["Id"].iloc[0]
|
|
1900
|
+
|
|
1901
|
+
request_body = {"capacityId": capacity_id}
|
|
1902
|
+
|
|
1903
|
+
client = fabric.FabricRestClient()
|
|
1904
|
+
response = client.post(
|
|
1905
|
+
f"/v1/workspaces/{workspace_id}/assignToCapacity", json=request_body
|
|
1906
|
+
)
|
|
1907
|
+
|
|
1908
|
+
if response.status_code == 202:
|
|
1909
|
+
print(
|
|
1910
|
+
f"{icons.green_dot} The '{workspace}' workspace has been assigned to the '{capacity_name}' capacity."
|
|
1911
|
+
)
|
|
1912
|
+
else:
|
|
1913
|
+
raise ValueError(f"{icons.red_dot} {response.status_code}")
|
|
1914
|
+
|
|
1915
|
+
|
|
1916
|
+
def unassign_workspace_from_capacity(workspace: Optional[str] = None):
|
|
1917
|
+
"""
|
|
1918
|
+
Unassigns a workspace from its assigned capacity.
|
|
1919
|
+
|
|
1920
|
+
Parameters
|
|
1921
|
+
----------
|
|
1922
|
+
workspace : str, default=None
|
|
1923
|
+
The name of the Fabric workspace.
|
|
1924
|
+
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
1925
|
+
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
1926
|
+
|
|
1927
|
+
Returns
|
|
1928
|
+
-------
|
|
1929
|
+
"""
|
|
1930
|
+
|
|
1931
|
+
# https://learn.microsoft.com/en-us/rest/api/fabric/core/workspaces/unassign-from-capacity?tabs=HTTP
|
|
1932
|
+
(workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
|
|
1933
|
+
|
|
1934
|
+
client = fabric.FabricRestClient()
|
|
1935
|
+
response = client.post(f"/v1/workspaces/{workspace_id}/unassignFromCapacity")
|
|
1936
|
+
|
|
1937
|
+
if response.status_code == 202:
|
|
1938
|
+
print(
|
|
1939
|
+
f"{icons.green_dot} The '{workspace}' workspace has been unassigned from its capacity."
|
|
1940
|
+
)
|
|
1941
|
+
else:
|
|
1942
|
+
raise ValueError(f"{icons.red_dot} {response.status_code}")
|
|
1943
|
+
|
|
1944
|
+
|
|
1945
|
+
def get_spark_settings(workspace: Optional[str] = None) -> pd.DataFrame:
|
|
1946
|
+
"""
|
|
1947
|
+
Shows the spark settings for a workspace.
|
|
1948
|
+
|
|
1949
|
+
Parameters
|
|
1950
|
+
----------
|
|
1951
|
+
workspace : str, default=None
|
|
1952
|
+
The name of the Fabric workspace.
|
|
1953
|
+
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
1954
|
+
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
1955
|
+
|
|
1956
|
+
Returns
|
|
1957
|
+
-------
|
|
1958
|
+
pandas.DataFrame
|
|
1959
|
+
A pandas dataframe showing the spark settings for a workspace.
|
|
1960
|
+
"""
|
|
1961
|
+
|
|
1962
|
+
# https://learn.microsoft.com/en-us/rest/api/fabric/spark/workspace-settings/get-spark-settings?tabs=HTTP
|
|
1963
|
+
(workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
|
|
1964
|
+
|
|
1965
|
+
df = pd.DataFrame(
|
|
1966
|
+
columns=[
|
|
1967
|
+
"Automatic Log Enabled",
|
|
1968
|
+
"High Concurrency Enabled",
|
|
1969
|
+
"Customize Compute Enabled",
|
|
1970
|
+
"Default Pool Name",
|
|
1971
|
+
"Default Pool Type",
|
|
1972
|
+
"Max Node Count",
|
|
1973
|
+
"Max Executors",
|
|
1974
|
+
"Environment Name",
|
|
1975
|
+
"Runtime Version",
|
|
1976
|
+
]
|
|
1977
|
+
)
|
|
1978
|
+
|
|
1979
|
+
client = fabric.FabricRestClient()
|
|
1980
|
+
response = client.get(f"/v1/workspaces/{workspace_id}/spark/settings")
|
|
1981
|
+
|
|
1982
|
+
i = response.json()
|
|
1983
|
+
p = i.get("pool")
|
|
1984
|
+
dp = i.get("pool", {}).get("defaultPool", {})
|
|
1985
|
+
sp = i.get("pool", {}).get("starterPool", {})
|
|
1986
|
+
e = i.get("environment", {})
|
|
1987
|
+
|
|
1988
|
+
new_data = {
|
|
1989
|
+
"Automatic Log Enabled": i.get("automaticLog").get("enabled"),
|
|
1990
|
+
"High Concurrency Enabled": i.get("highConcurrency").get(
|
|
1991
|
+
"notebookInteractiveRunEnabled"
|
|
1992
|
+
),
|
|
1993
|
+
"Customize Compute Enabled": p.get("customizeComputeEnabled"),
|
|
1994
|
+
"Default Pool Name": dp.get("name"),
|
|
1995
|
+
"Default Pool Type": dp.get("type"),
|
|
1996
|
+
"Max Node Count": sp.get("maxNodeCount"),
|
|
1997
|
+
"Max Node Executors": sp.get("maxExecutors"),
|
|
1998
|
+
"Environment Name": e.get("name"),
|
|
1999
|
+
"Runtime Version": e.get("runtimeVersion"),
|
|
2000
|
+
}
|
|
2001
|
+
df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
|
|
2002
|
+
|
|
2003
|
+
bool_cols = [
|
|
2004
|
+
"Automatic Log Enabled",
|
|
2005
|
+
"High Concurrency Enabled",
|
|
2006
|
+
"Customize Compute Enabled",
|
|
2007
|
+
]
|
|
2008
|
+
int_cols = ["Max Node Count", "Max Executors"]
|
|
2009
|
+
|
|
2010
|
+
df[bool_cols] = df[bool_cols].astype(bool)
|
|
2011
|
+
df[int_cols] = df[int_cols].astype(int)
|
|
2012
|
+
|
|
2013
|
+
return df
|
|
2014
|
+
|
|
2015
|
+
|
|
2016
|
+
def update_spark_settings(
|
|
2017
|
+
automatic_log_enabled: Optional[bool] = None,
|
|
2018
|
+
high_concurrency_enabled: Optional[bool] = None,
|
|
2019
|
+
customize_compute_enabled: Optional[bool] = None,
|
|
2020
|
+
default_pool_name: Optional[str] = None,
|
|
2021
|
+
max_node_count: Optional[int] = None,
|
|
2022
|
+
max_executors: Optional[int] = None,
|
|
2023
|
+
environment_name: Optional[str] = None,
|
|
2024
|
+
runtime_version: Optional[str] = None,
|
|
2025
|
+
workspace: Optional[str] = None,
|
|
2026
|
+
):
|
|
2027
|
+
"""
|
|
2028
|
+
Updates the spark settings for a workspace.
|
|
2029
|
+
|
|
2030
|
+
Parameters
|
|
2031
|
+
----------
|
|
2032
|
+
automatic_log_enabled : bool, default=None
|
|
2033
|
+
The status of the `automatic log <https://learn.microsoft.com/rest/api/fabric/spark/workspace-settings/update-spark-settings?tabs=HTTP#automaticlogproperties>`_.
|
|
2034
|
+
Defaults to None which keeps the existing property setting.
|
|
2035
|
+
high_concurrency_enabled : bool, default=None
|
|
2036
|
+
The status of the `high concurrency <https://learn.microsoft.com/rest/api/fabric/spark/workspace-settings/update-spark-settings?tabs=HTTP#highconcurrencyproperties>`_ for notebook interactive run.
|
|
2037
|
+
Defaults to None which keeps the existing property setting.
|
|
2038
|
+
customize_compute_enabled : bool, default=None
|
|
2039
|
+
`Customize compute <https://learn.microsoft.com/rest/api/fabric/spark/workspace-settings/update-spark-settings?tabs=HTTP#poolproperties>`_ configurations for items.
|
|
2040
|
+
Defaults to None which keeps the existing property setting.
|
|
2041
|
+
default_pool_name : str, default=None
|
|
2042
|
+
`Default pool <https://learn.microsoft.com/rest/api/fabric/spark/workspace-settings/update-spark-settings?tabs=HTTP#poolproperties>`_ for workspace.
|
|
2043
|
+
Defaults to None which keeps the existing property setting.
|
|
2044
|
+
max_node_count : int, default=None
|
|
2045
|
+
The `maximum node count <https://learn.microsoft.com/rest/api/fabric/spark/workspace-settings/update-spark-settings?tabs=HTTP#starterpoolproperties>`_.
|
|
2046
|
+
Defaults to None which keeps the existing property setting.
|
|
2047
|
+
max_executors : int, default=None
|
|
2048
|
+
The `maximum executors <https://learn.microsoft.com/rest/api/fabric/spark/workspace-settings/update-spark-settings?tabs=HTTP#starterpoolproperties>`_.
|
|
2049
|
+
Defaults to None which keeps the existing property setting.
|
|
2050
|
+
environment_name : str, default=None
|
|
2051
|
+
The name of the `default environment <https://learn.microsoft.com/rest/api/fabric/spark/workspace-settings/update-spark-settings?tabs=HTTP#environmentproperties>`_. Empty string indicated there is no workspace default environment
|
|
2052
|
+
Defaults to None which keeps the existing property setting.
|
|
2053
|
+
runtime_version : str, default=None
|
|
2054
|
+
The `runtime version <https://learn.microsoft.com/rest/api/fabric/spark/workspace-settings/update-spark-settings?tabs=HTTP#environmentproperties>`_.
|
|
2055
|
+
Defaults to None which keeps the existing property setting.
|
|
2056
|
+
workspace : str, default=None
|
|
2057
|
+
The name of the Fabric workspace.
|
|
2058
|
+
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
2059
|
+
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
2060
|
+
|
|
2061
|
+
Returns
|
|
2062
|
+
-------
|
|
2063
|
+
"""
|
|
2064
|
+
|
|
2065
|
+
# https://learn.microsoft.com/en-us/rest/api/fabric/spark/workspace-settings/update-spark-settings?tabs=HTTP
|
|
2066
|
+
(workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
|
|
2067
|
+
|
|
2068
|
+
dfS = get_spark_settings(workspace=workspace)
|
|
2069
|
+
|
|
2070
|
+
if automatic_log_enabled is None:
|
|
2071
|
+
automatic_log_enabled = bool(dfS["Automatic Log Enabled"].iloc[0])
|
|
2072
|
+
if high_concurrency_enabled is None:
|
|
2073
|
+
high_concurrency_enabled = bool(dfS["High Concurrency Enabled"].iloc[0])
|
|
2074
|
+
if customize_compute_enabled is None:
|
|
2075
|
+
customize_compute_enabled = bool(dfS["Customize Compute Enabled"].iloc[0])
|
|
2076
|
+
if default_pool_name is None:
|
|
2077
|
+
default_pool_name = dfS["Default Pool Name"].iloc[0]
|
|
2078
|
+
if max_node_count is None:
|
|
2079
|
+
max_node_count = int(dfS["Max Node Count"].iloc[0])
|
|
2080
|
+
if max_executors is None:
|
|
2081
|
+
max_executors = int(dfS["Max Executors"].iloc[0])
|
|
2082
|
+
if environment_name is None:
|
|
2083
|
+
environment_name = dfS["Environment Name"].iloc[0]
|
|
2084
|
+
if runtime_version is None:
|
|
2085
|
+
runtime_version = dfS["Runtime Version"].iloc[0]
|
|
2086
|
+
|
|
2087
|
+
request_body = {
|
|
2088
|
+
"automaticLog": {"enabled": automatic_log_enabled},
|
|
2089
|
+
"highConcurrency": {"notebookInteractiveRunEnabled": high_concurrency_enabled},
|
|
2090
|
+
"pool": {
|
|
2091
|
+
"customizeComputeEnabled": customize_compute_enabled,
|
|
2092
|
+
"defaultPool": {"name": default_pool_name, "type": "Workspace"},
|
|
2093
|
+
"starterPool": {
|
|
2094
|
+
"maxNodeCount": max_node_count,
|
|
2095
|
+
"maxExecutors": max_executors,
|
|
2096
|
+
},
|
|
2097
|
+
},
|
|
2098
|
+
"environment": {"name": environment_name, "runtimeVersion": runtime_version},
|
|
2099
|
+
}
|
|
2100
|
+
|
|
2101
|
+
client = fabric.FabricRestClient()
|
|
2102
|
+
response = client.patch(
|
|
2103
|
+
f"/v1/workspaces/{workspace_id}/spark/settings", json=request_body
|
|
2104
|
+
)
|
|
2105
|
+
|
|
2106
|
+
if response.status_code != 200:
|
|
2107
|
+
raise FabricHTTPException(response)
|
|
2108
|
+
print(
|
|
2109
|
+
f"{icons.green_dot} The spark settings within the '{workspace}' workspace have been updated accordingly."
|
|
2110
|
+
)
|
|
2111
|
+
|
|
2112
|
+
|
|
2113
|
+
def add_user_to_workspace(
|
|
2114
|
+
email_address: str, role_name: str, workspace: Optional[str] = None
|
|
2115
|
+
):
|
|
2116
|
+
"""
|
|
2117
|
+
Adds a user to a workspace.
|
|
2118
|
+
|
|
2119
|
+
Parameters
|
|
2120
|
+
----------
|
|
2121
|
+
email_address : str
|
|
2122
|
+
The email address of the user.
|
|
2123
|
+
role_name : str
|
|
2124
|
+
The `role <https://learn.microsoft.com/rest/api/power-bi/groups/add-group-user#groupuseraccessright>`_ of the user within the workspace.
|
|
2125
|
+
workspace : str, default=None
|
|
2126
|
+
The name of the workspace.
|
|
2127
|
+
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
2128
|
+
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
2129
|
+
|
|
2130
|
+
Returns
|
|
2131
|
+
-------
|
|
2132
|
+
"""
|
|
2133
|
+
|
|
2134
|
+
(workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
|
|
2135
|
+
|
|
2136
|
+
role_names = ["Admin", "Member", "Viewer", "Contributor"]
|
|
2137
|
+
role_name = role_name.capitalize()
|
|
2138
|
+
if role_name not in role_names:
|
|
2139
|
+
raise ValueError(
|
|
2140
|
+
f"{icons.red_dot} Invalid role. The 'role_name' parameter must be one of the following: {role_names}."
|
|
2141
|
+
)
|
|
2142
|
+
plural = "n" if role_name == "Admin" else ""
|
|
2143
|
+
|
|
2144
|
+
client = fabric.PowerBIRestClient()
|
|
2145
|
+
|
|
2146
|
+
request_body = {"emailAddress": email_address, "groupUserAccessRight": role_name}
|
|
2147
|
+
|
|
2148
|
+
response = client.post(
|
|
2149
|
+
f"/v1.0/myorg/groups/{workspace_id}/users", json=request_body
|
|
2150
|
+
)
|
|
2151
|
+
|
|
2152
|
+
if response.status_code != 200:
|
|
2153
|
+
raise FabricHTTPException(response)
|
|
2154
|
+
print(
|
|
2155
|
+
f"{icons.green_dot} The '{email_address}' user has been added as a{plural} '{role_name}' within the '{workspace}' workspace."
|
|
2156
|
+
)
|
|
2157
|
+
|
|
2158
|
+
|
|
2159
|
+
def delete_user_from_workspace(email_address: str, workspace: Optional[str] = None):
|
|
2160
|
+
"""
|
|
2161
|
+
Removes a user from a workspace.
|
|
2162
|
+
|
|
2163
|
+
Parameters
|
|
2164
|
+
----------
|
|
2165
|
+
email_address : str
|
|
2166
|
+
The email address of the user.
|
|
2167
|
+
workspace : str, default=None
|
|
2168
|
+
The name of the workspace.
|
|
2169
|
+
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
2170
|
+
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
2171
|
+
|
|
2172
|
+
Returns
|
|
2173
|
+
-------
|
|
2174
|
+
"""
|
|
2175
|
+
|
|
2176
|
+
(workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
|
|
2177
|
+
|
|
2178
|
+
client = fabric.PowerBIRestClient()
|
|
2179
|
+
response = client.delete(f"/v1.0/myorg/groups/{workspace_id}/users/{email_address}")
|
|
2180
|
+
|
|
2181
|
+
if response.status_code != 200:
|
|
2182
|
+
raise FabricHTTPException(response)
|
|
2183
|
+
print(
|
|
2184
|
+
f"{icons.green_dot} The '{email_address}' user has been removed from accessing the '{workspace}' workspace."
|
|
2185
|
+
)
|
|
2186
|
+
|
|
2187
|
+
|
|
2188
|
+
def update_workspace_user(
|
|
2189
|
+
email_address: str, role_name: str, workspace: Optional[str] = None
|
|
2190
|
+
):
|
|
2191
|
+
"""
|
|
2192
|
+
Updates a user's role within a workspace.
|
|
2193
|
+
|
|
2194
|
+
Parameters
|
|
2195
|
+
----------
|
|
2196
|
+
email_address : str
|
|
2197
|
+
The email address of the user.
|
|
2198
|
+
role_name : str
|
|
2199
|
+
The `role <https://learn.microsoft.com/rest/api/power-bi/groups/add-group-user#groupuseraccessright>`_ of the user within the workspace.
|
|
2200
|
+
workspace : str, default=None
|
|
2201
|
+
The name of the workspace.
|
|
2202
|
+
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
2203
|
+
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
2204
|
+
|
|
2205
|
+
Returns
|
|
2206
|
+
-------
|
|
2207
|
+
"""
|
|
2208
|
+
|
|
2209
|
+
(workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
|
|
2210
|
+
|
|
2211
|
+
role_names = ["Admin", "Member", "Viewer", "Contributor"]
|
|
2212
|
+
role_name = role_name.capitalize()
|
|
2213
|
+
if role_name not in role_names:
|
|
2214
|
+
raise ValueError(
|
|
2215
|
+
f"{icons.red_dot} Invalid role. The 'role_name' parameter must be one of the following: {role_names}."
|
|
2216
|
+
)
|
|
2217
|
+
|
|
2218
|
+
request_body = {"emailAddress": email_address, "groupUserAccessRight": role_name}
|
|
2219
|
+
|
|
2220
|
+
client = fabric.PowerBIRestClient()
|
|
2221
|
+
response = client.put(f"/v1.0/myorg/groups/{workspace_id}/users", json=request_body)
|
|
2222
|
+
|
|
2223
|
+
if response.status_code != 200:
|
|
2224
|
+
raise FabricHTTPException(response)
|
|
2225
|
+
print(
|
|
2226
|
+
f"{icons.green_dot} The '{email_address}' user has been updated to a '{role_name}' within the '{workspace}' workspace."
|
|
2227
|
+
)
|
|
2228
|
+
|
|
2229
|
+
|
|
2230
|
+
def list_workspace_users(workspace: Optional[str] = None) -> pd.DataFrame:
|
|
2231
|
+
"""
|
|
2232
|
+
A list of all the users of a workspace and their roles.
|
|
2233
|
+
|
|
2234
|
+
Parameters
|
|
2235
|
+
----------
|
|
2236
|
+
workspace : str, default=None
|
|
2237
|
+
The name of the workspace.
|
|
2238
|
+
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
2239
|
+
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
2240
|
+
|
|
2241
|
+
Returns
|
|
2242
|
+
-------
|
|
2243
|
+
pandas.DataFrame
|
|
2244
|
+
A pandas dataframe the users of a workspace and their properties.
|
|
2245
|
+
"""
|
|
2246
|
+
|
|
2247
|
+
(workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
|
|
2248
|
+
|
|
2249
|
+
df = pd.DataFrame(columns=["User Name", "Email Address", "Role", "Type", "User ID"])
|
|
2250
|
+
client = fabric.FabricRestClient()
|
|
2251
|
+
response = client.get(f"/v1/workspaces/{workspace_id}/roleAssignments")
|
|
2252
|
+
|
|
2253
|
+
for v in response.json()["value"]:
|
|
2254
|
+
p = v.get("principal", {})
|
|
2255
|
+
|
|
2256
|
+
new_data = {
|
|
2257
|
+
"User Name": p.get("displayName"),
|
|
2258
|
+
"User ID": p.get("id"),
|
|
2259
|
+
"Type": p.get("type"),
|
|
2260
|
+
"Role": v.get("role"),
|
|
2261
|
+
"Email Address": p.get("userDetails", {}).get("userPrincipalName"),
|
|
2262
|
+
}
|
|
2263
|
+
df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
|
|
2264
|
+
|
|
2265
|
+
return df
|
|
2266
|
+
|
|
2267
|
+
|
|
2268
|
+
def assign_workspace_to_dataflow_storage(
|
|
2269
|
+
dataflow_storage_account: str, workspace: Optional[str] = None
|
|
2270
|
+
):
|
|
2271
|
+
"""
|
|
2272
|
+
Assigns a dataflow storage account to a workspace.
|
|
2273
|
+
|
|
2274
|
+
Parameters
|
|
2275
|
+
----------
|
|
2276
|
+
dataflow_storage_account : str
|
|
2277
|
+
The name of the dataflow storage account.
|
|
2278
|
+
workspace : str, default=None
|
|
2279
|
+
The name of the workspace.
|
|
2280
|
+
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
2281
|
+
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
2282
|
+
|
|
2283
|
+
Returns
|
|
2284
|
+
-------
|
|
2285
|
+
"""
|
|
2286
|
+
|
|
2287
|
+
(workspace, workspace_id) = resolve_workspace_name_and_id(workspace)
|
|
2288
|
+
|
|
2289
|
+
df = list_dataflow_storage_accounts()
|
|
2290
|
+
df_filt = df[df["Dataflow Storage Account Name"] == dataflow_storage_account]
|
|
2291
|
+
dataflow_storage_id = df_filt["Dataflow Storage Account ID"].iloc[0]
|
|
2292
|
+
|
|
2293
|
+
client = fabric.PowerBIRestClient()
|
|
2294
|
+
|
|
2295
|
+
request_body = {"dataflowStorageId": dataflow_storage_id}
|
|
2296
|
+
|
|
2297
|
+
response = client.post(
|
|
2298
|
+
f"/v1.0/myorg/groups/{workspace_id}/AssignToDataflowStorage", json=request_body
|
|
2299
|
+
)
|
|
2300
|
+
|
|
2301
|
+
if response.status_code != 200:
|
|
2302
|
+
raise FabricHTTPException(response)
|
|
2303
|
+
print(
|
|
2304
|
+
f"{icons.green_dot} The '{dataflow_storage_account}' dataflow storage account has been assigned to the '{workspace}' workspacce."
|
|
2305
|
+
)
|
|
2306
|
+
|
|
2307
|
+
|
|
2308
|
+
def list_capacities() -> pd.DataFrame:
|
|
2309
|
+
"""
|
|
2310
|
+
Shows the capacities and their properties.
|
|
2311
|
+
|
|
2312
|
+
Parameters
|
|
2313
|
+
----------
|
|
2314
|
+
|
|
2315
|
+
Returns
|
|
2316
|
+
-------
|
|
2317
|
+
pandas.DataFrame
|
|
2318
|
+
A pandas dataframe showing the capacities and their properties
|
|
2319
|
+
"""
|
|
2320
|
+
|
|
2321
|
+
df = pd.DataFrame(
|
|
2322
|
+
columns=["Id", "Display Name", "Sku", "Region", "State", "Admins"]
|
|
2323
|
+
)
|
|
2324
|
+
|
|
2325
|
+
client = fabric.PowerBIRestClient()
|
|
2326
|
+
response = client.get("/v1.0/myorg/capacities")
|
|
2327
|
+
|
|
2328
|
+
for i in response.json()["value"]:
|
|
2329
|
+
new_data = {
|
|
2330
|
+
"Id": i.get("id", {}).lower(),
|
|
2331
|
+
"Display Name": i.get("displayName", {}),
|
|
2332
|
+
"Sku": i.get("sku", {}),
|
|
2333
|
+
"Region": i.get("region", {}),
|
|
2334
|
+
"State": i.get("state", {}),
|
|
2335
|
+
"Admins": [i.get("admins", [])],
|
|
2336
|
+
}
|
|
2337
|
+
df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
|
|
2338
|
+
|
|
2339
|
+
return df
|