msfabricpysdkcore 0.0.13__py3-none-any.whl → 0.1.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- msfabricpysdkcore/__init__.py +2 -1
- msfabricpysdkcore/admin_item.py +19 -45
- msfabricpysdkcore/admin_workspace.py +13 -60
- msfabricpysdkcore/adminapi.py +401 -476
- msfabricpysdkcore/auth.py +10 -6
- msfabricpysdkcore/client.py +124 -7
- msfabricpysdkcore/coreapi.py +2570 -822
- msfabricpysdkcore/deployment_pipeline.py +34 -146
- msfabricpysdkcore/domain.py +20 -219
- msfabricpysdkcore/environment.py +13 -172
- msfabricpysdkcore/fabric_azure_capacity.py +77 -0
- msfabricpysdkcore/fabric_azure_client.py +228 -0
- msfabricpysdkcore/item.py +55 -331
- msfabricpysdkcore/job_instance.py +8 -22
- msfabricpysdkcore/lakehouse.py +9 -118
- msfabricpysdkcore/long_running_operation.py +7 -37
- msfabricpysdkcore/onelakeshortcut.py +7 -21
- msfabricpysdkcore/otheritems.py +66 -91
- msfabricpysdkcore/spark_custom_pool.py +7 -47
- msfabricpysdkcore/tests/test_admin_apis.py +9 -10
- msfabricpysdkcore/tests/test_datapipelines.py +15 -18
- msfabricpysdkcore/tests/test_deployment_pipeline.py +3 -3
- msfabricpysdkcore/tests/test_domains.py +6 -5
- msfabricpysdkcore/tests/test_environments.py +54 -5
- msfabricpysdkcore/tests/test_evenhouses.py +47 -0
- msfabricpysdkcore/tests/test_evenstreams.py +20 -20
- msfabricpysdkcore/tests/test_external_data_shares.py +3 -3
- msfabricpysdkcore/tests/test_fabric_azure_client.py +78 -0
- msfabricpysdkcore/tests/test_git.py +8 -9
- msfabricpysdkcore/tests/test_items.py +81 -0
- msfabricpysdkcore/tests/test_jobs.py +2 -2
- msfabricpysdkcore/tests/test_kql_queryset.py +49 -0
- msfabricpysdkcore/tests/test_kqldatabases.py +3 -3
- msfabricpysdkcore/tests/test_lakehouse.py +84 -0
- msfabricpysdkcore/tests/test_ml_experiments.py +47 -0
- msfabricpysdkcore/tests/test_ml_models.py +47 -0
- msfabricpysdkcore/tests/test_notebooks.py +57 -0
- msfabricpysdkcore/tests/test_one_lake_data_access_security.py +2 -4
- msfabricpysdkcore/tests/test_other_items.py +45 -0
- msfabricpysdkcore/tests/test_reports.py +52 -0
- msfabricpysdkcore/tests/test_semantic_model.py +50 -0
- msfabricpysdkcore/tests/test_shortcuts.py +4 -4
- msfabricpysdkcore/tests/test_spark.py +9 -9
- msfabricpysdkcore/tests/test_sparkjobdefinition.py +2 -2
- msfabricpysdkcore/tests/test_warehouses.py +50 -0
- msfabricpysdkcore/tests/test_workspaces_capacities.py +16 -13
- msfabricpysdkcore/workspace.py +397 -1163
- {msfabricpysdkcore-0.0.13.dist-info → msfabricpysdkcore-0.1.2.dist-info}/METADATA +72 -10
- msfabricpysdkcore-0.1.2.dist-info/RECORD +55 -0
- {msfabricpysdkcore-0.0.13.dist-info → msfabricpysdkcore-0.1.2.dist-info}/WHEEL +1 -1
- msfabricpysdkcore-0.0.13.dist-info/RECORD +0 -41
- {msfabricpysdkcore-0.0.13.dist-info → msfabricpysdkcore-0.1.2.dist-info}/LICENSE +0 -0
- {msfabricpysdkcore-0.0.13.dist-info → msfabricpysdkcore-0.1.2.dist-info}/top_level.txt +0 -0
msfabricpysdkcore/item.py
CHANGED
@@ -1,15 +1,11 @@
|
|
1
1
|
import json
|
2
|
-
import requests
|
3
|
-
from time import sleep
|
4
2
|
|
5
|
-
from msfabricpysdkcore.
|
6
|
-
from msfabricpysdkcore.job_instance import JobInstance
|
7
|
-
from msfabricpysdkcore.long_running_operation import check_long_running_operation
|
3
|
+
from msfabricpysdkcore.coreapi import FabricClientCore
|
8
4
|
|
9
5
|
class Item:
|
10
6
|
"""Class to represent a item in Microsoft Fabric"""
|
11
7
|
|
12
|
-
def __init__(self, id, display_name, type, workspace_id,
|
8
|
+
def __init__(self, id, display_name, type, workspace_id, core_client: FabricClientCore, properties = None, definition=None, description="") -> None:
|
13
9
|
|
14
10
|
self.id = id
|
15
11
|
self.display_name = display_name
|
@@ -19,7 +15,7 @@ class Item:
|
|
19
15
|
self.properties = properties
|
20
16
|
self.workspace_id = workspace_id
|
21
17
|
|
22
|
-
self.
|
18
|
+
self.core_client = core_client
|
23
19
|
|
24
20
|
def __str__(self) -> str:
|
25
21
|
"""Return a string representation of the workspace object"""
|
@@ -37,376 +33,104 @@ class Item:
|
|
37
33
|
def __repr__(self) -> str:
|
38
34
|
return self.__str__()
|
39
35
|
|
40
|
-
def from_dict(item_dict,
|
36
|
+
def from_dict(item_dict, core_client):
|
41
37
|
"""Create Item object from dictionary"""
|
42
38
|
|
43
39
|
return Item(id=item_dict['id'], display_name=item_dict['displayName'], type=item_dict['type'], workspace_id=item_dict['workspaceId'],
|
44
40
|
properties=item_dict.get('properties', None),
|
45
|
-
definition=item_dict.get('definition', None), description=item_dict.get('description', ""),
|
41
|
+
definition=item_dict.get('definition', None), description=item_dict.get('description', ""), core_client=core_client)
|
46
42
|
|
47
43
|
def delete(self, type = None):
|
48
44
|
"""Delete the workspace item"""
|
49
45
|
|
50
|
-
|
51
|
-
if type:
|
52
|
-
url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/{type}/{self.id}"
|
53
|
-
for _ in range(10):
|
54
|
-
response = requests.delete(url=url, headers=self.auth.get_headers())
|
55
|
-
if response.status_code == 429:
|
56
|
-
print("Too many requests, waiting 10 seconds")
|
57
|
-
sleep(10)
|
58
|
-
continue
|
59
|
-
if response.status_code not in (200, 429):
|
60
|
-
raise Exception(f"Error deleting item: {response.status_code}, {response.text}")
|
61
|
-
break
|
62
|
-
|
63
|
-
return response.status_code
|
46
|
+
return self.core_client.delete_item(self.workspace_id, self.id, type=type)
|
64
47
|
|
65
48
|
def get_definition(self, type = None, format = None):
|
66
49
|
"""Get the definition of the item"""
|
67
|
-
|
68
|
-
url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/items/{self.id}/getDefinition"
|
69
|
-
if type:
|
70
|
-
url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/{type}/{self.id}/getDefinition"
|
71
|
-
|
72
|
-
if format:
|
73
|
-
url += f"?format={format}"
|
50
|
+
resp_dict = self.core_client.get_item_definition(self.workspace_id, self.id, type=type, format=format)
|
74
51
|
|
75
|
-
for _ in range(10):
|
76
|
-
response = requests.post(url=url, headers=self.auth.get_headers())
|
77
|
-
if response.status_code == 429:
|
78
|
-
print("Too many requests, waiting 10 seconds")
|
79
|
-
sleep(10)
|
80
|
-
continue
|
81
|
-
if response.status_code == 202:
|
82
|
-
operation_result = check_long_running_operation( response.headers, self.auth)
|
83
|
-
self.definition = operation_result['definition']
|
84
|
-
return operation_result
|
85
|
-
|
86
|
-
if response.status_code not in (200, 202, 429):
|
87
|
-
print(response.status_code)
|
88
|
-
print(response.text)
|
89
|
-
raise Exception(f"Error getting item definition: {response.text}")
|
90
|
-
break
|
91
|
-
|
92
|
-
resp_dict = json.loads(response.text)
|
93
52
|
self.definition = resp_dict['definition']
|
94
53
|
return resp_dict
|
95
|
-
|
96
|
-
def update(self, display_name = None, description = None, type = None):
|
97
|
-
"""Update the item"""
|
98
|
-
url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/items/{self.id}"
|
99
|
-
if type:
|
100
|
-
url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/{type}/{self.id}"
|
101
54
|
|
102
|
-
payload = dict()
|
103
|
-
if display_name:
|
104
|
-
payload['displayName'] = display_name
|
105
|
-
if description:
|
106
|
-
payload['description'] = description
|
107
|
-
for _ in range(10):
|
108
|
-
response = requests.patch(url=url, headers=self.auth.get_headers(), json=payload)
|
109
|
-
if response.status_code == 429:
|
110
|
-
print("Too many requests, waiting 10 seconds")
|
111
|
-
sleep(10)
|
112
|
-
continue
|
113
|
-
if response.status_code not in (200, 429):
|
114
|
-
print(response.status_code)
|
115
|
-
print(response.text)
|
116
55
|
|
117
|
-
|
118
|
-
|
56
|
+
def update(self, display_name = None, description = None, type = None, return_item="Default"):
|
57
|
+
"""Update the item"""
|
58
|
+
|
59
|
+
resp_dict = self.core_client.update_item(workspace_id=self.workspace_id, item_id=self.id,
|
60
|
+
display_name=display_name, description=description, type=type,
|
61
|
+
return_item=return_item)
|
119
62
|
if display_name:
|
120
|
-
self.display_name =
|
63
|
+
self.display_name = display_name
|
121
64
|
if description:
|
122
|
-
self.description =
|
65
|
+
self.description = description
|
123
66
|
|
124
|
-
return
|
67
|
+
return resp_dict
|
125
68
|
|
126
69
|
def update_definition(self, definition, type = None):
|
127
70
|
"""Update the item definition"""
|
128
|
-
|
129
|
-
|
130
|
-
if type:
|
131
|
-
url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/{type}/{self.id}/updateDefinition"
|
132
|
-
|
133
|
-
payload = {
|
134
|
-
'definition': definition
|
135
|
-
}
|
136
|
-
for _ in range(10):
|
137
|
-
response = requests.post(url=url, headers=self.auth.get_headers(), json=payload)
|
138
|
-
if response.status_code == 429:
|
139
|
-
print("Too many requests, waiting 10 seconds")
|
140
|
-
sleep(10)
|
141
|
-
continue
|
142
|
-
if response.status_code == 202:
|
143
|
-
check_long_running_operation( response.headers, self.auth)
|
144
|
-
if response.status_code not in (200, 202, 429):
|
145
|
-
print(response.status_code)
|
146
|
-
print(response.text)
|
71
|
+
response = self.core_client.update_item_definition(workspace_id=self.workspace_id, item_id=self.id,
|
72
|
+
definition=definition, type=type)
|
147
73
|
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
self.definition = payload['definition']
|
152
|
-
return self
|
74
|
+
self.definition = definition
|
75
|
+
return response
|
153
76
|
|
154
|
-
|
155
|
-
"""Get the shortcut in the item"""
|
156
|
-
url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/items/{self.id}/shortcuts/{path}/{name}"
|
157
|
-
|
158
|
-
for _ in range(10):
|
159
|
-
response = requests.get(url=url, headers=self.auth.get_headers())
|
160
|
-
if response.status_code == 429:
|
161
|
-
print("Too many requests, waiting 10 seconds")
|
162
|
-
sleep(10)
|
163
|
-
continue
|
164
|
-
if response.status_code not in (200, 429):
|
165
|
-
print(response.status_code)
|
166
|
-
print(response.text)
|
77
|
+
# Shortcut
|
167
78
|
|
168
|
-
raise Exception(f"Error getting shortcut: {response.text}")
|
169
|
-
break
|
170
|
-
|
171
|
-
shortcut_dict = json.loads(response.text)
|
172
|
-
shortcut_dict['workspaceId'] = self.workspace_id
|
173
|
-
shortcut_dict['itemId'] = self.id
|
174
|
-
return OneLakeShortcut.from_dict(shortcut_dict,
|
175
|
-
auth = self.auth)
|
176
|
-
|
177
79
|
def create_shortcut(self, path, name, target):
|
178
80
|
"""Create a shortcut in the item"""
|
179
|
-
|
180
|
-
|
181
|
-
body = {'name': name,
|
182
|
-
'path': path,
|
183
|
-
'target': target}
|
184
|
-
|
185
|
-
for _ in range(10):
|
186
|
-
response = requests.post(url=url, headers=self.auth.get_headers(), json=body)
|
187
|
-
if response.status_code == 429:
|
188
|
-
print("Too many requests, waiting 10 seconds")
|
189
|
-
sleep(10)
|
190
|
-
continue
|
191
|
-
if response.status_code not in (201, 429):
|
192
|
-
print(response.status_code)
|
193
|
-
print(response.text)
|
194
|
-
|
195
|
-
raise Exception(f"Error creating shortcut: {response.text}")
|
196
|
-
break
|
197
|
-
|
198
|
-
shortcut_dict = json.loads(response.text)
|
199
|
-
shortcut_dict['workspaceId'] = self.workspace_id
|
200
|
-
shortcut_dict['itemId'] = self.id
|
201
|
-
return OneLakeShortcut.from_dict(shortcut_dict,
|
202
|
-
auth = self.auth)
|
81
|
+
return self.core_client.create_shortcut(workspace_id=self.workspace_id, item_id=self.id,
|
82
|
+
path=path, name=name, target=target)
|
203
83
|
|
204
84
|
def delete_shortcut(self, path, name):
|
205
85
|
"""Delete the shortcut in the item"""
|
206
|
-
return self.
|
207
|
-
|
208
|
-
|
209
|
-
def run_on_demand_item_job(self, job_type, execution_data = None):
|
210
|
-
"""Run an on demand job on the item"""
|
86
|
+
return self.core_client.delete_shortcut(workspace_id=self.workspace_id, item_id=self.id,
|
87
|
+
path=path, name=name)
|
211
88
|
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
for _ in range(10):
|
219
|
-
if execution_data:
|
220
|
-
response = requests.post(url=url, headers=self.auth.get_headers(), json=payload)
|
221
|
-
else:
|
222
|
-
response = requests.post(url=url, headers=self.auth.get_headers())
|
223
|
-
if response.status_code == 429:
|
224
|
-
print("Too many requests, waiting 10 seconds")
|
225
|
-
sleep(10)
|
226
|
-
continue
|
227
|
-
|
228
|
-
if response.status_code not in (202, 429):
|
229
|
-
print(response.status_code)
|
230
|
-
print(response.text)
|
231
|
-
|
232
|
-
raise Exception(f"Error running on demand job: {response.text}")
|
233
|
-
break
|
234
|
-
job_instance_id = response.headers["Location"].split("/")[-1]
|
235
|
-
job_instance = self.get_item_job_instance(job_instance_id=job_instance_id)
|
236
|
-
return job_instance
|
237
|
-
|
238
|
-
def get_item_job_instance(self, job_instance_id):
|
239
|
-
"""Get the job instance of the item"""
|
89
|
+
def get_shortcut(self, path, name):
|
90
|
+
"""Get the shortcut in the item"""
|
91
|
+
return self.core_client.get_shortcut(workspace_id=self.workspace_id, item_id=self.id,
|
92
|
+
path=path, name=name)
|
93
|
+
|
240
94
|
|
241
|
-
|
242
|
-
for _ in range(10):
|
243
|
-
response = requests.get(url=url, headers=self.auth.get_headers())
|
244
|
-
if response.status_code == 429:
|
245
|
-
print("Too many requests, waiting 10 seconds")
|
246
|
-
sleep(10)
|
247
|
-
continue
|
248
|
-
if response.status_code not in (200, 429):
|
249
|
-
raise Exception(f"Error getting job instance: {response.status_code}, {response.text}")
|
250
|
-
break
|
95
|
+
# Job Scheduler
|
251
96
|
|
252
|
-
job_dict = json.loads(response.text)
|
253
|
-
job_dict['workspaceId'] = self.workspace_id
|
254
|
-
job_dict['itemId'] = self.id
|
255
|
-
return JobInstance.from_dict(job_dict, auth=self.auth)
|
256
|
-
|
257
97
|
def cancel_item_job_instance(self, job_instance_id):
|
258
98
|
"""Cancel a job instance ofjob the item"""
|
259
|
-
return self.
|
99
|
+
return self.core_client.cancel_item_job_instance(workspace_id=self.workspace_id, item_id=self.id,
|
100
|
+
job_instance_id=job_instance_id)
|
260
101
|
|
261
|
-
def
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
file_extension = None, format_options = None,
|
266
|
-
mode = None, recursive = None, wait_for_completion = True):
|
267
|
-
raise NotImplementedError("Load table only works on Lakehouse Items")
|
268
|
-
|
269
|
-
def create_external_data_share(self, paths, recipient):
|
270
|
-
# POST https://api.fabric.microsoft.com/v1/workspaces/{workspaceId}/items/{itemId}/externalDataShares
|
271
|
-
url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/items/{self.id}/externalDataShares"
|
272
|
-
|
273
|
-
body = {
|
274
|
-
'paths': paths,
|
275
|
-
'recipient': recipient
|
276
|
-
}
|
102
|
+
def get_item_job_instance(self, job_instance_id):
|
103
|
+
"""Get the job instance of the item"""
|
104
|
+
return self.core_client.get_item_job_instance(workspace_id=self.workspace_id, item_id=self.id,
|
105
|
+
job_instance_id=job_instance_id)
|
277
106
|
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
print("Too many requests, waiting 10 seconds")
|
282
|
-
sleep(10)
|
283
|
-
continue
|
284
|
-
if response.status_code not in (201, 429):
|
285
|
-
raise Exception(f"Error creating external data share: {response.status_code}, {response.text}")
|
286
|
-
break
|
287
|
-
return json.loads(response.text)
|
288
|
-
|
289
|
-
def get_external_data_share(self, external_data_share_id):
|
290
|
-
# GET https://api.fabric.microsoft.com/v1/workspaces/{workspaceId}/items/{itemId}/externalDataShares/{externalDataShareId}
|
291
|
-
url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/items/{self.id}/externalDataShares/{external_data_share_id}"
|
107
|
+
def run_on_demand_item_job(self, job_type, execution_data = None):
|
108
|
+
return self.core_client.run_on_demand_item_job(workspace_id=self.workspace_id, item_id=self.id,
|
109
|
+
job_type=job_type, execution_data=execution_data)
|
292
110
|
|
293
|
-
for _ in range(10):
|
294
|
-
response = requests.get(url=url, headers=self.auth.get_headers())
|
295
|
-
if response.status_code == 429:
|
296
|
-
print("Too many requests, waiting 10 seconds")
|
297
|
-
sleep(10)
|
298
|
-
continue
|
299
|
-
if response.status_code not in (200, 429):
|
300
|
-
raise Exception(f"Error getting external data share: {response.status_code}, {response.text}")
|
301
|
-
break
|
302
|
-
return json.loads(response.text)
|
303
111
|
|
304
|
-
|
305
|
-
# GET GET https://api.fabric.microsoft.com/v1/workspaces/{workspaceId}/items/{itemId}/externalDataShares?continuationToken={continuationToken}
|
112
|
+
# External Data Shares
|
306
113
|
|
307
|
-
|
308
|
-
|
309
|
-
|
310
|
-
url += f"?continuationToken={continuationToken}"
|
311
|
-
|
312
|
-
for _ in range(10):
|
313
|
-
response = requests.get(url=url, headers=self.auth.get_headers())
|
314
|
-
if response.status_code == 429:
|
315
|
-
print("Too many requests, waiting 10 seconds")
|
316
|
-
sleep(10)
|
317
|
-
continue
|
318
|
-
|
319
|
-
if response.status_code not in (200, 429):
|
320
|
-
raise Exception(f"Error listing external data shares: {response.status_code}, {response.text}")
|
321
|
-
break
|
322
|
-
|
323
|
-
resp = json.loads(response.text)
|
324
|
-
external_data_shares = resp['value']
|
114
|
+
def create_external_data_share(self, paths, recipient):
|
115
|
+
return self.core_client.create_external_data_share(workspace_id=self.workspace_id, item_id=self.id,
|
116
|
+
paths=paths, recipient=recipient)
|
325
117
|
|
326
|
-
|
327
|
-
|
328
|
-
|
118
|
+
def get_external_data_share(self, external_data_share_id):
|
119
|
+
return self.core_client.get_external_data_share(workspace_id=self.workspace_id, item_id=self.id,
|
120
|
+
external_data_share_id=external_data_share_id)
|
329
121
|
|
330
|
-
|
122
|
+
def list_external_data_shares_in_item(self):
|
123
|
+
return self.core_client.list_external_data_shares_in_item(workspace_id=self.workspace_id, item_id=self.id)
|
331
124
|
|
332
125
|
def revoke_external_data_share(self, external_data_share_id):
|
333
|
-
|
334
|
-
|
335
|
-
url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/items/{self.id}/externalDataShares/{external_data_share_id}/revoke"
|
336
|
-
|
337
|
-
for _ in range(10):
|
338
|
-
response = requests.post(url=url, headers=self.auth.get_headers())
|
339
|
-
if response.status_code == 429:
|
340
|
-
print("Too many requests, waiting 10 seconds")
|
341
|
-
sleep(10)
|
342
|
-
continue
|
343
|
-
if response.status_code not in (200, 429):
|
344
|
-
raise Exception(f"Error revoking external data share: {response.status_code}, {response.text}")
|
345
|
-
break
|
346
|
-
|
347
|
-
return response.status_code
|
348
|
-
|
126
|
+
return self.core_client.revoke_external_data_share(workspace_id=self.workspace_id, item_id=self.id,
|
127
|
+
external_data_share_id=external_data_share_id)
|
349
128
|
|
350
129
|
# One Lake data access security
|
351
130
|
|
352
|
-
def list_data_access_roles(self
|
353
|
-
|
354
|
-
|
355
|
-
url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/items/{self.id}/dataAccessRoles"
|
356
|
-
|
357
|
-
if continuationToken:
|
358
|
-
url += f"?continuationToken={continuationToken}"
|
359
|
-
|
360
|
-
for _ in range(10):
|
361
|
-
response = requests.get(url=url, headers=self.auth.get_headers())
|
362
|
-
if response.status_code == 429:
|
363
|
-
print("Too many requests, waiting 10 seconds")
|
364
|
-
sleep(10)
|
365
|
-
continue
|
366
|
-
if response.status_code not in (200, 429):
|
367
|
-
raise Exception(f"Error revoking external data share: {response.status_code}, {response.text}")
|
368
|
-
break
|
369
|
-
|
370
|
-
resp_dict = json.loads(response.text)
|
371
|
-
data_access_roles = resp_dict['value']
|
372
|
-
etag = response.headers.get('ETag', None)
|
373
|
-
|
374
|
-
if 'continuationToken' in resp_dict and resp_dict['continuationToken']:
|
375
|
-
data_access_roles_new, etag = self.list_data_access_roles(continuationToken=resp_dict['continuationToken'])
|
376
|
-
data_access_roles.extend(data_access_roles_new)
|
377
|
-
|
378
|
-
return data_access_roles, etag
|
131
|
+
def list_data_access_roles(self):
|
132
|
+
return self.core_client.list_data_access_roles(workspace_id=self.workspace_id, item_id=self.id)
|
379
133
|
|
380
134
|
def create_or_update_data_access_roles(self, data_access_roles, dryrun = False, etag_match = None):
|
381
|
-
|
382
|
-
|
383
|
-
url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/items/{self.id}/dataAccessRoles"
|
384
|
-
|
385
|
-
if dryrun:
|
386
|
-
url += "?dryrun=true"
|
387
|
-
|
388
|
-
if etag_match:
|
389
|
-
if 'If-Match' in etag_match:
|
390
|
-
headers = self.auth.get_headers()
|
391
|
-
headers['If-Match'] = etag_match['If-Match']
|
392
|
-
elif 'If-None-Match' in etag_match:
|
393
|
-
headers = self.auth.get_headers()
|
394
|
-
headers['If-None-Match'] = etag_match['If-None-Match']
|
395
|
-
else:
|
396
|
-
raise Exception("Etag match should include If-Match or If-None-Match")
|
397
|
-
else:
|
398
|
-
headers = self.auth.get_headers()
|
399
|
-
|
400
|
-
body = {"value" : data_access_roles}
|
401
|
-
|
402
|
-
for _ in range(10):
|
403
|
-
response = requests.put(url=url, headers=headers, json=body)
|
404
|
-
if response.status_code == 429:
|
405
|
-
print("Too many requests, waiting 10 seconds")
|
406
|
-
sleep(10)
|
407
|
-
continue
|
408
|
-
if response.status_code not in (200, 429):
|
409
|
-
raise Exception(f"Error creating or updating data access roles: {response.status_code}, {response.text}")
|
410
|
-
break
|
411
|
-
|
412
|
-
return response
|
135
|
+
return self.core_client.create_or_update_data_access_roles(workspace_id=self.workspace_id, item_id=self.id,
|
136
|
+
data_access_roles=data_access_roles, dryrun=dryrun, etag_match=etag_match)
|
@@ -1,11 +1,10 @@
|
|
1
1
|
import json
|
2
|
-
import
|
3
|
-
from time import sleep
|
2
|
+
from msfabricpysdkcore.coreapi import FabricClientCore
|
4
3
|
|
5
4
|
class JobInstance:
|
6
5
|
"""Class to represent a job instance in Microsoft Fabric"""
|
7
6
|
|
8
|
-
def __init__(self, id, item_id, workspace_id,
|
7
|
+
def __init__(self, id, item_id, workspace_id, core_client: FabricClientCore, job_type, invoke_type, status, root_activity_id,
|
9
8
|
start_time_utc, end_time_utc, failureReason):
|
10
9
|
|
11
10
|
self.id = id
|
@@ -19,7 +18,7 @@ class JobInstance:
|
|
19
18
|
self.end_time_utc = end_time_utc
|
20
19
|
self.failureReason = failureReason
|
21
20
|
|
22
|
-
self.
|
21
|
+
self.core_client = core_client
|
23
22
|
|
24
23
|
|
25
24
|
def __str__(self) -> str:
|
@@ -41,28 +40,15 @@ class JobInstance:
|
|
41
40
|
def __repr__(self) -> str:
|
42
41
|
return self.__str__()
|
43
42
|
|
44
|
-
def from_dict(job_dict,
|
43
|
+
def from_dict(job_dict, core_client):
|
45
44
|
"""Create JobInstance object from dictionary"""
|
46
45
|
return JobInstance(id=job_dict['id'], item_id=job_dict['itemId'], workspace_id=job_dict['workspaceId'],
|
47
46
|
job_type=job_dict['jobType'], invoke_type=job_dict['invokeType'], status=job_dict['status'],
|
48
47
|
root_activity_id=job_dict['rootActivityId'], start_time_utc=job_dict['startTimeUtc'],
|
49
|
-
end_time_utc=job_dict['endTimeUtc'], failureReason=job_dict['failureReason'],
|
48
|
+
end_time_utc=job_dict['endTimeUtc'], failureReason=job_dict['failureReason'], core_client=core_client)
|
50
49
|
|
51
50
|
def cancel(self):
|
52
51
|
"""Cancel the job instance"""
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
if response.status_code == 429:
|
57
|
-
print("Too many requests, waiting 10 seconds")
|
58
|
-
sleep(10)
|
59
|
-
continue
|
60
|
-
|
61
|
-
if response.status_code not in (202, 429):
|
62
|
-
print(response.status_code)
|
63
|
-
print(response.text)
|
64
|
-
|
65
|
-
raise Exception(f"Error running on demand job: {response.text}")
|
66
|
-
break
|
67
|
-
|
68
|
-
return response.status_code
|
52
|
+
return self.core_client.cancel_item_job_instance(workspace_id=self.workspace_id,
|
53
|
+
item_id=self.item_id,
|
54
|
+
job_instance_id=self.id)
|
msfabricpysdkcore/lakehouse.py
CHANGED
@@ -1,136 +1,27 @@
|
|
1
|
-
import json
|
2
|
-
import requests
|
3
|
-
from time import sleep
|
4
|
-
|
5
1
|
from msfabricpysdkcore.item import Item
|
6
|
-
from msfabricpysdkcore.long_running_operation import check_long_running_operation
|
7
2
|
|
8
3
|
class Lakehouse(Item):
|
9
4
|
"""Class to represent a item in Microsoft Fabric"""
|
10
5
|
|
11
|
-
def __init__(self, id, display_name, type, workspace_id,
|
12
|
-
super().__init__(id, display_name, type, workspace_id,
|
6
|
+
def __init__(self, id, display_name, type, workspace_id, core_client, properties = None, definition=None, description=""):
|
7
|
+
super().__init__(id, display_name, type, workspace_id, core_client, properties, definition, description)
|
13
8
|
|
14
|
-
def from_dict(item_dict,
|
9
|
+
def from_dict(item_dict, core_client):
|
15
10
|
return Lakehouse(id=item_dict['id'], display_name=item_dict['displayName'], type=item_dict['type'], workspace_id=item_dict['workspaceId'],
|
16
11
|
properties=item_dict.get('properties', None),
|
17
|
-
definition=item_dict.get('definition', None), description=item_dict.get('description', ""),
|
12
|
+
definition=item_dict.get('definition', None), description=item_dict.get('description', ""), core_client=core_client)
|
18
13
|
|
19
|
-
def list_tables(self
|
14
|
+
def list_tables(self):
|
20
15
|
"""List all tables in the lakehouse"""
|
21
|
-
|
22
|
-
url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/lakehouses/{self.id}/tables"
|
23
|
-
|
24
|
-
if continuationToken:
|
25
|
-
url = f"{url}?continuationToken={continuationToken}"
|
26
|
-
|
27
|
-
for _ in range(10):
|
28
|
-
response = requests.get(url=url, headers=self.auth.get_headers())
|
29
|
-
if response.status_code == 429:
|
30
|
-
print("Too many requests, waiting 10 seconds")
|
31
|
-
sleep(10)
|
32
|
-
continue
|
33
|
-
if response.status_code not in (200, 429):
|
34
|
-
print(response.status_code)
|
35
|
-
print(response.text)
|
36
|
-
raise Exception(f"Error listing tables: {response.status_code}, {response.text}")
|
37
|
-
break
|
38
|
-
resp_dict = json.loads(response.text)
|
39
|
-
|
40
|
-
table_list = resp_dict["data"]
|
41
|
-
|
42
|
-
if "continuationToken" in resp_dict and resp_dict["continuationToken"] is not None:
|
43
|
-
table_list_next = self.list_tables(continuationToken=resp_dict["continuationToken"])
|
44
|
-
table_list.extend(table_list_next)
|
45
|
-
|
46
|
-
return table_list
|
16
|
+
return self.core_client.list_tables(self.workspace_id, self.id)
|
47
17
|
|
48
18
|
def load_table(self, table_name, path_type, relative_path,
|
49
19
|
file_extension = None, format_options = None,
|
50
20
|
mode = None, recursive = None, wait_for_completion = True):
|
51
21
|
"""Load a table in the lakehouse"""
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
body = {
|
56
|
-
"relativePath": relative_path,
|
57
|
-
"pathType": path_type,
|
58
|
-
}
|
59
|
-
|
60
|
-
if file_extension:
|
61
|
-
body["fileExtension"] = file_extension
|
62
|
-
if format_options:
|
63
|
-
body["formatOptions"] = format_options
|
64
|
-
if mode:
|
65
|
-
body["mode"] = mode
|
66
|
-
if recursive:
|
67
|
-
body["recursive"] = recursive
|
68
|
-
|
69
|
-
for _ in range(10):
|
70
|
-
response = requests.post(url=url, headers=self.auth.get_headers(), json=body)
|
71
|
-
if response.status_code == 429:
|
72
|
-
print("Too many requests, waiting 10 seconds")
|
73
|
-
sleep(10)
|
74
|
-
continue
|
75
|
-
if response.status_code == 202:
|
76
|
-
if wait_for_completion:
|
77
|
-
success = self.check_if_table_is_created(table_name)
|
78
|
-
|
79
|
-
if not success:
|
80
|
-
print("Warning: Table not created after 3 minutes")
|
81
|
-
else:
|
82
|
-
print("Table created")
|
83
|
-
if response.status_code not in (202, 429):
|
84
|
-
print(response.status_code)
|
85
|
-
print(response.text)
|
86
|
-
raise Exception(f"Error loading table: {response.status_code}, {response.text}")
|
87
|
-
break
|
88
|
-
|
89
|
-
return response.status_code
|
22
|
+
return self.core_client.load_table(self.workspace_id, self.id, table_name, path_type, relative_path,
|
23
|
+
file_extension, format_options, mode, recursive, wait_for_completion)
|
90
24
|
|
91
|
-
def check_if_table_is_created(self, table_name):
|
92
|
-
"""Check if the table is created"""
|
93
|
-
for _ in range(60):
|
94
|
-
table_names = [table["name"] for table in self.list_tables()]
|
95
|
-
if table_name in table_names:
|
96
|
-
return True
|
97
|
-
|
98
|
-
sleep(3)
|
99
|
-
return False
|
100
|
-
|
101
|
-
# run on demand table maintenance
|
102
|
-
# POST https://api.fabric.microsoft.com/v1/workspaces/{workspaceId}/lakehouses/{lakehouseId}/jobs/instances?jobType={jobType}
|
103
|
-
|
104
25
|
def run_on_demand_table_maintenance(self, execution_data, job_type = "TableMaintenance", wait_for_completion = True):
|
105
26
|
"""Run on demand table maintenance"""
|
106
|
-
|
107
|
-
|
108
|
-
body = {
|
109
|
-
"executionData": execution_data
|
110
|
-
}
|
111
|
-
|
112
|
-
for _ in range(10):
|
113
|
-
response = requests.post(url=url, headers=self.auth.get_headers(), json=body)
|
114
|
-
if response.status_code == 429:
|
115
|
-
print("Too many requests, waiting 10 seconds")
|
116
|
-
sleep(10)
|
117
|
-
continue
|
118
|
-
if response.status_code == 202 and wait_for_completion:
|
119
|
-
print("successfully started the operation")
|
120
|
-
try:
|
121
|
-
operation_result = check_long_running_operation( response.headers, self.auth)
|
122
|
-
return operation_result
|
123
|
-
except Exception as e:
|
124
|
-
|
125
|
-
print("Problem waiting for long running operation. Returning initial response.")
|
126
|
-
print(e)
|
127
|
-
return response
|
128
|
-
|
129
|
-
if response.status_code not in (200, 202, 429):
|
130
|
-
print(response.status_code)
|
131
|
-
print(response.text)
|
132
|
-
|
133
|
-
raise Exception(f"Error at run on demand table maintenance: {response.text}")
|
134
|
-
break
|
135
|
-
|
136
|
-
return response
|
27
|
+
return self.core_client.run_on_demand_table_maintenance(self.workspace_id, self.id, execution_data, job_type, wait_for_completion)
|