msfabricpysdkcore 0.0.11__py3-none-any.whl → 0.0.13__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- msfabricpysdkcore/adminapi.py +49 -1
- msfabricpysdkcore/coreapi.py +72 -7
- msfabricpysdkcore/environment.py +11 -10
- msfabricpysdkcore/item.py +147 -6
- msfabricpysdkcore/long_running_operation.py +9 -4
- msfabricpysdkcore/otheritems.py +29 -0
- msfabricpysdkcore/tests/test_admin_apis.py +20 -0
- msfabricpysdkcore/tests/test_deployment_pipeline.py +1 -2
- msfabricpysdkcore/tests/test_external_data_shares.py +51 -0
- msfabricpysdkcore/tests/test_git.py +2 -2
- msfabricpysdkcore/tests/test_items_incl_lakehouse.py +14 -52
- msfabricpysdkcore/tests/test_one_lake_data_access_security.py +65 -0
- msfabricpysdkcore/tests/test_sparkjobdefinition.py +55 -0
- msfabricpysdkcore/tests/test_workspaces_capacities.py +57 -47
- msfabricpysdkcore/workspace.py +123 -8
- {msfabricpysdkcore-0.0.11.dist-info → msfabricpysdkcore-0.0.13.dist-info}/METADATA +123 -8
- {msfabricpysdkcore-0.0.11.dist-info → msfabricpysdkcore-0.0.13.dist-info}/RECORD +20 -17
- {msfabricpysdkcore-0.0.11.dist-info → msfabricpysdkcore-0.0.13.dist-info}/WHEEL +1 -1
- {msfabricpysdkcore-0.0.11.dist-info → msfabricpysdkcore-0.0.13.dist-info}/LICENSE +0 -0
- {msfabricpysdkcore-0.0.11.dist-info → msfabricpysdkcore-0.0.13.dist-info}/top_level.txt +0 -0
msfabricpysdkcore/adminapi.py
CHANGED
@@ -658,4 +658,52 @@ class FabricClientAdmin(FabricClient):
|
|
658
658
|
break
|
659
659
|
|
660
660
|
response = json.loads(response.text)
|
661
|
-
return response
|
661
|
+
return response
|
662
|
+
|
663
|
+
def list_external_data_shares(self, continuationToken = None):
|
664
|
+
# GET https://api.fabric.microsoft.com/v1/admin/items/externalDataShares
|
665
|
+
"""List external data shares
|
666
|
+
|
667
|
+
Returns:
|
668
|
+
list: The list of external data shares
|
669
|
+
"""
|
670
|
+
url = "https://api.fabric.microsoft.com/v1/admin/items/externalDataShares"
|
671
|
+
|
672
|
+
if continuationToken:
|
673
|
+
url = f"{url}?continuationToken={continuationToken}"
|
674
|
+
|
675
|
+
for _ in range(10):
|
676
|
+
response = requests.get(url=url, headers=self.auth.get_headers())
|
677
|
+
if response.status_code == 429:
|
678
|
+
print("Too many requests, waiting 10 seconds")
|
679
|
+
sleep(10)
|
680
|
+
continue
|
681
|
+
if response.status_code not in (200, 429):
|
682
|
+
raise Exception(f"Error listing external data shares: {response.status_code}, {response.text}")
|
683
|
+
break
|
684
|
+
|
685
|
+
response = json.loads(response.text)
|
686
|
+
list_data_shares = response["value"]
|
687
|
+
|
688
|
+
if "continuationToken" in response and response["continuationToken"] is not None:
|
689
|
+
list_data_shares_next = self.list_external_data_shares(continuationToken=response["continuationToken"])
|
690
|
+
list_data_shares.extend(list_data_shares_next)
|
691
|
+
return list_data_shares
|
692
|
+
|
693
|
+
def revoke_external_data_share(self, external_data_share_id, item_id, workspace_id):
|
694
|
+
# POST https://api.fabric.microsoft.com/v1/admin/workspaces/{workspaceId}/items/{itemId}/externalDataShares/{externalDataShareId}/revoke
|
695
|
+
"""Revoke an external data share"""
|
696
|
+
url = f"https://api.fabric.microsoft.com/v1/admin/workspaces/{workspace_id}/items/{item_id}/externalDataShares/{external_data_share_id}/revoke"
|
697
|
+
|
698
|
+
for _ in range(10):
|
699
|
+
response = requests.post(url=url, headers=self.auth.get_headers())
|
700
|
+
if response.status_code == 429:
|
701
|
+
print("Too many requests, waiting 10 seconds")
|
702
|
+
sleep(10)
|
703
|
+
continue
|
704
|
+
if response.status_code not in (200, 429):
|
705
|
+
raise Exception(f"Error revoking external data share: {response.status_code}, {response.text}")
|
706
|
+
break
|
707
|
+
|
708
|
+
return response.status_code
|
709
|
+
|
msfabricpysdkcore/coreapi.py
CHANGED
@@ -81,10 +81,10 @@ class FabricClientCore(FabricClient):
|
|
81
81
|
return self.get_workspace_by_name(name)
|
82
82
|
raise ValueError("Either id or name must be provided")
|
83
83
|
|
84
|
-
def
|
84
|
+
def list_workspace_role_assignments(self, workspace_id):
|
85
85
|
"""Get role assignments for a workspace"""
|
86
86
|
ws = self.get_workspace_by_id(workspace_id)
|
87
|
-
return ws.
|
87
|
+
return ws.list_role_assignments()
|
88
88
|
|
89
89
|
def create_workspace(self, display_name, capacity_id = None, description = None, exists_ok = True):
|
90
90
|
"""Create a workspace"""
|
@@ -129,20 +129,25 @@ class FabricClientCore(FabricClient):
|
|
129
129
|
ws = self.get_workspace_by_id(workspace_id)
|
130
130
|
return ws.add_role_assignment(role, principal)
|
131
131
|
|
132
|
-
def delete_workspace_role_assignment(self, workspace_id,
|
132
|
+
def delete_workspace_role_assignment(self, workspace_id, workspace_role_assignment_id):
|
133
133
|
"""Delete a role assignment from a workspace"""
|
134
134
|
ws = self.get_workspace_by_id(workspace_id)
|
135
|
-
return ws.delete_role_assignment(
|
135
|
+
return ws.delete_role_assignment(workspace_role_assignment_id)
|
136
136
|
|
137
137
|
def update_workspace(self, workspace_id, display_name = None, description = None):
|
138
138
|
"""Update a workspace"""
|
139
139
|
ws = self.get_workspace_by_id(workspace_id)
|
140
140
|
return ws.update(display_name, description)
|
141
|
+
|
142
|
+
def get_workspace_role_assignment(self, workspace_id, workspace_role_assignment_id):
|
143
|
+
"""Get a role assignment for a workspace"""
|
144
|
+
ws = self.get_workspace_by_id(workspace_id)
|
145
|
+
return ws.get_role_assignment(workspace_role_assignment_id)
|
141
146
|
|
142
|
-
def update_workspace_role_assignment(self, workspace_id, role,
|
147
|
+
def update_workspace_role_assignment(self, workspace_id, role, workspace_role_assignment_id):
|
143
148
|
"""Update a role assignment for a workspace"""
|
144
149
|
ws = self.get_workspace_by_id(workspace_id)
|
145
|
-
return ws.update_role_assignment(role,
|
150
|
+
return ws.update_role_assignment(role, workspace_role_assignment_id)
|
146
151
|
|
147
152
|
def assign_to_capacity(self, workspace_id, capacity_id):
|
148
153
|
"""Assign a workspace to a capacity"""
|
@@ -153,6 +158,16 @@ class FabricClientCore(FabricClient):
|
|
153
158
|
"""Unassign a workspace from a capacity"""
|
154
159
|
ws = self.get_workspace_by_id(workspace_id)
|
155
160
|
return ws.unassign_from_capacity()
|
161
|
+
|
162
|
+
def provision_identity(self, workspace_id):
|
163
|
+
"""Provision an identity for a workspace"""
|
164
|
+
ws = self.get_workspace_by_id(workspace_id)
|
165
|
+
return ws.provision_identity()
|
166
|
+
|
167
|
+
def deprovision_identity(self, workspace_id):
|
168
|
+
"""Deprovision an identity for a workspace"""
|
169
|
+
ws = self.get_workspace_by_id(workspace_id)
|
170
|
+
return ws.deprovision_identity()
|
156
171
|
|
157
172
|
def list_capacities(self, continuationToken = None):
|
158
173
|
"""List all capacities in the tenant"""
|
@@ -809,6 +824,11 @@ class FabricClientCore(FabricClient):
|
|
809
824
|
ws = self.get_workspace_by_id(workspace_id)
|
810
825
|
return ws.update_spark_job_definition_definition(spark_job_definition_id, definition)
|
811
826
|
|
827
|
+
def run_on_demand_spark_job_definition(self, workspace_id, spark_job_definition_id, job_type = "sparkjob"):
|
828
|
+
"""Run an on demand spark job definition"""
|
829
|
+
ws = self.get_workspace_by_id(workspace_id)
|
830
|
+
return ws.run_on_demand_spark_job_definition(spark_job_definition_id, job_type)
|
831
|
+
|
812
832
|
# warehouses
|
813
833
|
|
814
834
|
def list_warehouses(self, workspace_id, with_properties = False):
|
@@ -991,4 +1011,49 @@ class FabricClientCore(FabricClient):
|
|
991
1011
|
return ws.update_spark_settings(automatic_log=automatic_log,
|
992
1012
|
environment=environment,
|
993
1013
|
high_concurrency=high_concurrency,
|
994
|
-
pool=pool)
|
1014
|
+
pool=pool)
|
1015
|
+
|
1016
|
+
|
1017
|
+
# External Data Shares
|
1018
|
+
|
1019
|
+
# create
|
1020
|
+
|
1021
|
+
def create_external_data_share(self, workspace_id, item_id, paths, recipient):
|
1022
|
+
ws = self.get_workspace_by_id(workspace_id)
|
1023
|
+
return ws.create_external_data_share(item_id=item_id, paths = paths, recipient = recipient)
|
1024
|
+
|
1025
|
+
# get
|
1026
|
+
|
1027
|
+
def get_external_data_share(self, workspace_id, item_id, external_data_share_id):
|
1028
|
+
ws = self.get_workspace_by_id(workspace_id)
|
1029
|
+
return ws.get_external_data_share(item_id=item_id, external_data_share_id=external_data_share_id)
|
1030
|
+
|
1031
|
+
# list
|
1032
|
+
|
1033
|
+
def list_external_data_shares_in_item(self, workspace_id, item_id):
|
1034
|
+
ws = self.get_workspace_by_id(workspace_id)
|
1035
|
+
return ws.list_external_data_shares_in_item(item_id=item_id)
|
1036
|
+
|
1037
|
+
# revoke
|
1038
|
+
|
1039
|
+
def revoke_external_data_share(self, workspace_id, item_id, external_data_share_id):
|
1040
|
+
ws = self.get_workspace_by_id(workspace_id)
|
1041
|
+
return ws.revoke_external_data_share(item_id=item_id, external_data_share_id=external_data_share_id)
|
1042
|
+
|
1043
|
+
|
1044
|
+
# One Lake Data Access Security
|
1045
|
+
|
1046
|
+
# create and update
|
1047
|
+
|
1048
|
+
def create_or_update_data_access_roles(self, workspace_id, item_id, data_access_roles, dryrun = False, etag_match = None):
|
1049
|
+
ws = self.get_workspace_by_id(workspace_id)
|
1050
|
+
item = ws.get_item(item_id=item_id).create_or_update_data_access_roles(data_access_roles = data_access_roles,
|
1051
|
+
dryrun = dryrun, etag_match = etag_match)
|
1052
|
+
return item
|
1053
|
+
|
1054
|
+
# list
|
1055
|
+
|
1056
|
+
def list_data_access_roles(self, workspace_id, item_id):
|
1057
|
+
ws = self.get_workspace_by_id(workspace_id)
|
1058
|
+
item = ws.get_item(item_id=item_id)
|
1059
|
+
return item.list_data_access_roles()
|
msfabricpysdkcore/environment.py
CHANGED
@@ -135,16 +135,17 @@ class Environment(Item):
|
|
135
135
|
# POST https://api.fabric.microsoft.com/v1/workspaces/{workspaceId}/environments/{environmentId}/staging/libraries
|
136
136
|
raise NotImplementedError("Not implemented yet")
|
137
137
|
# url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/environments/{self.id}/staging/libraries"
|
138
|
-
|
139
|
-
#
|
140
|
-
#
|
141
|
-
#
|
142
|
-
#
|
143
|
-
#
|
144
|
-
#
|
145
|
-
#
|
146
|
-
#
|
147
|
-
#
|
138
|
+
# with open(file_path, 'rb') as f:
|
139
|
+
# files = {"upload_file.whl": f}
|
140
|
+
# for _ in range(10):
|
141
|
+
# response = requests.post(url=url, files=files, headers=self.auth.get_headers())
|
142
|
+
# if response.status_code == 429:
|
143
|
+
# print("Too many requests, waiting 10 seconds")
|
144
|
+
# sleep(10)
|
145
|
+
# continue
|
146
|
+
# if response.status_code not in (200, 429):
|
147
|
+
# raise Exception(f"Error uploading staging libraries: {response.status_code}, {response.text}")
|
148
|
+
# break
|
148
149
|
|
149
150
|
# return json.loads(response.text)
|
150
151
|
|
msfabricpysdkcore/item.py
CHANGED
@@ -6,7 +6,6 @@ from msfabricpysdkcore.onelakeshortcut import OneLakeShortcut
|
|
6
6
|
from msfabricpysdkcore.job_instance import JobInstance
|
7
7
|
from msfabricpysdkcore.long_running_operation import check_long_running_operation
|
8
8
|
|
9
|
-
|
10
9
|
class Item:
|
11
10
|
"""Class to represent a item in Microsoft Fabric"""
|
12
11
|
|
@@ -247,10 +246,7 @@ class Item:
|
|
247
246
|
sleep(10)
|
248
247
|
continue
|
249
248
|
if response.status_code not in (200, 429):
|
250
|
-
|
251
|
-
print(response.text)
|
252
|
-
|
253
|
-
raise Exception(f"Error getting job instance: {response.text}")
|
249
|
+
raise Exception(f"Error getting job instance: {response.status_code}, {response.text}")
|
254
250
|
break
|
255
251
|
|
256
252
|
job_dict = json.loads(response.text)
|
@@ -268,4 +264,149 @@ class Item:
|
|
268
264
|
def load_table(self, table_name, path_type, relative_path,
|
269
265
|
file_extension = None, format_options = None,
|
270
266
|
mode = None, recursive = None, wait_for_completion = True):
|
271
|
-
raise NotImplementedError("Load table only works on Lakehouse Items")
|
267
|
+
raise NotImplementedError("Load table only works on Lakehouse Items")
|
268
|
+
|
269
|
+
def create_external_data_share(self, paths, recipient):
|
270
|
+
# POST https://api.fabric.microsoft.com/v1/workspaces/{workspaceId}/items/{itemId}/externalDataShares
|
271
|
+
url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/items/{self.id}/externalDataShares"
|
272
|
+
|
273
|
+
body = {
|
274
|
+
'paths': paths,
|
275
|
+
'recipient': recipient
|
276
|
+
}
|
277
|
+
|
278
|
+
for _ in range(10):
|
279
|
+
response = requests.post(url=url, headers=self.auth.get_headers(), json=body)
|
280
|
+
if response.status_code == 429:
|
281
|
+
print("Too many requests, waiting 10 seconds")
|
282
|
+
sleep(10)
|
283
|
+
continue
|
284
|
+
if response.status_code not in (201, 429):
|
285
|
+
raise Exception(f"Error creating external data share: {response.status_code}, {response.text}")
|
286
|
+
break
|
287
|
+
return json.loads(response.text)
|
288
|
+
|
289
|
+
def get_external_data_share(self, external_data_share_id):
|
290
|
+
# GET https://api.fabric.microsoft.com/v1/workspaces/{workspaceId}/items/{itemId}/externalDataShares/{externalDataShareId}
|
291
|
+
url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/items/{self.id}/externalDataShares/{external_data_share_id}"
|
292
|
+
|
293
|
+
for _ in range(10):
|
294
|
+
response = requests.get(url=url, headers=self.auth.get_headers())
|
295
|
+
if response.status_code == 429:
|
296
|
+
print("Too many requests, waiting 10 seconds")
|
297
|
+
sleep(10)
|
298
|
+
continue
|
299
|
+
if response.status_code not in (200, 429):
|
300
|
+
raise Exception(f"Error getting external data share: {response.status_code}, {response.text}")
|
301
|
+
break
|
302
|
+
return json.loads(response.text)
|
303
|
+
|
304
|
+
def list_external_data_shares_in_item(self, continuationToken = None):
|
305
|
+
# GET GET https://api.fabric.microsoft.com/v1/workspaces/{workspaceId}/items/{itemId}/externalDataShares?continuationToken={continuationToken}
|
306
|
+
|
307
|
+
url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/items/{self.id}/externalDataShares"
|
308
|
+
|
309
|
+
if continuationToken:
|
310
|
+
url += f"?continuationToken={continuationToken}"
|
311
|
+
|
312
|
+
for _ in range(10):
|
313
|
+
response = requests.get(url=url, headers=self.auth.get_headers())
|
314
|
+
if response.status_code == 429:
|
315
|
+
print("Too many requests, waiting 10 seconds")
|
316
|
+
sleep(10)
|
317
|
+
continue
|
318
|
+
|
319
|
+
if response.status_code not in (200, 429):
|
320
|
+
raise Exception(f"Error listing external data shares: {response.status_code}, {response.text}")
|
321
|
+
break
|
322
|
+
|
323
|
+
resp = json.loads(response.text)
|
324
|
+
external_data_shares = resp['value']
|
325
|
+
|
326
|
+
if 'continuationToken' in resp:
|
327
|
+
external_data_shares_new = self.list_external_data_shares_in_item(continuationToken=resp['continuationToken'])
|
328
|
+
external_data_shares.extend(external_data_shares_new)
|
329
|
+
|
330
|
+
return external_data_shares
|
331
|
+
|
332
|
+
def revoke_external_data_share(self, external_data_share_id):
|
333
|
+
# POST https://api.fabric.microsoft.com/v1/workspaces/{workspaceId}/items/{itemId}/externalDataShares/{externalDataShareId}/revoke
|
334
|
+
|
335
|
+
url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/items/{self.id}/externalDataShares/{external_data_share_id}/revoke"
|
336
|
+
|
337
|
+
for _ in range(10):
|
338
|
+
response = requests.post(url=url, headers=self.auth.get_headers())
|
339
|
+
if response.status_code == 429:
|
340
|
+
print("Too many requests, waiting 10 seconds")
|
341
|
+
sleep(10)
|
342
|
+
continue
|
343
|
+
if response.status_code not in (200, 429):
|
344
|
+
raise Exception(f"Error revoking external data share: {response.status_code}, {response.text}")
|
345
|
+
break
|
346
|
+
|
347
|
+
return response.status_code
|
348
|
+
|
349
|
+
|
350
|
+
# One Lake data access security
|
351
|
+
|
352
|
+
def list_data_access_roles(self, continuationToken = None):
|
353
|
+
# GET https://api.fabric.microsoft.com/v1/workspaces/{workspaceId}/items/{itemId}/dataAccessRoles
|
354
|
+
|
355
|
+
url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/items/{self.id}/dataAccessRoles"
|
356
|
+
|
357
|
+
if continuationToken:
|
358
|
+
url += f"?continuationToken={continuationToken}"
|
359
|
+
|
360
|
+
for _ in range(10):
|
361
|
+
response = requests.get(url=url, headers=self.auth.get_headers())
|
362
|
+
if response.status_code == 429:
|
363
|
+
print("Too many requests, waiting 10 seconds")
|
364
|
+
sleep(10)
|
365
|
+
continue
|
366
|
+
if response.status_code not in (200, 429):
|
367
|
+
raise Exception(f"Error revoking external data share: {response.status_code}, {response.text}")
|
368
|
+
break
|
369
|
+
|
370
|
+
resp_dict = json.loads(response.text)
|
371
|
+
data_access_roles = resp_dict['value']
|
372
|
+
etag = response.headers.get('ETag', None)
|
373
|
+
|
374
|
+
if 'continuationToken' in resp_dict and resp_dict['continuationToken']:
|
375
|
+
data_access_roles_new, etag = self.list_data_access_roles(continuationToken=resp_dict['continuationToken'])
|
376
|
+
data_access_roles.extend(data_access_roles_new)
|
377
|
+
|
378
|
+
return data_access_roles, etag
|
379
|
+
|
380
|
+
def create_or_update_data_access_roles(self, data_access_roles, dryrun = False, etag_match = None):
|
381
|
+
# PUT https://api.fabric.microsoft.com/v1/workspaces/{workspaceId}/items/{itemId}/dataAccessRoles
|
382
|
+
|
383
|
+
url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/items/{self.id}/dataAccessRoles"
|
384
|
+
|
385
|
+
if dryrun:
|
386
|
+
url += "?dryrun=true"
|
387
|
+
|
388
|
+
if etag_match:
|
389
|
+
if 'If-Match' in etag_match:
|
390
|
+
headers = self.auth.get_headers()
|
391
|
+
headers['If-Match'] = etag_match['If-Match']
|
392
|
+
elif 'If-None-Match' in etag_match:
|
393
|
+
headers = self.auth.get_headers()
|
394
|
+
headers['If-None-Match'] = etag_match['If-None-Match']
|
395
|
+
else:
|
396
|
+
raise Exception("Etag match should include If-Match or If-None-Match")
|
397
|
+
else:
|
398
|
+
headers = self.auth.get_headers()
|
399
|
+
|
400
|
+
body = {"value" : data_access_roles}
|
401
|
+
|
402
|
+
for _ in range(10):
|
403
|
+
response = requests.put(url=url, headers=headers, json=body)
|
404
|
+
if response.status_code == 429:
|
405
|
+
print("Too many requests, waiting 10 seconds")
|
406
|
+
sleep(10)
|
407
|
+
continue
|
408
|
+
if response.status_code not in (200, 429):
|
409
|
+
raise Exception(f"Error creating or updating data access roles: {response.status_code}, {response.text}")
|
410
|
+
break
|
411
|
+
|
412
|
+
return response
|
@@ -1,6 +1,6 @@
|
|
1
1
|
import json
|
2
2
|
import requests
|
3
|
-
from time import sleep
|
3
|
+
from time import sleep, time
|
4
4
|
|
5
5
|
class LongRunningOperation:
|
6
6
|
"""Class to represent a workspace in Microsoft Fabric"""
|
@@ -49,12 +49,17 @@ class LongRunningOperation:
|
|
49
49
|
def wait_for_completion(self):
|
50
50
|
"""Wait for the operation to complete"""
|
51
51
|
max_iter = 20
|
52
|
+
start_time = time()
|
52
53
|
while self.state not in ('Succeeded', 'Failed'):
|
53
54
|
self.state = self.get_operation_state()["status"]
|
55
|
+
duration = int(time() - start_time)
|
56
|
+
if duration > 60:
|
57
|
+
|
58
|
+
if self.state == "Running":
|
59
|
+
print(f"Operation did not complete after {duration} seconds")
|
60
|
+
return "Running"
|
61
|
+
raise Exception(f"Operation did not complete after {duration} seconds")
|
54
62
|
sleep(3)
|
55
|
-
if max_iter == 0:
|
56
|
-
raise Exception("Operation did not complete after 60 seconds")
|
57
|
-
max_iter -= 1
|
58
63
|
return self.state
|
59
64
|
|
60
65
|
|
msfabricpysdkcore/otheritems.py
CHANGED
@@ -2,6 +2,7 @@ import json
|
|
2
2
|
from time import sleep
|
3
3
|
import requests
|
4
4
|
from msfabricpysdkcore.item import Item
|
5
|
+
from msfabricpysdkcore.long_running_operation import check_long_running_operation
|
5
6
|
|
6
7
|
|
7
8
|
class Eventhouse(Item):
|
@@ -52,6 +53,34 @@ class SparkJobDefinition(Item):
|
|
52
53
|
def update_definition(self, definition):
|
53
54
|
return super().update_definition(definition=definition, type="sparkJobDefinitions")
|
54
55
|
|
56
|
+
def run_on_demand_spark_job_definition(self, job_type = "sparkjob"):
|
57
|
+
# POST https://api.fabric.microsoft.com/v1/workspaces/{workspaceId}/sparkJobDefinitions/{sparkJobDefinitionId}/jobs/instances?jobType={jobType}
|
58
|
+
"""Method to run a spark job definition on demand"""
|
59
|
+
url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/sparkJobDefinitions/{self.id}/jobs/instances?jobType={job_type}"
|
60
|
+
|
61
|
+
for _ in range(10):
|
62
|
+
response = requests.post(url=url, headers=self.auth.get_headers())
|
63
|
+
if response.status_code == 429:
|
64
|
+
print("Too many requests, waiting 10 seconds")
|
65
|
+
sleep(10)
|
66
|
+
continue
|
67
|
+
if response.status_code == 202:
|
68
|
+
location = response.headers['Location']
|
69
|
+
job_instance_id = location.split('/')[-1]
|
70
|
+
|
71
|
+
from msfabricpysdkcore import FabricClientCore
|
72
|
+
fc = FabricClientCore(silent=True)
|
73
|
+
fc.auth = self.auth
|
74
|
+
return fc.get_item_job_instance(workspace_id = self.workspace_id,
|
75
|
+
item_id = self.id,
|
76
|
+
job_instance_id = job_instance_id)
|
77
|
+
if response.status_code not in (200, 201, 202, 429):
|
78
|
+
raise Exception(f"Error running on demand spark job definition: {response.status_code}, {response.text}")
|
79
|
+
break
|
80
|
+
|
81
|
+
return response
|
82
|
+
|
83
|
+
|
55
84
|
class Warehouse(Item):
|
56
85
|
"""Class to represent a warehouse in Microsoft Fabric"""
|
57
86
|
|
@@ -77,3 +77,23 @@ class TestFabricClientCore(unittest.TestCase):
|
|
77
77
|
self.assertEqual(resp["itemsChangeLabelStatus"][0]["status"], "Succeeded")
|
78
78
|
resp = fca.bulk_remove_labels(items=items)
|
79
79
|
self.assertEqual(resp["itemsChangeLabelStatus"][0]["status"], "Succeeded")
|
80
|
+
|
81
|
+
def test_admin_external_data_shares(self):
|
82
|
+
|
83
|
+
fca = self.fca
|
84
|
+
|
85
|
+
data_shares = fca.list_external_data_shares()
|
86
|
+
ws = fca.list_workspaces(name="testworkspace")[0]
|
87
|
+
|
88
|
+
data_shares = [d for d in data_shares if d['workspaceId'] == ws.id]
|
89
|
+
|
90
|
+
self.assertGreater(len(data_shares), 0)
|
91
|
+
fca.revoke_external_data_share(external_data_share_id = data_shares[0]['id'],
|
92
|
+
item_id = data_shares[0]['itemId'],
|
93
|
+
workspace_id = data_shares[0]['workspaceId'])
|
94
|
+
data_shares = fca.list_external_data_shares()
|
95
|
+
ws = fca.list_workspaces(name="testworkspace")[0]
|
96
|
+
|
97
|
+
data_shares = [d for d in data_shares if d['workspaceId'] == ws.id]
|
98
|
+
|
99
|
+
self.assertEqual(data_shares[0]['status'], 'Revoked')
|
@@ -53,8 +53,7 @@ class TestFabricClientCore(unittest.TestCase):
|
|
53
53
|
"itemType": item["itemType"]}
|
54
54
|
items = [item]
|
55
55
|
|
56
|
-
|
57
|
-
response = pipe.deploy(source_stage_id=dev_stage.id,target_stage_id=prod_stage.id, items=items)
|
56
|
+
response = fc.deploy_stage_content(deployment_pipeline_id=pipe_id, source_stage_id=dev_stage.id,target_stage_id=prod_stage.id, items=items)
|
58
57
|
|
59
58
|
self.assertEqual(response["status"], "Succeeded")
|
60
59
|
|
@@ -0,0 +1,51 @@
|
|
1
|
+
import unittest
|
2
|
+
from dotenv import load_dotenv
|
3
|
+
from msfabricpysdkcore.coreapi import FabricClientCore
|
4
|
+
|
5
|
+
load_dotenv()
|
6
|
+
|
7
|
+
class TestFabricClientCore(unittest.TestCase):
|
8
|
+
|
9
|
+
def __init__(self, *args, **kwargs):
|
10
|
+
super(TestFabricClientCore, self).__init__(*args, **kwargs)
|
11
|
+
self.fc = FabricClientCore()
|
12
|
+
|
13
|
+
def test_external_data_shares(self):
|
14
|
+
|
15
|
+
fc = self.fc
|
16
|
+
|
17
|
+
workspace_id = 'c3352d34-0b54-40f0-b204-cc964b1beb8d'
|
18
|
+
item_id = 'e2c09c89-bf97-4f71-bdeb-36338795ec36'
|
19
|
+
|
20
|
+
recipient = {
|
21
|
+
"userPrincipalName": "lisa4@fabrikam.com"
|
22
|
+
}
|
23
|
+
paths=["Files/external"]
|
24
|
+
|
25
|
+
resp = fc.create_external_data_share(workspace_id, item_id, paths, recipient)
|
26
|
+
self.assertIsNotNone(resp)
|
27
|
+
self.assertIn('id', resp)
|
28
|
+
|
29
|
+
|
30
|
+
get = fc.get_external_data_share(workspace_id, item_id, resp['id'])
|
31
|
+
self.assertIsNotNone(get)
|
32
|
+
self.assertEqual(get['id'], resp['id'])
|
33
|
+
|
34
|
+
|
35
|
+
resp = fc.list_external_data_shares_in_item(workspace_id, item_id)
|
36
|
+
self.assertGreater(len(resp), 0)
|
37
|
+
|
38
|
+
data_share_ids = [ds['id'] for ds in resp]
|
39
|
+
self.assertIn(get['id'], data_share_ids)
|
40
|
+
|
41
|
+
|
42
|
+
resp = fc.revoke_external_data_share(workspace_id, item_id, get['id'])
|
43
|
+
self.assertEqual(resp, 200)
|
44
|
+
|
45
|
+
get2 = fc.get_external_data_share(workspace_id, item_id, get['id'])
|
46
|
+
self.assertIsNotNone(get2)
|
47
|
+
|
48
|
+
self.assertEqual(get['id'], get2['id'])
|
49
|
+
self.assertEqual(get2['status'], 'Revoked')
|
50
|
+
|
51
|
+
|
@@ -30,7 +30,7 @@ class TestFabricClientCore(unittest.TestCase):
|
|
30
30
|
|
31
31
|
status_code = self.fc.git_connect(workspace_id=ws2.id, git_provider_details=git_provider_details)
|
32
32
|
|
33
|
-
self.assertEqual(status_code,
|
33
|
+
self.assertEqual(status_code, 200)
|
34
34
|
|
35
35
|
initialization_strategy = "PreferWorkspace"
|
36
36
|
|
@@ -56,7 +56,7 @@ class TestFabricClientCore(unittest.TestCase):
|
|
56
56
|
|
57
57
|
status_code = self.fc.git_disconnect(workspace_id=ws2.id)
|
58
58
|
|
59
|
-
self.assertEqual(status_code,
|
59
|
+
self.assertEqual(status_code, 200)
|
60
60
|
|
61
61
|
ws2.delete()
|
62
62
|
|
@@ -173,23 +173,24 @@ class TestFabricClientCore(unittest.TestCase):
|
|
173
173
|
|
174
174
|
fc = self.fc
|
175
175
|
workspace_id = 'd8a5abe0-9eed-406d-ab46-343bc57ddbe5'
|
176
|
-
|
177
|
-
|
178
|
-
|
176
|
+
datetime_str = datetime.now().strftime("%Y%m%d%H%M%S")
|
177
|
+
eventhouse_name = "evh" + datetime_str
|
178
|
+
eventhouse1 = fc.create_eventhouse(workspace_id, display_name=eventhouse_name)
|
179
|
+
self.assertEqual(eventhouse1.display_name, eventhouse_name)
|
179
180
|
|
180
181
|
eventhouses = fc.list_eventhouses(workspace_id)
|
181
182
|
eventhouse_names = [eh.display_name for eh in eventhouses]
|
182
183
|
self.assertGreater(len(eventhouses), 0)
|
183
|
-
self.assertIn(
|
184
|
+
self.assertIn(eventhouse_name, eventhouse_names)
|
184
185
|
|
185
|
-
eh = fc.get_eventhouse(workspace_id, eventhouse_name=
|
186
|
+
eh = fc.get_eventhouse(workspace_id, eventhouse_name=eventhouse_name)
|
186
187
|
self.assertIsNotNone(eh.id)
|
187
|
-
self.assertEqual(eh.display_name,
|
188
|
-
|
189
|
-
eh2 = fc.update_eventhouse(workspace_id, eh.id, display_name=
|
188
|
+
self.assertEqual(eh.display_name, eventhouse_name)
|
189
|
+
new_display_name = eventhouse_name + "2"
|
190
|
+
eh2 = fc.update_eventhouse(workspace_id, eh.id, display_name=new_display_name)
|
190
191
|
|
191
192
|
eh = fc.get_eventhouse(workspace_id, eventhouse_id=eh.id)
|
192
|
-
self.assertEqual(eh.display_name,
|
193
|
+
self.assertEqual(eh.display_name, new_display_name)
|
193
194
|
self.assertEqual(eh.id, eh2.id)
|
194
195
|
|
195
196
|
status_code = fc.delete_eventhouse(workspace_id, eh.id)
|
@@ -273,11 +274,11 @@ class TestFabricClientCore(unittest.TestCase):
|
|
273
274
|
self.assertIsNotNone(mlm.id)
|
274
275
|
self.assertEqual(mlm.display_name, model_name)
|
275
276
|
|
276
|
-
|
277
|
+
mlm2 = fc.update_ml_model(workspace_id=workspace_id,ml_model_id= mlm.id, description=model_name)
|
277
278
|
|
278
|
-
|
279
|
-
|
280
|
-
|
279
|
+
mlm = fc.get_ml_model(workspace_id, ml_model_id=mlm.id)
|
280
|
+
self.assertEqual(mlm.description, model_name)
|
281
|
+
self.assertEqual(mlm.id, mlm2.id)
|
281
282
|
|
282
283
|
status_code = fc.delete_ml_model(workspace_id, mlm.id)
|
283
284
|
self.assertEqual(status_code, 200)
|
@@ -385,45 +386,6 @@ class TestFabricClientCore(unittest.TestCase):
|
|
385
386
|
status_code = fc.delete_semantic_model(workspace_id, sm.id)
|
386
387
|
self.assertEqual(status_code, 200)
|
387
388
|
|
388
|
-
|
389
|
-
def test_spark_job_definitions(self):
|
390
|
-
|
391
|
-
fc = self.fc
|
392
|
-
workspace_id = 'd8a5abe0-9eed-406d-ab46-343bc57ddbe5'
|
393
|
-
|
394
|
-
datetime_str = datetime.now().strftime("%Y%m%d%H%M%S")
|
395
|
-
spark_job_definition_name = f"sjd{datetime_str}"
|
396
|
-
|
397
|
-
spark_job_definition_w_content = fc.get_spark_job_definition(workspace_id, spark_job_definition_name="helloworld")
|
398
|
-
|
399
|
-
definition = fc.get_spark_job_definition_definition(workspace_id, spark_job_definition_w_content.id)
|
400
|
-
|
401
|
-
self.assertIsNotNone(definition)
|
402
|
-
self.assertIn("definition", definition)
|
403
|
-
definition = definition["definition"]
|
404
|
-
|
405
|
-
spark_job_definition = fc.create_spark_job_definition(workspace_id, display_name=spark_job_definition_name)
|
406
|
-
fc.update_spark_job_definition_definition(workspace_id, spark_job_definition.id, definition=definition)
|
407
|
-
spark_job_definition = fc.get_spark_job_definition(workspace_id, spark_job_definition_id=spark_job_definition.id)
|
408
|
-
self.assertEqual(spark_job_definition.display_name, spark_job_definition_name)
|
409
|
-
self.assertIsNotNone(spark_job_definition.definition)
|
410
|
-
|
411
|
-
spark_job_definitions = fc.list_spark_job_definitions(workspace_id)
|
412
|
-
spark_job_definition_names = [sjd.display_name for sjd in spark_job_definitions]
|
413
|
-
self.assertGreater(len(spark_job_definitions), 0)
|
414
|
-
self.assertIn(spark_job_definition_name, spark_job_definition_names)
|
415
|
-
|
416
|
-
sjd = fc.get_spark_job_definition(workspace_id, spark_job_definition_name=spark_job_definition_name)
|
417
|
-
self.assertIsNotNone(sjd.id)
|
418
|
-
self.assertEqual(sjd.display_name, spark_job_definition_name)
|
419
|
-
|
420
|
-
sjd = fc.update_spark_job_definition(workspace_id, sjd.id, display_name=f"{spark_job_definition_name}2")
|
421
|
-
sjd = fc.get_spark_job_definition(workspace_id, spark_job_definition_id=sjd.id)
|
422
|
-
self.assertEqual(sjd.display_name, f"{spark_job_definition_name}2")
|
423
|
-
|
424
|
-
status_code = fc.delete_spark_job_definition(workspace_id, sjd.id)
|
425
|
-
self.assertEqual(status_code, 200)
|
426
|
-
|
427
389
|
def test_warehouses(self):
|
428
390
|
|
429
391
|
fc = self.fc
|