msfabricpysdkcore 0.0.13__py3-none-any.whl → 0.1.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. msfabricpysdkcore/__init__.py +2 -1
  2. msfabricpysdkcore/admin_item.py +19 -45
  3. msfabricpysdkcore/admin_workspace.py +13 -60
  4. msfabricpysdkcore/adminapi.py +401 -476
  5. msfabricpysdkcore/auth.py +10 -6
  6. msfabricpysdkcore/client.py +124 -7
  7. msfabricpysdkcore/coreapi.py +2570 -822
  8. msfabricpysdkcore/deployment_pipeline.py +34 -146
  9. msfabricpysdkcore/domain.py +20 -219
  10. msfabricpysdkcore/environment.py +13 -172
  11. msfabricpysdkcore/fabric_azure_capacity.py +77 -0
  12. msfabricpysdkcore/fabric_azure_client.py +228 -0
  13. msfabricpysdkcore/item.py +55 -331
  14. msfabricpysdkcore/job_instance.py +8 -22
  15. msfabricpysdkcore/lakehouse.py +9 -118
  16. msfabricpysdkcore/long_running_operation.py +7 -37
  17. msfabricpysdkcore/onelakeshortcut.py +7 -21
  18. msfabricpysdkcore/otheritems.py +66 -91
  19. msfabricpysdkcore/spark_custom_pool.py +7 -47
  20. msfabricpysdkcore/tests/test_admin_apis.py +9 -10
  21. msfabricpysdkcore/tests/test_datapipelines.py +15 -18
  22. msfabricpysdkcore/tests/test_deployment_pipeline.py +3 -3
  23. msfabricpysdkcore/tests/test_domains.py +6 -5
  24. msfabricpysdkcore/tests/test_environments.py +54 -5
  25. msfabricpysdkcore/tests/test_evenhouses.py +47 -0
  26. msfabricpysdkcore/tests/test_evenstreams.py +20 -20
  27. msfabricpysdkcore/tests/test_external_data_shares.py +3 -3
  28. msfabricpysdkcore/tests/test_fabric_azure_client.py +78 -0
  29. msfabricpysdkcore/tests/test_git.py +8 -9
  30. msfabricpysdkcore/tests/test_items.py +81 -0
  31. msfabricpysdkcore/tests/test_jobs.py +2 -2
  32. msfabricpysdkcore/tests/test_kql_queryset.py +49 -0
  33. msfabricpysdkcore/tests/test_kqldatabases.py +3 -3
  34. msfabricpysdkcore/tests/test_lakehouse.py +84 -0
  35. msfabricpysdkcore/tests/test_ml_experiments.py +47 -0
  36. msfabricpysdkcore/tests/test_ml_models.py +47 -0
  37. msfabricpysdkcore/tests/test_notebooks.py +57 -0
  38. msfabricpysdkcore/tests/test_one_lake_data_access_security.py +2 -4
  39. msfabricpysdkcore/tests/test_other_items.py +45 -0
  40. msfabricpysdkcore/tests/test_reports.py +52 -0
  41. msfabricpysdkcore/tests/test_semantic_model.py +50 -0
  42. msfabricpysdkcore/tests/test_shortcuts.py +4 -4
  43. msfabricpysdkcore/tests/test_spark.py +9 -9
  44. msfabricpysdkcore/tests/test_sparkjobdefinition.py +2 -2
  45. msfabricpysdkcore/tests/test_warehouses.py +50 -0
  46. msfabricpysdkcore/tests/test_workspaces_capacities.py +16 -13
  47. msfabricpysdkcore/workspace.py +397 -1163
  48. {msfabricpysdkcore-0.0.13.dist-info → msfabricpysdkcore-0.1.2.dist-info}/METADATA +72 -10
  49. msfabricpysdkcore-0.1.2.dist-info/RECORD +55 -0
  50. {msfabricpysdkcore-0.0.13.dist-info → msfabricpysdkcore-0.1.2.dist-info}/WHEEL +1 -1
  51. msfabricpysdkcore-0.0.13.dist-info/RECORD +0 -41
  52. {msfabricpysdkcore-0.0.13.dist-info → msfabricpysdkcore-0.1.2.dist-info}/LICENSE +0 -0
  53. {msfabricpysdkcore-0.0.13.dist-info → msfabricpysdkcore-0.1.2.dist-info}/top_level.txt +0 -0
msfabricpysdkcore/item.py CHANGED
@@ -1,15 +1,11 @@
1
1
  import json
2
- import requests
3
- from time import sleep
4
2
 
5
- from msfabricpysdkcore.onelakeshortcut import OneLakeShortcut
6
- from msfabricpysdkcore.job_instance import JobInstance
7
- from msfabricpysdkcore.long_running_operation import check_long_running_operation
3
+ from msfabricpysdkcore.coreapi import FabricClientCore
8
4
 
9
5
  class Item:
10
6
  """Class to represent a item in Microsoft Fabric"""
11
7
 
12
- def __init__(self, id, display_name, type, workspace_id, auth, properties = None, definition=None, description="") -> None:
8
+ def __init__(self, id, display_name, type, workspace_id, core_client: FabricClientCore, properties = None, definition=None, description="") -> None:
13
9
 
14
10
  self.id = id
15
11
  self.display_name = display_name
@@ -19,7 +15,7 @@ class Item:
19
15
  self.properties = properties
20
16
  self.workspace_id = workspace_id
21
17
 
22
- self.auth = auth
18
+ self.core_client = core_client
23
19
 
24
20
  def __str__(self) -> str:
25
21
  """Return a string representation of the workspace object"""
@@ -37,376 +33,104 @@ class Item:
37
33
  def __repr__(self) -> str:
38
34
  return self.__str__()
39
35
 
40
- def from_dict(item_dict, auth):
36
+ def from_dict(item_dict, core_client):
41
37
  """Create Item object from dictionary"""
42
38
 
43
39
  return Item(id=item_dict['id'], display_name=item_dict['displayName'], type=item_dict['type'], workspace_id=item_dict['workspaceId'],
44
40
  properties=item_dict.get('properties', None),
45
- definition=item_dict.get('definition', None), description=item_dict.get('description', ""), auth=auth)
41
+ definition=item_dict.get('definition', None), description=item_dict.get('description', ""), core_client=core_client)
46
42
 
47
43
  def delete(self, type = None):
48
44
  """Delete the workspace item"""
49
45
 
50
- url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/items/{self.id}"
51
- if type:
52
- url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/{type}/{self.id}"
53
- for _ in range(10):
54
- response = requests.delete(url=url, headers=self.auth.get_headers())
55
- if response.status_code == 429:
56
- print("Too many requests, waiting 10 seconds")
57
- sleep(10)
58
- continue
59
- if response.status_code not in (200, 429):
60
- raise Exception(f"Error deleting item: {response.status_code}, {response.text}")
61
- break
62
-
63
- return response.status_code
46
+ return self.core_client.delete_item(self.workspace_id, self.id, type=type)
64
47
 
65
48
  def get_definition(self, type = None, format = None):
66
49
  """Get the definition of the item"""
67
-
68
- url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/items/{self.id}/getDefinition"
69
- if type:
70
- url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/{type}/{self.id}/getDefinition"
71
-
72
- if format:
73
- url += f"?format={format}"
50
+ resp_dict = self.core_client.get_item_definition(self.workspace_id, self.id, type=type, format=format)
74
51
 
75
- for _ in range(10):
76
- response = requests.post(url=url, headers=self.auth.get_headers())
77
- if response.status_code == 429:
78
- print("Too many requests, waiting 10 seconds")
79
- sleep(10)
80
- continue
81
- if response.status_code == 202:
82
- operation_result = check_long_running_operation( response.headers, self.auth)
83
- self.definition = operation_result['definition']
84
- return operation_result
85
-
86
- if response.status_code not in (200, 202, 429):
87
- print(response.status_code)
88
- print(response.text)
89
- raise Exception(f"Error getting item definition: {response.text}")
90
- break
91
-
92
- resp_dict = json.loads(response.text)
93
52
  self.definition = resp_dict['definition']
94
53
  return resp_dict
95
-
96
- def update(self, display_name = None, description = None, type = None):
97
- """Update the item"""
98
- url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/items/{self.id}"
99
- if type:
100
- url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/{type}/{self.id}"
101
54
 
102
- payload = dict()
103
- if display_name:
104
- payload['displayName'] = display_name
105
- if description:
106
- payload['description'] = description
107
- for _ in range(10):
108
- response = requests.patch(url=url, headers=self.auth.get_headers(), json=payload)
109
- if response.status_code == 429:
110
- print("Too many requests, waiting 10 seconds")
111
- sleep(10)
112
- continue
113
- if response.status_code not in (200, 429):
114
- print(response.status_code)
115
- print(response.text)
116
55
 
117
- raise Exception(f"Error updating item: {response.text}")
118
- break
56
+ def update(self, display_name = None, description = None, type = None, return_item="Default"):
57
+ """Update the item"""
58
+
59
+ resp_dict = self.core_client.update_item(workspace_id=self.workspace_id, item_id=self.id,
60
+ display_name=display_name, description=description, type=type,
61
+ return_item=return_item)
119
62
  if display_name:
120
- self.display_name = payload['displayName']
63
+ self.display_name = display_name
121
64
  if description:
122
- self.description = payload['description']
65
+ self.description = description
123
66
 
124
- return self
67
+ return resp_dict
125
68
 
126
69
  def update_definition(self, definition, type = None):
127
70
  """Update the item definition"""
128
- url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/items/{self.id}/updateDefinition"
129
-
130
- if type:
131
- url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/{type}/{self.id}/updateDefinition"
132
-
133
- payload = {
134
- 'definition': definition
135
- }
136
- for _ in range(10):
137
- response = requests.post(url=url, headers=self.auth.get_headers(), json=payload)
138
- if response.status_code == 429:
139
- print("Too many requests, waiting 10 seconds")
140
- sleep(10)
141
- continue
142
- if response.status_code == 202:
143
- check_long_running_operation( response.headers, self.auth)
144
- if response.status_code not in (200, 202, 429):
145
- print(response.status_code)
146
- print(response.text)
71
+ response = self.core_client.update_item_definition(workspace_id=self.workspace_id, item_id=self.id,
72
+ definition=definition, type=type)
147
73
 
148
- raise Exception(f"Error updating item definition: {response.text}")
149
- break
150
-
151
- self.definition = payload['definition']
152
- return self
74
+ self.definition = definition
75
+ return response
153
76
 
154
- def get_shortcut(self, path, name):
155
- """Get the shortcut in the item"""
156
- url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/items/{self.id}/shortcuts/{path}/{name}"
157
-
158
- for _ in range(10):
159
- response = requests.get(url=url, headers=self.auth.get_headers())
160
- if response.status_code == 429:
161
- print("Too many requests, waiting 10 seconds")
162
- sleep(10)
163
- continue
164
- if response.status_code not in (200, 429):
165
- print(response.status_code)
166
- print(response.text)
77
+ # Shortcut
167
78
 
168
- raise Exception(f"Error getting shortcut: {response.text}")
169
- break
170
-
171
- shortcut_dict = json.loads(response.text)
172
- shortcut_dict['workspaceId'] = self.workspace_id
173
- shortcut_dict['itemId'] = self.id
174
- return OneLakeShortcut.from_dict(shortcut_dict,
175
- auth = self.auth)
176
-
177
79
  def create_shortcut(self, path, name, target):
178
80
  """Create a shortcut in the item"""
179
- url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/items/{self.id}/shortcuts"
180
-
181
- body = {'name': name,
182
- 'path': path,
183
- 'target': target}
184
-
185
- for _ in range(10):
186
- response = requests.post(url=url, headers=self.auth.get_headers(), json=body)
187
- if response.status_code == 429:
188
- print("Too many requests, waiting 10 seconds")
189
- sleep(10)
190
- continue
191
- if response.status_code not in (201, 429):
192
- print(response.status_code)
193
- print(response.text)
194
-
195
- raise Exception(f"Error creating shortcut: {response.text}")
196
- break
197
-
198
- shortcut_dict = json.loads(response.text)
199
- shortcut_dict['workspaceId'] = self.workspace_id
200
- shortcut_dict['itemId'] = self.id
201
- return OneLakeShortcut.from_dict(shortcut_dict,
202
- auth = self.auth)
81
+ return self.core_client.create_shortcut(workspace_id=self.workspace_id, item_id=self.id,
82
+ path=path, name=name, target=target)
203
83
 
204
84
  def delete_shortcut(self, path, name):
205
85
  """Delete the shortcut in the item"""
206
- return self.get_shortcut(path, name).delete()
207
-
208
-
209
- def run_on_demand_item_job(self, job_type, execution_data = None):
210
- """Run an on demand job on the item"""
86
+ return self.core_client.delete_shortcut(workspace_id=self.workspace_id, item_id=self.id,
87
+ path=path, name=name)
211
88
 
212
- # POST https://api.fabric.microsoft.com/v1/workspaces/{workspaceId}/items/{itemId}/jobs/instances?jobType={jobType}
213
- url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/items/{self.id}/jobs/instances?jobType={job_type}"
214
- payload = {
215
- 'executionData': execution_data
216
- }
217
-
218
- for _ in range(10):
219
- if execution_data:
220
- response = requests.post(url=url, headers=self.auth.get_headers(), json=payload)
221
- else:
222
- response = requests.post(url=url, headers=self.auth.get_headers())
223
- if response.status_code == 429:
224
- print("Too many requests, waiting 10 seconds")
225
- sleep(10)
226
- continue
227
-
228
- if response.status_code not in (202, 429):
229
- print(response.status_code)
230
- print(response.text)
231
-
232
- raise Exception(f"Error running on demand job: {response.text}")
233
- break
234
- job_instance_id = response.headers["Location"].split("/")[-1]
235
- job_instance = self.get_item_job_instance(job_instance_id=job_instance_id)
236
- return job_instance
237
-
238
- def get_item_job_instance(self, job_instance_id):
239
- """Get the job instance of the item"""
89
+ def get_shortcut(self, path, name):
90
+ """Get the shortcut in the item"""
91
+ return self.core_client.get_shortcut(workspace_id=self.workspace_id, item_id=self.id,
92
+ path=path, name=name)
93
+
240
94
 
241
- url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/items/{self.id}/jobs/instances/{job_instance_id}"
242
- for _ in range(10):
243
- response = requests.get(url=url, headers=self.auth.get_headers())
244
- if response.status_code == 429:
245
- print("Too many requests, waiting 10 seconds")
246
- sleep(10)
247
- continue
248
- if response.status_code not in (200, 429):
249
- raise Exception(f"Error getting job instance: {response.status_code}, {response.text}")
250
- break
95
+ # Job Scheduler
251
96
 
252
- job_dict = json.loads(response.text)
253
- job_dict['workspaceId'] = self.workspace_id
254
- job_dict['itemId'] = self.id
255
- return JobInstance.from_dict(job_dict, auth=self.auth)
256
-
257
97
  def cancel_item_job_instance(self, job_instance_id):
258
98
  """Cancel a job instance ofjob the item"""
259
- return self.get_item_job_instance(job_instance_id=job_instance_id).cancel()
99
+ return self.core_client.cancel_item_job_instance(workspace_id=self.workspace_id, item_id=self.id,
100
+ job_instance_id=job_instance_id)
260
101
 
261
- def list_tables(self):
262
- raise NotImplementedError("List tables only works on Lakehouse Items")
263
-
264
- def load_table(self, table_name, path_type, relative_path,
265
- file_extension = None, format_options = None,
266
- mode = None, recursive = None, wait_for_completion = True):
267
- raise NotImplementedError("Load table only works on Lakehouse Items")
268
-
269
- def create_external_data_share(self, paths, recipient):
270
- # POST https://api.fabric.microsoft.com/v1/workspaces/{workspaceId}/items/{itemId}/externalDataShares
271
- url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/items/{self.id}/externalDataShares"
272
-
273
- body = {
274
- 'paths': paths,
275
- 'recipient': recipient
276
- }
102
+ def get_item_job_instance(self, job_instance_id):
103
+ """Get the job instance of the item"""
104
+ return self.core_client.get_item_job_instance(workspace_id=self.workspace_id, item_id=self.id,
105
+ job_instance_id=job_instance_id)
277
106
 
278
- for _ in range(10):
279
- response = requests.post(url=url, headers=self.auth.get_headers(), json=body)
280
- if response.status_code == 429:
281
- print("Too many requests, waiting 10 seconds")
282
- sleep(10)
283
- continue
284
- if response.status_code not in (201, 429):
285
- raise Exception(f"Error creating external data share: {response.status_code}, {response.text}")
286
- break
287
- return json.loads(response.text)
288
-
289
- def get_external_data_share(self, external_data_share_id):
290
- # GET https://api.fabric.microsoft.com/v1/workspaces/{workspaceId}/items/{itemId}/externalDataShares/{externalDataShareId}
291
- url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/items/{self.id}/externalDataShares/{external_data_share_id}"
107
+ def run_on_demand_item_job(self, job_type, execution_data = None):
108
+ return self.core_client.run_on_demand_item_job(workspace_id=self.workspace_id, item_id=self.id,
109
+ job_type=job_type, execution_data=execution_data)
292
110
 
293
- for _ in range(10):
294
- response = requests.get(url=url, headers=self.auth.get_headers())
295
- if response.status_code == 429:
296
- print("Too many requests, waiting 10 seconds")
297
- sleep(10)
298
- continue
299
- if response.status_code not in (200, 429):
300
- raise Exception(f"Error getting external data share: {response.status_code}, {response.text}")
301
- break
302
- return json.loads(response.text)
303
111
 
304
- def list_external_data_shares_in_item(self, continuationToken = None):
305
- # GET GET https://api.fabric.microsoft.com/v1/workspaces/{workspaceId}/items/{itemId}/externalDataShares?continuationToken={continuationToken}
112
+ # External Data Shares
306
113
 
307
- url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/items/{self.id}/externalDataShares"
308
-
309
- if continuationToken:
310
- url += f"?continuationToken={continuationToken}"
311
-
312
- for _ in range(10):
313
- response = requests.get(url=url, headers=self.auth.get_headers())
314
- if response.status_code == 429:
315
- print("Too many requests, waiting 10 seconds")
316
- sleep(10)
317
- continue
318
-
319
- if response.status_code not in (200, 429):
320
- raise Exception(f"Error listing external data shares: {response.status_code}, {response.text}")
321
- break
322
-
323
- resp = json.loads(response.text)
324
- external_data_shares = resp['value']
114
+ def create_external_data_share(self, paths, recipient):
115
+ return self.core_client.create_external_data_share(workspace_id=self.workspace_id, item_id=self.id,
116
+ paths=paths, recipient=recipient)
325
117
 
326
- if 'continuationToken' in resp:
327
- external_data_shares_new = self.list_external_data_shares_in_item(continuationToken=resp['continuationToken'])
328
- external_data_shares.extend(external_data_shares_new)
118
+ def get_external_data_share(self, external_data_share_id):
119
+ return self.core_client.get_external_data_share(workspace_id=self.workspace_id, item_id=self.id,
120
+ external_data_share_id=external_data_share_id)
329
121
 
330
- return external_data_shares
122
+ def list_external_data_shares_in_item(self):
123
+ return self.core_client.list_external_data_shares_in_item(workspace_id=self.workspace_id, item_id=self.id)
331
124
 
332
125
  def revoke_external_data_share(self, external_data_share_id):
333
- # POST https://api.fabric.microsoft.com/v1/workspaces/{workspaceId}/items/{itemId}/externalDataShares/{externalDataShareId}/revoke
334
-
335
- url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/items/{self.id}/externalDataShares/{external_data_share_id}/revoke"
336
-
337
- for _ in range(10):
338
- response = requests.post(url=url, headers=self.auth.get_headers())
339
- if response.status_code == 429:
340
- print("Too many requests, waiting 10 seconds")
341
- sleep(10)
342
- continue
343
- if response.status_code not in (200, 429):
344
- raise Exception(f"Error revoking external data share: {response.status_code}, {response.text}")
345
- break
346
-
347
- return response.status_code
348
-
126
+ return self.core_client.revoke_external_data_share(workspace_id=self.workspace_id, item_id=self.id,
127
+ external_data_share_id=external_data_share_id)
349
128
 
350
129
  # One Lake data access security
351
130
 
352
- def list_data_access_roles(self, continuationToken = None):
353
- # GET https://api.fabric.microsoft.com/v1/workspaces/{workspaceId}/items/{itemId}/dataAccessRoles
354
-
355
- url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/items/{self.id}/dataAccessRoles"
356
-
357
- if continuationToken:
358
- url += f"?continuationToken={continuationToken}"
359
-
360
- for _ in range(10):
361
- response = requests.get(url=url, headers=self.auth.get_headers())
362
- if response.status_code == 429:
363
- print("Too many requests, waiting 10 seconds")
364
- sleep(10)
365
- continue
366
- if response.status_code not in (200, 429):
367
- raise Exception(f"Error revoking external data share: {response.status_code}, {response.text}")
368
- break
369
-
370
- resp_dict = json.loads(response.text)
371
- data_access_roles = resp_dict['value']
372
- etag = response.headers.get('ETag', None)
373
-
374
- if 'continuationToken' in resp_dict and resp_dict['continuationToken']:
375
- data_access_roles_new, etag = self.list_data_access_roles(continuationToken=resp_dict['continuationToken'])
376
- data_access_roles.extend(data_access_roles_new)
377
-
378
- return data_access_roles, etag
131
+ def list_data_access_roles(self):
132
+ return self.core_client.list_data_access_roles(workspace_id=self.workspace_id, item_id=self.id)
379
133
 
380
134
  def create_or_update_data_access_roles(self, data_access_roles, dryrun = False, etag_match = None):
381
- # PUT https://api.fabric.microsoft.com/v1/workspaces/{workspaceId}/items/{itemId}/dataAccessRoles
382
-
383
- url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/items/{self.id}/dataAccessRoles"
384
-
385
- if dryrun:
386
- url += "?dryrun=true"
387
-
388
- if etag_match:
389
- if 'If-Match' in etag_match:
390
- headers = self.auth.get_headers()
391
- headers['If-Match'] = etag_match['If-Match']
392
- elif 'If-None-Match' in etag_match:
393
- headers = self.auth.get_headers()
394
- headers['If-None-Match'] = etag_match['If-None-Match']
395
- else:
396
- raise Exception("Etag match should include If-Match or If-None-Match")
397
- else:
398
- headers = self.auth.get_headers()
399
-
400
- body = {"value" : data_access_roles}
401
-
402
- for _ in range(10):
403
- response = requests.put(url=url, headers=headers, json=body)
404
- if response.status_code == 429:
405
- print("Too many requests, waiting 10 seconds")
406
- sleep(10)
407
- continue
408
- if response.status_code not in (200, 429):
409
- raise Exception(f"Error creating or updating data access roles: {response.status_code}, {response.text}")
410
- break
411
-
412
- return response
135
+ return self.core_client.create_or_update_data_access_roles(workspace_id=self.workspace_id, item_id=self.id,
136
+ data_access_roles=data_access_roles, dryrun=dryrun, etag_match=etag_match)
@@ -1,11 +1,10 @@
1
1
  import json
2
- import requests
3
- from time import sleep
2
+ from msfabricpysdkcore.coreapi import FabricClientCore
4
3
 
5
4
  class JobInstance:
6
5
  """Class to represent a job instance in Microsoft Fabric"""
7
6
 
8
- def __init__(self, id, item_id, workspace_id, auth, job_type, invoke_type, status, root_activity_id,
7
+ def __init__(self, id, item_id, workspace_id, core_client: FabricClientCore, job_type, invoke_type, status, root_activity_id,
9
8
  start_time_utc, end_time_utc, failureReason):
10
9
 
11
10
  self.id = id
@@ -19,7 +18,7 @@ class JobInstance:
19
18
  self.end_time_utc = end_time_utc
20
19
  self.failureReason = failureReason
21
20
 
22
- self.auth = auth
21
+ self.core_client = core_client
23
22
 
24
23
 
25
24
  def __str__(self) -> str:
@@ -41,28 +40,15 @@ class JobInstance:
41
40
  def __repr__(self) -> str:
42
41
  return self.__str__()
43
42
 
44
- def from_dict(job_dict, auth):
43
+ def from_dict(job_dict, core_client):
45
44
  """Create JobInstance object from dictionary"""
46
45
  return JobInstance(id=job_dict['id'], item_id=job_dict['itemId'], workspace_id=job_dict['workspaceId'],
47
46
  job_type=job_dict['jobType'], invoke_type=job_dict['invokeType'], status=job_dict['status'],
48
47
  root_activity_id=job_dict['rootActivityId'], start_time_utc=job_dict['startTimeUtc'],
49
- end_time_utc=job_dict['endTimeUtc'], failureReason=job_dict['failureReason'], auth=auth)
48
+ end_time_utc=job_dict['endTimeUtc'], failureReason=job_dict['failureReason'], core_client=core_client)
50
49
 
51
50
  def cancel(self):
52
51
  """Cancel the job instance"""
53
- url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/items/{self.item_id}/jobs/instances/{self.id}/cancel"
54
- for _ in range(10):
55
- response = requests.post(url=url, headers=self.auth.get_headers())
56
- if response.status_code == 429:
57
- print("Too many requests, waiting 10 seconds")
58
- sleep(10)
59
- continue
60
-
61
- if response.status_code not in (202, 429):
62
- print(response.status_code)
63
- print(response.text)
64
-
65
- raise Exception(f"Error running on demand job: {response.text}")
66
- break
67
-
68
- return response.status_code
52
+ return self.core_client.cancel_item_job_instance(workspace_id=self.workspace_id,
53
+ item_id=self.item_id,
54
+ job_instance_id=self.id)
@@ -1,136 +1,27 @@
1
- import json
2
- import requests
3
- from time import sleep
4
-
5
1
  from msfabricpysdkcore.item import Item
6
- from msfabricpysdkcore.long_running_operation import check_long_running_operation
7
2
 
8
3
  class Lakehouse(Item):
9
4
  """Class to represent a item in Microsoft Fabric"""
10
5
 
11
- def __init__(self, id, display_name, type, workspace_id, auth, properties = None, definition=None, description=""):
12
- super().__init__(id, display_name, type, workspace_id, auth, properties, definition, description)
6
+ def __init__(self, id, display_name, type, workspace_id, core_client, properties = None, definition=None, description=""):
7
+ super().__init__(id, display_name, type, workspace_id, core_client, properties, definition, description)
13
8
 
14
- def from_dict(item_dict, auth):
9
+ def from_dict(item_dict, core_client):
15
10
  return Lakehouse(id=item_dict['id'], display_name=item_dict['displayName'], type=item_dict['type'], workspace_id=item_dict['workspaceId'],
16
11
  properties=item_dict.get('properties', None),
17
- definition=item_dict.get('definition', None), description=item_dict.get('description', ""), auth=auth)
12
+ definition=item_dict.get('definition', None), description=item_dict.get('description', ""), core_client=core_client)
18
13
 
19
- def list_tables(self, continuationToken = None):
14
+ def list_tables(self):
20
15
  """List all tables in the lakehouse"""
21
- # GET https://api.fabric.microsoft.com/v1/workspaces/{workspaceId}/lakehouses/{lakehouseId}/tables
22
- url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/lakehouses/{self.id}/tables"
23
-
24
- if continuationToken:
25
- url = f"{url}?continuationToken={continuationToken}"
26
-
27
- for _ in range(10):
28
- response = requests.get(url=url, headers=self.auth.get_headers())
29
- if response.status_code == 429:
30
- print("Too many requests, waiting 10 seconds")
31
- sleep(10)
32
- continue
33
- if response.status_code not in (200, 429):
34
- print(response.status_code)
35
- print(response.text)
36
- raise Exception(f"Error listing tables: {response.status_code}, {response.text}")
37
- break
38
- resp_dict = json.loads(response.text)
39
-
40
- table_list = resp_dict["data"]
41
-
42
- if "continuationToken" in resp_dict and resp_dict["continuationToken"] is not None:
43
- table_list_next = self.list_tables(continuationToken=resp_dict["continuationToken"])
44
- table_list.extend(table_list_next)
45
-
46
- return table_list
16
+ return self.core_client.list_tables(self.workspace_id, self.id)
47
17
 
48
18
  def load_table(self, table_name, path_type, relative_path,
49
19
  file_extension = None, format_options = None,
50
20
  mode = None, recursive = None, wait_for_completion = True):
51
21
  """Load a table in the lakehouse"""
52
- # POST https://api.fabric.microsoft.com/v1/workspaces/{workspaceId}/lakehouses/{lakehouseId}/tables/{tableName}/load
53
- url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/lakehouses/{self.id}/tables/{table_name}/load"
54
-
55
- body = {
56
- "relativePath": relative_path,
57
- "pathType": path_type,
58
- }
59
-
60
- if file_extension:
61
- body["fileExtension"] = file_extension
62
- if format_options:
63
- body["formatOptions"] = format_options
64
- if mode:
65
- body["mode"] = mode
66
- if recursive:
67
- body["recursive"] = recursive
68
-
69
- for _ in range(10):
70
- response = requests.post(url=url, headers=self.auth.get_headers(), json=body)
71
- if response.status_code == 429:
72
- print("Too many requests, waiting 10 seconds")
73
- sleep(10)
74
- continue
75
- if response.status_code == 202:
76
- if wait_for_completion:
77
- success = self.check_if_table_is_created(table_name)
78
-
79
- if not success:
80
- print("Warning: Table not created after 3 minutes")
81
- else:
82
- print("Table created")
83
- if response.status_code not in (202, 429):
84
- print(response.status_code)
85
- print(response.text)
86
- raise Exception(f"Error loading table: {response.status_code}, {response.text}")
87
- break
88
-
89
- return response.status_code
22
+ return self.core_client.load_table(self.workspace_id, self.id, table_name, path_type, relative_path,
23
+ file_extension, format_options, mode, recursive, wait_for_completion)
90
24
 
91
- def check_if_table_is_created(self, table_name):
92
- """Check if the table is created"""
93
- for _ in range(60):
94
- table_names = [table["name"] for table in self.list_tables()]
95
- if table_name in table_names:
96
- return True
97
-
98
- sleep(3)
99
- return False
100
-
101
- # run on demand table maintenance
102
- # POST https://api.fabric.microsoft.com/v1/workspaces/{workspaceId}/lakehouses/{lakehouseId}/jobs/instances?jobType={jobType}
103
-
104
25
  def run_on_demand_table_maintenance(self, execution_data, job_type = "TableMaintenance", wait_for_completion = True):
105
26
  """Run on demand table maintenance"""
106
- url = f"https://api.fabric.microsoft.com/v1/workspaces/{self.workspace_id}/lakehouses/{self.id}/jobs/instances?jobType={job_type}"
107
-
108
- body = {
109
- "executionData": execution_data
110
- }
111
-
112
- for _ in range(10):
113
- response = requests.post(url=url, headers=self.auth.get_headers(), json=body)
114
- if response.status_code == 429:
115
- print("Too many requests, waiting 10 seconds")
116
- sleep(10)
117
- continue
118
- if response.status_code == 202 and wait_for_completion:
119
- print("successfully started the operation")
120
- try:
121
- operation_result = check_long_running_operation( response.headers, self.auth)
122
- return operation_result
123
- except Exception as e:
124
-
125
- print("Problem waiting for long running operation. Returning initial response.")
126
- print(e)
127
- return response
128
-
129
- if response.status_code not in (200, 202, 429):
130
- print(response.status_code)
131
- print(response.text)
132
-
133
- raise Exception(f"Error at run on demand table maintenance: {response.text}")
134
- break
135
-
136
- return response
27
+ return self.core_client.run_on_demand_table_maintenance(self.workspace_id, self.id, execution_data, job_type, wait_for_completion)