semantic-link-labs 0.8.10__py3-none-any.whl → 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of semantic-link-labs might be problematic. Click here for more details.
- {semantic_link_labs-0.8.10.dist-info → semantic_link_labs-0.9.0.dist-info}/METADATA +6 -5
- {semantic_link_labs-0.8.10.dist-info → semantic_link_labs-0.9.0.dist-info}/RECORD +81 -80
- {semantic_link_labs-0.8.10.dist-info → semantic_link_labs-0.9.0.dist-info}/WHEEL +1 -1
- sempy_labs/__init__.py +34 -3
- sempy_labs/_authentication.py +80 -4
- sempy_labs/_capacities.py +770 -200
- sempy_labs/_capacity_migration.py +7 -37
- sempy_labs/_clear_cache.py +37 -35
- sempy_labs/_connections.py +13 -13
- sempy_labs/_data_pipelines.py +20 -20
- sempy_labs/_dataflows.py +27 -28
- sempy_labs/_dax.py +41 -47
- sempy_labs/_deployment_pipelines.py +1 -1
- sempy_labs/_environments.py +26 -23
- sempy_labs/_eventhouses.py +16 -15
- sempy_labs/_eventstreams.py +16 -15
- sempy_labs/_external_data_shares.py +18 -20
- sempy_labs/_gateways.py +16 -14
- sempy_labs/_generate_semantic_model.py +107 -62
- sempy_labs/_git.py +105 -43
- sempy_labs/_helper_functions.py +251 -194
- sempy_labs/_job_scheduler.py +227 -0
- sempy_labs/_kql_databases.py +16 -15
- sempy_labs/_kql_querysets.py +16 -15
- sempy_labs/_list_functions.py +150 -126
- sempy_labs/_managed_private_endpoints.py +19 -17
- sempy_labs/_mirrored_databases.py +51 -48
- sempy_labs/_mirrored_warehouses.py +5 -4
- sempy_labs/_ml_experiments.py +16 -15
- sempy_labs/_ml_models.py +15 -14
- sempy_labs/_model_bpa.py +210 -207
- sempy_labs/_model_bpa_bulk.py +2 -2
- sempy_labs/_model_bpa_rules.py +3 -3
- sempy_labs/_model_dependencies.py +55 -29
- sempy_labs/_notebooks.py +29 -25
- sempy_labs/_one_lake_integration.py +23 -26
- sempy_labs/_query_scale_out.py +75 -64
- sempy_labs/_refresh_semantic_model.py +25 -26
- sempy_labs/_spark.py +33 -32
- sempy_labs/_sql.py +19 -12
- sempy_labs/_translations.py +10 -7
- sempy_labs/_vertipaq.py +38 -33
- sempy_labs/_warehouses.py +26 -25
- sempy_labs/_workspace_identity.py +11 -10
- sempy_labs/_workspaces.py +40 -33
- sempy_labs/admin/_basic_functions.py +166 -115
- sempy_labs/admin/_domains.py +7 -2
- sempy_labs/admin/_external_data_share.py +3 -3
- sempy_labs/admin/_git.py +4 -1
- sempy_labs/admin/_items.py +11 -6
- sempy_labs/admin/_scanner.py +10 -5
- sempy_labs/directlake/_directlake_schema_compare.py +25 -16
- sempy_labs/directlake/_directlake_schema_sync.py +24 -12
- sempy_labs/directlake/_dl_helper.py +74 -55
- sempy_labs/directlake/_generate_shared_expression.py +10 -9
- sempy_labs/directlake/_get_directlake_lakehouse.py +32 -36
- sempy_labs/directlake/_get_shared_expression.py +4 -3
- sempy_labs/directlake/_guardrails.py +12 -6
- sempy_labs/directlake/_list_directlake_model_calc_tables.py +15 -9
- sempy_labs/directlake/_show_unsupported_directlake_objects.py +16 -10
- sempy_labs/directlake/_update_directlake_model_lakehouse_connection.py +35 -31
- sempy_labs/directlake/_update_directlake_partition_entity.py +39 -31
- sempy_labs/directlake/_warm_cache.py +87 -65
- sempy_labs/lakehouse/_get_lakehouse_columns.py +23 -26
- sempy_labs/lakehouse/_get_lakehouse_tables.py +27 -38
- sempy_labs/lakehouse/_lakehouse.py +7 -20
- sempy_labs/lakehouse/_shortcuts.py +42 -23
- sempy_labs/migration/_create_pqt_file.py +16 -11
- sempy_labs/migration/_refresh_calc_tables.py +16 -10
- sempy_labs/report/_download_report.py +9 -8
- sempy_labs/report/_generate_report.py +85 -44
- sempy_labs/report/_paginated.py +9 -9
- sempy_labs/report/_report_bpa.py +15 -11
- sempy_labs/report/_report_functions.py +80 -91
- sempy_labs/report/_report_helper.py +8 -4
- sempy_labs/report/_report_list_functions.py +24 -13
- sempy_labs/report/_report_rebind.py +17 -16
- sempy_labs/report/_reportwrapper.py +41 -33
- sempy_labs/tom/_model.py +139 -21
- {semantic_link_labs-0.8.10.dist-info → semantic_link_labs-0.9.0.dist-info}/LICENSE +0 -0
- {semantic_link_labs-0.8.10.dist-info → semantic_link_labs-0.9.0.dist-info}/top_level.txt +0 -0
sempy_labs/_helper_functions.py
CHANGED
|
@@ -11,8 +11,8 @@ import datetime
|
|
|
11
11
|
from typing import Optional, Tuple, List
|
|
12
12
|
from uuid import UUID
|
|
13
13
|
import sempy_labs._icons as icons
|
|
14
|
-
import urllib.parse
|
|
15
14
|
from azure.core.credentials import TokenCredential, AccessToken
|
|
15
|
+
import urllib.parse
|
|
16
16
|
import numpy as np
|
|
17
17
|
from IPython.display import display, HTML
|
|
18
18
|
|
|
@@ -52,6 +52,24 @@ def create_abfss_path(
|
|
|
52
52
|
return f"abfss://{lakehouse_workspace_id}@onelake.dfs.fabric.microsoft.com/{lakehouse_id}/Tables/{delta_table_name}"
|
|
53
53
|
|
|
54
54
|
|
|
55
|
+
def _get_default_file_path() -> str:
|
|
56
|
+
|
|
57
|
+
default_file_storage = _get_fabric_context_setting(name="fs.defaultFS")
|
|
58
|
+
|
|
59
|
+
return default_file_storage.split("@")[-1][:-1]
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _split_abfss_path(path: str) -> Tuple[UUID, UUID, str]:
|
|
63
|
+
|
|
64
|
+
parsed_url = urllib.parse.urlparse(path)
|
|
65
|
+
|
|
66
|
+
workspace_id = parsed_url.netloc.split("@")[0]
|
|
67
|
+
item_id = parsed_url.path.lstrip("/").split("/")[0]
|
|
68
|
+
delta_table_name = parsed_url.path.split("/")[-1]
|
|
69
|
+
|
|
70
|
+
return workspace_id, item_id, delta_table_name
|
|
71
|
+
|
|
72
|
+
|
|
55
73
|
def format_dax_object_name(table: str, column: str) -> str:
|
|
56
74
|
"""
|
|
57
75
|
Formats a table/column combination to the 'Table Name'[Column Name] format.
|
|
@@ -102,7 +120,7 @@ def create_relationship_name(
|
|
|
102
120
|
)
|
|
103
121
|
|
|
104
122
|
|
|
105
|
-
def resolve_report_id(report: str, workspace: Optional[str] = None) -> UUID:
|
|
123
|
+
def resolve_report_id(report: str, workspace: Optional[str | UUID] = None) -> UUID:
|
|
106
124
|
"""
|
|
107
125
|
Obtains the ID of the Power BI report.
|
|
108
126
|
|
|
@@ -110,8 +128,8 @@ def resolve_report_id(report: str, workspace: Optional[str] = None) -> UUID:
|
|
|
110
128
|
----------
|
|
111
129
|
report : str
|
|
112
130
|
The name of the Power BI report.
|
|
113
|
-
workspace : str, default=None
|
|
114
|
-
The Fabric workspace name.
|
|
131
|
+
workspace : str | uuid.UUID, default=None
|
|
132
|
+
The Fabric workspace name or ID.
|
|
115
133
|
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
116
134
|
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
117
135
|
|
|
@@ -121,25 +139,19 @@ def resolve_report_id(report: str, workspace: Optional[str] = None) -> UUID:
|
|
|
121
139
|
The ID of the Power BI report.
|
|
122
140
|
"""
|
|
123
141
|
|
|
124
|
-
|
|
125
|
-
workspace_id = fabric.get_workspace_id()
|
|
126
|
-
workspace = fabric.resolve_workspace_name(workspace_id)
|
|
127
|
-
|
|
128
|
-
obj = fabric.resolve_item_id(item_name=report, type="Report", workspace=workspace)
|
|
142
|
+
return fabric.resolve_item_id(item_name=report, type="Report", workspace=workspace)
|
|
129
143
|
|
|
130
|
-
return obj
|
|
131
144
|
|
|
132
|
-
|
|
133
|
-
def resolve_report_name(report_id: UUID, workspace: Optional[str] = None) -> str:
|
|
145
|
+
def resolve_report_name(report_id: UUID, workspace: Optional[str | UUID] = None) -> str:
|
|
134
146
|
"""
|
|
135
147
|
Obtains the name of the Power BI report.
|
|
136
148
|
|
|
137
149
|
Parameters
|
|
138
150
|
----------
|
|
139
|
-
report_id : UUID
|
|
151
|
+
report_id : uuid.UUID
|
|
140
152
|
The name of the Power BI report.
|
|
141
|
-
workspace : str, default=None
|
|
142
|
-
The Fabric workspace name.
|
|
153
|
+
workspace : str | uuid.UUID, default=None
|
|
154
|
+
The Fabric workspace name or ID.
|
|
143
155
|
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
144
156
|
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
145
157
|
|
|
@@ -149,46 +161,83 @@ def resolve_report_name(report_id: UUID, workspace: Optional[str] = None) -> str
|
|
|
149
161
|
The name of the Power BI report.
|
|
150
162
|
"""
|
|
151
163
|
|
|
152
|
-
|
|
153
|
-
workspace_id = fabric.get_workspace_id()
|
|
154
|
-
workspace = fabric.resolve_workspace_name(workspace_id)
|
|
155
|
-
|
|
156
|
-
obj = fabric.resolve_item_name(
|
|
164
|
+
return fabric.resolve_item_name(
|
|
157
165
|
item_id=report_id, type="Report", workspace=workspace
|
|
158
166
|
)
|
|
159
167
|
|
|
160
|
-
return obj
|
|
161
168
|
|
|
169
|
+
def resolve_item_name_and_id(
|
|
170
|
+
item: str | UUID, type: Optional[str] = None, workspace: Optional[str | UUID] = None
|
|
171
|
+
) -> Tuple[str, UUID]:
|
|
162
172
|
|
|
163
|
-
|
|
164
|
-
|
|
173
|
+
(workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
|
|
174
|
+
|
|
175
|
+
if _is_valid_uuid(item):
|
|
176
|
+
item_id = item
|
|
177
|
+
item_name = fabric.resolve_item_name(
|
|
178
|
+
item_id=item_id, type=type, workspace=workspace_id
|
|
179
|
+
)
|
|
180
|
+
else:
|
|
181
|
+
if type is None:
|
|
182
|
+
raise ValueError(
|
|
183
|
+
f"{icons.warning} Must specify a 'type' if specifying a name as the 'item'."
|
|
184
|
+
)
|
|
185
|
+
item_name = item
|
|
186
|
+
item_id = fabric.resolve_item_id(
|
|
187
|
+
item_name=item, type=type, workspace=workspace_id
|
|
188
|
+
)
|
|
189
|
+
|
|
190
|
+
return item_name, item_id
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def resolve_lakehouse_name_and_id(
|
|
194
|
+
lakehouse: Optional[str | UUID] = None, workspace: Optional[str | UUID] = None
|
|
165
195
|
) -> Tuple[str, UUID]:
|
|
166
196
|
|
|
167
197
|
(workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
|
|
198
|
+
type = "Lakehouse"
|
|
168
199
|
|
|
169
|
-
if
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
item_id=
|
|
200
|
+
if lakehouse is None:
|
|
201
|
+
lakehouse_id = fabric.get_lakehouse_id()
|
|
202
|
+
lakehouse_name = fabric.resolve_item_name(
|
|
203
|
+
item_id=lakehouse_id, type=type, workspace=workspace_id
|
|
204
|
+
)
|
|
205
|
+
elif _is_valid_uuid(lakehouse):
|
|
206
|
+
lakehouse_id = lakehouse
|
|
207
|
+
lakehouse_name = fabric.resolve_item_name(
|
|
208
|
+
item_id=lakehouse_id, type=type, workspace=workspace_id
|
|
173
209
|
)
|
|
174
210
|
else:
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
item_name=
|
|
211
|
+
lakehouse_name = lakehouse
|
|
212
|
+
lakehouse_id = fabric.resolve_item_id(
|
|
213
|
+
item_name=lakehouse, type=type, workspace=workspace_id
|
|
178
214
|
)
|
|
179
215
|
|
|
216
|
+
return lakehouse_name, lakehouse_id
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def resolve_dataset_name_and_id(
|
|
220
|
+
dataset: str | UUID, workspace: Optional[str | UUID] = None
|
|
221
|
+
) -> Tuple[str, UUID]:
|
|
222
|
+
|
|
223
|
+
(dataset_name, dataset_id) = resolve_item_name_and_id(
|
|
224
|
+
item=dataset, type="SemanticModel", workspace=workspace
|
|
225
|
+
)
|
|
226
|
+
|
|
180
227
|
return dataset_name, dataset_id
|
|
181
228
|
|
|
182
229
|
|
|
183
|
-
def resolve_dataset_id(
|
|
230
|
+
def resolve_dataset_id(
|
|
231
|
+
dataset: str | UUID, workspace: Optional[str | UUID] = None
|
|
232
|
+
) -> UUID:
|
|
184
233
|
"""
|
|
185
234
|
Obtains the ID of the semantic model.
|
|
186
235
|
|
|
187
236
|
Parameters
|
|
188
237
|
----------
|
|
189
|
-
dataset : str | UUID
|
|
238
|
+
dataset : str | uuid.UUID
|
|
190
239
|
The name or ID of the semantic model.
|
|
191
|
-
workspace : str, default=None
|
|
240
|
+
workspace : str | uuid.UUID, default=None
|
|
192
241
|
The Fabric workspace name.
|
|
193
242
|
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
194
243
|
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
@@ -209,15 +258,17 @@ def resolve_dataset_id(dataset: str | UUID, workspace: Optional[str] = None) ->
|
|
|
209
258
|
return dataset_id
|
|
210
259
|
|
|
211
260
|
|
|
212
|
-
def resolve_dataset_name(
|
|
261
|
+
def resolve_dataset_name(
|
|
262
|
+
dataset_id: UUID, workspace: Optional[str | UUID] = None
|
|
263
|
+
) -> str:
|
|
213
264
|
"""
|
|
214
265
|
Obtains the name of the semantic model.
|
|
215
266
|
|
|
216
267
|
Parameters
|
|
217
268
|
----------
|
|
218
|
-
dataset_id : UUID
|
|
269
|
+
dataset_id : uuid.UUID
|
|
219
270
|
The name of the semantic model.
|
|
220
|
-
workspace : str, default=None
|
|
271
|
+
workspace : str | uuid.UUID, default=None
|
|
221
272
|
The Fabric workspace name.
|
|
222
273
|
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
223
274
|
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
@@ -228,30 +279,24 @@ def resolve_dataset_name(dataset_id: UUID, workspace: Optional[str] = None) -> s
|
|
|
228
279
|
The name of the semantic model.
|
|
229
280
|
"""
|
|
230
281
|
|
|
231
|
-
|
|
232
|
-
workspace_id = fabric.get_workspace_id()
|
|
233
|
-
workspace = fabric.resolve_workspace_name(workspace_id)
|
|
234
|
-
|
|
235
|
-
obj = fabric.resolve_item_name(
|
|
282
|
+
return fabric.resolve_item_name(
|
|
236
283
|
item_id=dataset_id, type="SemanticModel", workspace=workspace
|
|
237
284
|
)
|
|
238
285
|
|
|
239
|
-
return obj
|
|
240
|
-
|
|
241
286
|
|
|
242
287
|
def resolve_lakehouse_name(
|
|
243
|
-
lakehouse_id: Optional[UUID] = None, workspace: Optional[str] = None
|
|
288
|
+
lakehouse_id: Optional[UUID] = None, workspace: Optional[str | UUID] = None
|
|
244
289
|
) -> str:
|
|
245
290
|
"""
|
|
246
291
|
Obtains the name of the Fabric lakehouse.
|
|
247
292
|
|
|
248
293
|
Parameters
|
|
249
294
|
----------
|
|
250
|
-
lakehouse_id : UUID, default=None
|
|
295
|
+
lakehouse_id : uuid.UUID, default=None
|
|
251
296
|
The name of the Fabric lakehouse.
|
|
252
297
|
Defaults to None which resolves to the lakehouse attached to the notebook.
|
|
253
|
-
workspace : str, default=None
|
|
254
|
-
The Fabric workspace name.
|
|
298
|
+
workspace : str | uuid.UUID, default=None
|
|
299
|
+
The Fabric workspace name or ID.
|
|
255
300
|
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
256
301
|
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
257
302
|
|
|
@@ -261,60 +306,59 @@ def resolve_lakehouse_name(
|
|
|
261
306
|
The name of the Fabric lakehouse.
|
|
262
307
|
"""
|
|
263
308
|
|
|
264
|
-
if workspace is None:
|
|
265
|
-
workspace_id = fabric.get_workspace_id()
|
|
266
|
-
workspace = fabric.resolve_workspace_name(workspace_id)
|
|
267
|
-
|
|
268
309
|
if lakehouse_id is None:
|
|
269
310
|
lakehouse_id = fabric.get_lakehouse_id()
|
|
270
311
|
|
|
271
|
-
|
|
312
|
+
return fabric.resolve_item_name(
|
|
272
313
|
item_id=lakehouse_id, type="Lakehouse", workspace=workspace
|
|
273
314
|
)
|
|
274
315
|
|
|
275
|
-
return obj
|
|
276
|
-
|
|
277
316
|
|
|
278
|
-
def resolve_lakehouse_id(
|
|
317
|
+
def resolve_lakehouse_id(
|
|
318
|
+
lakehouse: Optional[str | UUID] = None, workspace: Optional[str | UUID] = None
|
|
319
|
+
) -> UUID:
|
|
279
320
|
"""
|
|
280
321
|
Obtains the ID of the Fabric lakehouse.
|
|
281
322
|
|
|
282
323
|
Parameters
|
|
283
324
|
----------
|
|
284
|
-
lakehouse : str
|
|
285
|
-
The name of the Fabric lakehouse.
|
|
286
|
-
workspace : str, default=None
|
|
287
|
-
The Fabric workspace name.
|
|
325
|
+
lakehouse : str | uuid.UUID, default=None
|
|
326
|
+
The name or ID of the Fabric lakehouse.
|
|
327
|
+
workspace : str | uuid.UUID, default=None
|
|
328
|
+
The Fabric workspace name or ID.
|
|
288
329
|
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
289
330
|
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
290
331
|
|
|
291
332
|
Returns
|
|
292
333
|
-------
|
|
293
|
-
UUID
|
|
334
|
+
uuid.UUID
|
|
294
335
|
The ID of the Fabric lakehouse.
|
|
295
336
|
"""
|
|
296
337
|
|
|
297
|
-
if
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
338
|
+
if lakehouse is None:
|
|
339
|
+
lakehouse_id = fabric.get_lakehouse_id()
|
|
340
|
+
elif _is_valid_uuid(lakehouse):
|
|
341
|
+
lakehouse_id = lakehouse
|
|
342
|
+
else:
|
|
343
|
+
lakehouse_id = fabric.resolve_item_id(
|
|
344
|
+
item_name=lakehouse, type="Lakehouse", workspace=workspace
|
|
345
|
+
)
|
|
304
346
|
|
|
305
|
-
return
|
|
347
|
+
return lakehouse_id
|
|
306
348
|
|
|
307
349
|
|
|
308
|
-
def get_direct_lake_sql_endpoint(
|
|
350
|
+
def get_direct_lake_sql_endpoint(
|
|
351
|
+
dataset: str | UUID, workspace: Optional[str | UUID] = None
|
|
352
|
+
) -> UUID:
|
|
309
353
|
"""
|
|
310
354
|
Obtains the SQL Endpoint ID of the semantic model.
|
|
311
355
|
|
|
312
356
|
Parameters
|
|
313
357
|
----------
|
|
314
|
-
dataset : str
|
|
315
|
-
The name of the semantic model.
|
|
316
|
-
workspace : str, default=None
|
|
317
|
-
The Fabric workspace name.
|
|
358
|
+
dataset : str | uuid.UUID
|
|
359
|
+
The name or ID of the semantic model.
|
|
360
|
+
workspace : str | uuid.UUID, default=None
|
|
361
|
+
The Fabric workspace name or ID.
|
|
318
362
|
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
319
363
|
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
320
364
|
|
|
@@ -326,10 +370,6 @@ def get_direct_lake_sql_endpoint(dataset: str, workspace: Optional[str] = None)
|
|
|
326
370
|
|
|
327
371
|
from sempy_labs.tom import connect_semantic_model
|
|
328
372
|
|
|
329
|
-
if workspace is None:
|
|
330
|
-
workspace_id = fabric.get_workspace_id()
|
|
331
|
-
workspace = fabric.resolve_workspace_name(workspace_id)
|
|
332
|
-
|
|
333
373
|
# dfP = fabric.list_partitions(dataset=dataset, workspace=workspace)
|
|
334
374
|
# dfP_filt = dfP[dfP["Mode"] == "DirectLake"]
|
|
335
375
|
|
|
@@ -349,7 +389,7 @@ def get_direct_lake_sql_endpoint(dataset: str, workspace: Optional[str] = None)
|
|
|
349
389
|
sqlEndpointId = matches[1]
|
|
350
390
|
|
|
351
391
|
if sqlEndpointId is None:
|
|
352
|
-
raise ValueError("SQL Endpoint not found.")
|
|
392
|
+
raise ValueError(f"{icons.red_dot} SQL Endpoint not found.")
|
|
353
393
|
|
|
354
394
|
return sqlEndpointId
|
|
355
395
|
|
|
@@ -425,8 +465,8 @@ def save_as_delta_table(
|
|
|
425
465
|
write_mode: str,
|
|
426
466
|
merge_schema: bool = False,
|
|
427
467
|
schema: Optional[dict] = None,
|
|
428
|
-
lakehouse: Optional[str] = None,
|
|
429
|
-
workspace: Optional[str] = None,
|
|
468
|
+
lakehouse: Optional[str | UUID] = None,
|
|
469
|
+
workspace: Optional[str | UUID] = None,
|
|
430
470
|
):
|
|
431
471
|
"""
|
|
432
472
|
Saves a pandas dataframe as a delta table in a Fabric lakehouse.
|
|
@@ -443,11 +483,11 @@ def save_as_delta_table(
|
|
|
443
483
|
Merges the schemas of the dataframe to the delta table.
|
|
444
484
|
schema : dict, default=None
|
|
445
485
|
A dictionary showing the schema of the columns and their data types.
|
|
446
|
-
lakehouse : str, default=None
|
|
447
|
-
The Fabric lakehouse
|
|
486
|
+
lakehouse : str | uuid.UUID, default=None
|
|
487
|
+
The Fabric lakehouse name or ID.
|
|
448
488
|
Defaults to None which resolves to the lakehouse attached to the notebook.
|
|
449
|
-
workspace : str, default=None
|
|
450
|
-
The Fabric workspace name.
|
|
489
|
+
workspace : str | uuid.UUID, default=None
|
|
490
|
+
The Fabric workspace name or ID.
|
|
451
491
|
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
452
492
|
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
453
493
|
"""
|
|
@@ -466,26 +506,17 @@ def save_as_delta_table(
|
|
|
466
506
|
TimestampType,
|
|
467
507
|
)
|
|
468
508
|
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
workspace
|
|
472
|
-
|
|
473
|
-
workspace_id = fabric.resolve_workspace_id(workspace)
|
|
474
|
-
|
|
475
|
-
if lakehouse is None:
|
|
476
|
-
lakehouse_id = fabric.get_lakehouse_id()
|
|
477
|
-
lakehouse = resolve_lakehouse_name(
|
|
478
|
-
lakehouse_id=lakehouse_id, workspace=workspace
|
|
479
|
-
)
|
|
480
|
-
else:
|
|
481
|
-
lakehouse_id = resolve_lakehouse_id(lakehouse, workspace)
|
|
509
|
+
(workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
|
|
510
|
+
(lakehouse_name, lakehouse_id) = resolve_lakehouse_name_and_id(
|
|
511
|
+
lakehouse=lakehouse, workspace=workspace_id
|
|
512
|
+
)
|
|
482
513
|
|
|
483
|
-
|
|
514
|
+
write_modes = ["append", "overwrite"]
|
|
484
515
|
write_mode = write_mode.lower()
|
|
485
516
|
|
|
486
|
-
if write_mode not in
|
|
517
|
+
if write_mode not in write_modes:
|
|
487
518
|
raise ValueError(
|
|
488
|
-
f"{icons.red_dot} Invalid 'write_type' parameter. Choose from one of the following values: {
|
|
519
|
+
f"{icons.red_dot} Invalid 'write_type' parameter. Choose from one of the following values: {write_modes}."
|
|
489
520
|
)
|
|
490
521
|
|
|
491
522
|
if " " in delta_table_name:
|
|
@@ -510,16 +541,19 @@ def save_as_delta_table(
|
|
|
510
541
|
"timestamp": TimestampType(),
|
|
511
542
|
}
|
|
512
543
|
|
|
513
|
-
if
|
|
514
|
-
|
|
544
|
+
if isinstance(dataframe, pd.DataFrame):
|
|
545
|
+
if schema is None:
|
|
546
|
+
spark_df = spark.createDataFrame(dataframe)
|
|
547
|
+
else:
|
|
548
|
+
schema_map = StructType(
|
|
549
|
+
[
|
|
550
|
+
StructField(column_name, type_mapping[data_type], True)
|
|
551
|
+
for column_name, data_type in schema.items()
|
|
552
|
+
]
|
|
553
|
+
)
|
|
554
|
+
spark_df = spark.createDataFrame(dataframe, schema_map)
|
|
515
555
|
else:
|
|
516
|
-
|
|
517
|
-
[
|
|
518
|
-
StructField(column_name, type_mapping[data_type], True)
|
|
519
|
-
for column_name, data_type in schema.items()
|
|
520
|
-
]
|
|
521
|
-
)
|
|
522
|
-
spark_df = spark.createDataFrame(dataframe, schema_map)
|
|
556
|
+
spark_df = dataframe
|
|
523
557
|
|
|
524
558
|
filePath = create_abfss_path(
|
|
525
559
|
lakehouse_id=lakehouse_id,
|
|
@@ -534,7 +568,7 @@ def save_as_delta_table(
|
|
|
534
568
|
else:
|
|
535
569
|
spark_df.write.mode(write_mode).format("delta").save(filePath)
|
|
536
570
|
print(
|
|
537
|
-
f"{icons.green_dot} The dataframe has been saved as the '{delta_table_name}' table in the '{
|
|
571
|
+
f"{icons.green_dot} The dataframe has been saved as the '{delta_table_name}' table in the '{lakehouse_name}' lakehouse within the '{workspace_name}' workspace."
|
|
538
572
|
)
|
|
539
573
|
|
|
540
574
|
|
|
@@ -574,14 +608,16 @@ def language_validate(language: str):
|
|
|
574
608
|
return lang
|
|
575
609
|
|
|
576
610
|
|
|
577
|
-
def resolve_workspace_name_and_id(
|
|
611
|
+
def resolve_workspace_name_and_id(
|
|
612
|
+
workspace: Optional[str | UUID] = None,
|
|
613
|
+
) -> Tuple[str, str]:
|
|
578
614
|
"""
|
|
579
615
|
Obtains the name and ID of the Fabric workspace.
|
|
580
616
|
|
|
581
617
|
Parameters
|
|
582
618
|
----------
|
|
583
|
-
workspace : str, default=None
|
|
584
|
-
The Fabric workspace name.
|
|
619
|
+
workspace : str | uuid.UUID, default=None
|
|
620
|
+
The Fabric workspace name or ID.
|
|
585
621
|
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
586
622
|
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
587
623
|
|
|
@@ -593,11 +629,15 @@ def resolve_workspace_name_and_id(workspace: Optional[str] = None) -> Tuple[str,
|
|
|
593
629
|
|
|
594
630
|
if workspace is None:
|
|
595
631
|
workspace_id = fabric.get_workspace_id()
|
|
596
|
-
|
|
632
|
+
workspace_name = fabric.resolve_workspace_name(workspace_id)
|
|
633
|
+
elif _is_valid_uuid(workspace):
|
|
634
|
+
workspace_id = workspace
|
|
635
|
+
workspace_name = fabric.resolve_workspace_name(workspace_id)
|
|
597
636
|
else:
|
|
598
|
-
|
|
637
|
+
workspace_name = workspace
|
|
638
|
+
workspace_id = fabric.resolve_workspace_id(workspace_name)
|
|
599
639
|
|
|
600
|
-
return str(
|
|
640
|
+
return str(workspace_name), str(workspace_id)
|
|
601
641
|
|
|
602
642
|
|
|
603
643
|
def _extract_json(dataframe: pd.DataFrame) -> dict:
|
|
@@ -618,12 +658,12 @@ def _conv_b64(file):
|
|
|
618
658
|
|
|
619
659
|
def _decode_b64(file, format: Optional[str] = "utf-8"):
|
|
620
660
|
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
return result
|
|
661
|
+
return base64.b64decode(file).decode(format)
|
|
624
662
|
|
|
625
663
|
|
|
626
|
-
def is_default_semantic_model(
|
|
664
|
+
def is_default_semantic_model(
|
|
665
|
+
dataset: str, workspace: Optional[str | UUID] = None
|
|
666
|
+
) -> bool:
|
|
627
667
|
"""
|
|
628
668
|
Identifies whether a semantic model is a default semantic model.
|
|
629
669
|
|
|
@@ -631,8 +671,8 @@ def is_default_semantic_model(dataset: str, workspace: Optional[str] = None) ->
|
|
|
631
671
|
----------
|
|
632
672
|
dataset : str
|
|
633
673
|
The name of the semantic model.
|
|
634
|
-
workspace : str, default=None
|
|
635
|
-
The Fabric workspace name.
|
|
674
|
+
workspace : str | uuid.UUID, default=None
|
|
675
|
+
The Fabric workspace name or ID.
|
|
636
676
|
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
637
677
|
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
638
678
|
|
|
@@ -642,9 +682,9 @@ def is_default_semantic_model(dataset: str, workspace: Optional[str] = None) ->
|
|
|
642
682
|
A True/False value indicating whether the semantic model is a default semantic model.
|
|
643
683
|
"""
|
|
644
684
|
|
|
645
|
-
|
|
685
|
+
(workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
|
|
646
686
|
|
|
647
|
-
dfI = fabric.list_items(workspace=
|
|
687
|
+
dfI = fabric.list_items(workspace=workspace_id)
|
|
648
688
|
filtered_df = dfI.groupby("Display Name").filter(
|
|
649
689
|
lambda x: set(["Warehouse", "SemanticModel"]).issubset(set(x["Type"]))
|
|
650
690
|
or set(["Lakehouse", "SemanticModel"]).issubset(set(x["Type"]))
|
|
@@ -654,16 +694,16 @@ def is_default_semantic_model(dataset: str, workspace: Optional[str] = None) ->
|
|
|
654
694
|
return dataset in default_semantic_models
|
|
655
695
|
|
|
656
696
|
|
|
657
|
-
def resolve_item_type(item_id: UUID, workspace: Optional[str] = None) -> str:
|
|
697
|
+
def resolve_item_type(item_id: UUID, workspace: Optional[str | UUID] = None) -> str:
|
|
658
698
|
"""
|
|
659
699
|
Obtains the item type for a given Fabric Item Id within a Fabric workspace.
|
|
660
700
|
|
|
661
701
|
Parameters
|
|
662
702
|
----------
|
|
663
|
-
item_id : UUID
|
|
703
|
+
item_id : uuid.UUID
|
|
664
704
|
The item/artifact Id.
|
|
665
|
-
workspace : str, default=None
|
|
666
|
-
The Fabric workspace name.
|
|
705
|
+
workspace : str | uuid.UUID, default=None
|
|
706
|
+
The Fabric workspace name or ID.
|
|
667
707
|
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
668
708
|
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
669
709
|
|
|
@@ -673,50 +713,43 @@ def resolve_item_type(item_id: UUID, workspace: Optional[str] = None) -> str:
|
|
|
673
713
|
The item type for the item Id.
|
|
674
714
|
"""
|
|
675
715
|
|
|
676
|
-
|
|
677
|
-
dfI = fabric.list_items(workspace=
|
|
716
|
+
(workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
|
|
717
|
+
dfI = fabric.list_items(workspace=workspace_id)
|
|
678
718
|
dfI_filt = dfI[dfI["Id"] == item_id]
|
|
679
719
|
|
|
680
|
-
if
|
|
720
|
+
if dfI_filt.empty:
|
|
681
721
|
raise ValueError(
|
|
682
|
-
f"Invalid 'item_id' parameter. The '{item_id}' item was not found in the '{
|
|
722
|
+
f"Invalid 'item_id' parameter. The '{item_id}' item was not found in the '{workspace_name}' workspace."
|
|
683
723
|
)
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
return item_type
|
|
724
|
+
return dfI_filt["Type"].iloc[0]
|
|
687
725
|
|
|
688
726
|
|
|
689
727
|
def resolve_dataset_from_report(
|
|
690
|
-
report: str, workspace: Optional[str] = None
|
|
728
|
+
report: str | UUID, workspace: Optional[str | UUID] = None
|
|
691
729
|
) -> Tuple[UUID, str, UUID, str]:
|
|
692
730
|
"""
|
|
693
731
|
Obtains the basic semantic model properties from which the report's data is sourced.
|
|
694
732
|
|
|
695
733
|
Parameters
|
|
696
734
|
----------
|
|
697
|
-
report : str
|
|
698
|
-
The name of the Power BI report.
|
|
699
|
-
workspace : str, default=None
|
|
700
|
-
The Fabric workspace name.
|
|
735
|
+
report : str | uuid.UUID
|
|
736
|
+
The name or ID of the Power BI report.
|
|
737
|
+
workspace : str | uuid.UUID, default=None
|
|
738
|
+
The Fabric workspace name or ID.
|
|
701
739
|
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
702
740
|
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
703
741
|
|
|
704
742
|
Returns
|
|
705
743
|
-------
|
|
706
|
-
Tuple[UUID, str, UUID, str]
|
|
744
|
+
Tuple[uuid.UUID, str, uuid.UUID, str]
|
|
707
745
|
The semantic model UUID, semantic model name, semantic model workspace UUID, semantic model workspace name
|
|
708
746
|
"""
|
|
709
747
|
|
|
710
|
-
|
|
748
|
+
from sempy_labs.report._generate_report import _get_report
|
|
711
749
|
|
|
712
|
-
dfR =
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
raise ValueError(
|
|
716
|
-
f"{icons.red_dot} The '{report}' report does not exist within the '{workspace}' workspace."
|
|
717
|
-
)
|
|
718
|
-
dataset_id = dfR_filt["Dataset Id"].iloc[0]
|
|
719
|
-
dataset_workspace_id = dfR_filt["Dataset Workspace Id"].iloc[0]
|
|
750
|
+
dfR = _get_report(report=report, workspace=workspace)
|
|
751
|
+
dataset_id = dfR["Dataset Id"].iloc[0]
|
|
752
|
+
dataset_workspace_id = dfR["Dataset Workspace Id"].iloc[0]
|
|
720
753
|
dataset_workspace = fabric.resolve_workspace_name(dataset_workspace_id)
|
|
721
754
|
dataset_name = resolve_dataset_name(
|
|
722
755
|
dataset_id=dataset_id, workspace=dataset_workspace
|
|
@@ -732,14 +765,16 @@ def _add_part(target_dict, path, payload):
|
|
|
732
765
|
target_dict["definition"]["parts"].append(part)
|
|
733
766
|
|
|
734
767
|
|
|
735
|
-
def resolve_workspace_capacity(
|
|
768
|
+
def resolve_workspace_capacity(
|
|
769
|
+
workspace: Optional[str | UUID] = None,
|
|
770
|
+
) -> Tuple[UUID, str]:
|
|
736
771
|
"""
|
|
737
772
|
Obtains the capacity Id and capacity name for a given workspace.
|
|
738
773
|
|
|
739
774
|
Parameters
|
|
740
775
|
----------
|
|
741
|
-
workspace : str, default=None
|
|
742
|
-
The Fabric workspace name.
|
|
776
|
+
workspace : str | uuid.UUID, default=None
|
|
777
|
+
The Fabric workspace name or UUID.
|
|
743
778
|
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
744
779
|
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
745
780
|
|
|
@@ -749,9 +784,9 @@ def resolve_workspace_capacity(workspace: Optional[str] = None) -> Tuple[UUID, s
|
|
|
749
784
|
capacity Id; capacity came.
|
|
750
785
|
"""
|
|
751
786
|
|
|
752
|
-
|
|
753
|
-
filter_condition = urllib.parse.quote(
|
|
754
|
-
dfW = fabric.list_workspaces(filter=f"
|
|
787
|
+
(workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
|
|
788
|
+
filter_condition = urllib.parse.quote(workspace_id)
|
|
789
|
+
dfW = fabric.list_workspaces(filter=f"id eq '{filter_condition}'")
|
|
755
790
|
capacity_id = dfW["Capacity Id"].iloc[0]
|
|
756
791
|
dfC = fabric.list_capacities()
|
|
757
792
|
dfC_filt = dfC[dfC["Id"] == capacity_id]
|
|
@@ -763,14 +798,14 @@ def resolve_workspace_capacity(workspace: Optional[str] = None) -> Tuple[UUID, s
|
|
|
763
798
|
return capacity_id, capacity_name
|
|
764
799
|
|
|
765
800
|
|
|
766
|
-
def get_capacity_id(workspace: Optional[str] = None) -> UUID:
|
|
801
|
+
def get_capacity_id(workspace: Optional[str | UUID] = None) -> UUID:
|
|
767
802
|
"""
|
|
768
803
|
Obtains the Capacity Id for a given workspace.
|
|
769
804
|
|
|
770
805
|
Parameters
|
|
771
806
|
----------
|
|
772
|
-
workspace : str, default=None
|
|
773
|
-
The Fabric workspace name.
|
|
807
|
+
workspace : str | uuid.UUID, default=None
|
|
808
|
+
The Fabric workspace name or ID.
|
|
774
809
|
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
775
810
|
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
776
811
|
|
|
@@ -781,28 +816,27 @@ def get_capacity_id(workspace: Optional[str] = None) -> UUID:
|
|
|
781
816
|
"""
|
|
782
817
|
|
|
783
818
|
if workspace is None:
|
|
784
|
-
capacity_id =
|
|
819
|
+
capacity_id = _get_fabric_context_setting(name="trident.capacity.id")
|
|
785
820
|
else:
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
dfW = fabric.list_workspaces(filter=f"name eq '{filter_condition}'")
|
|
821
|
+
(workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
|
|
822
|
+
filter_condition = urllib.parse.quote(workspace_id)
|
|
823
|
+
dfW = fabric.list_workspaces(filter=f"id eq '{filter_condition}'")
|
|
790
824
|
if len(dfW) == 0:
|
|
791
|
-
raise ValueError(f"{icons.red_dot} The '{
|
|
825
|
+
raise ValueError(f"{icons.red_dot} The '{workspace_name}' does not exist'.")
|
|
792
826
|
|
|
793
827
|
capacity_id = dfW["Capacity Id"].iloc[0]
|
|
794
828
|
|
|
795
829
|
return capacity_id
|
|
796
830
|
|
|
797
831
|
|
|
798
|
-
def get_capacity_name(workspace: Optional[str] = None) -> str:
|
|
832
|
+
def get_capacity_name(workspace: Optional[str | UUID] = None) -> str:
|
|
799
833
|
"""
|
|
800
834
|
Obtains the capacity name for a given workspace.
|
|
801
835
|
|
|
802
836
|
Parameters
|
|
803
837
|
----------
|
|
804
|
-
workspace : str, default=None
|
|
805
|
-
The Fabric workspace name.
|
|
838
|
+
workspace : str | uuid.UUID, default=None
|
|
839
|
+
The Fabric workspace name or ID.
|
|
806
840
|
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
807
841
|
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
808
842
|
|
|
@@ -829,7 +863,7 @@ def resolve_capacity_name(capacity_id: Optional[UUID] = None) -> str:
|
|
|
829
863
|
|
|
830
864
|
Parameters
|
|
831
865
|
----------
|
|
832
|
-
capacity_id : UUID, default=None
|
|
866
|
+
capacity_id : uuid.UUID, default=None
|
|
833
867
|
The capacity Id.
|
|
834
868
|
Defaults to None which resolves to the capacity name of the workspace of the attached lakehouse
|
|
835
869
|
or if no lakehouse attached, resolves to the capacity name of the workspace of the notebook.
|
|
@@ -971,7 +1005,7 @@ def resolve_deployment_pipeline_id(deployment_pipeline: str) -> UUID:
|
|
|
971
1005
|
|
|
972
1006
|
Returns
|
|
973
1007
|
-------
|
|
974
|
-
UUID
|
|
1008
|
+
uuid.UUID
|
|
975
1009
|
The deployment pipeline Id.
|
|
976
1010
|
"""
|
|
977
1011
|
|
|
@@ -1020,25 +1054,33 @@ def _get_adls_client(account_name):
|
|
|
1020
1054
|
return service_client
|
|
1021
1055
|
|
|
1022
1056
|
|
|
1023
|
-
def resolve_warehouse_id(
|
|
1057
|
+
def resolve_warehouse_id(
|
|
1058
|
+
warehouse: str | UUID, workspace: Optional[str | UUID]
|
|
1059
|
+
) -> UUID:
|
|
1024
1060
|
"""
|
|
1025
1061
|
Obtains the Id for a given warehouse.
|
|
1026
1062
|
|
|
1027
1063
|
Parameters
|
|
1028
1064
|
----------
|
|
1029
|
-
warehouse : str
|
|
1030
|
-
The warehouse name
|
|
1065
|
+
warehouse : str | uuid.UUID
|
|
1066
|
+
The warehouse name or ID.
|
|
1067
|
+
workspace : str | uuid.UUID, default=None
|
|
1068
|
+
The Fabric workspace name or ID in which the semantic model resides.
|
|
1069
|
+
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
1070
|
+
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
1031
1071
|
|
|
1032
1072
|
Returns
|
|
1033
1073
|
-------
|
|
1034
|
-
UUID
|
|
1074
|
+
uuid.UUID
|
|
1035
1075
|
The warehouse Id.
|
|
1036
1076
|
"""
|
|
1037
1077
|
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1078
|
+
if _is_valid_uuid(warehouse):
|
|
1079
|
+
return warehouse
|
|
1080
|
+
else:
|
|
1081
|
+
return fabric.resolve_item_id(
|
|
1082
|
+
item_name=warehouse, type="Warehouse", workspace=workspace
|
|
1083
|
+
)
|
|
1042
1084
|
|
|
1043
1085
|
|
|
1044
1086
|
def get_language_codes(languages: str | List[str]):
|
|
@@ -1097,7 +1139,9 @@ def convert_to_alphanumeric_lowercase(input_string):
|
|
|
1097
1139
|
return cleaned_string
|
|
1098
1140
|
|
|
1099
1141
|
|
|
1100
|
-
def resolve_environment_id(
|
|
1142
|
+
def resolve_environment_id(
|
|
1143
|
+
environment: str, workspace: Optional[str | UUID] = None
|
|
1144
|
+
) -> UUID:
|
|
1101
1145
|
"""
|
|
1102
1146
|
Obtains the environment Id for a given environment.
|
|
1103
1147
|
|
|
@@ -1105,6 +1149,10 @@ def resolve_environment_id(environment: str, workspace: Optional[str] = None) ->
|
|
|
1105
1149
|
----------
|
|
1106
1150
|
environment: str
|
|
1107
1151
|
Name of the environment.
|
|
1152
|
+
workspace : str | uuid.UUID, default=None
|
|
1153
|
+
The Fabric workspace name or ID in which the semantic model resides.
|
|
1154
|
+
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
1155
|
+
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
1108
1156
|
|
|
1109
1157
|
Returns
|
|
1110
1158
|
-------
|
|
@@ -1112,7 +1160,6 @@ def resolve_environment_id(environment: str, workspace: Optional[str] = None) ->
|
|
|
1112
1160
|
The environment Id.
|
|
1113
1161
|
"""
|
|
1114
1162
|
|
|
1115
|
-
workspace = fabric.resolve_workspace_name(workspace)
|
|
1116
1163
|
return fabric.resolve_item_id(
|
|
1117
1164
|
item_name=environment, type="Environment", workspace=workspace
|
|
1118
1165
|
)
|
|
@@ -1147,7 +1194,7 @@ def convert_to_friendly_case(text: str) -> str:
|
|
|
1147
1194
|
return text
|
|
1148
1195
|
|
|
1149
1196
|
|
|
1150
|
-
def resolve_notebook_id(notebook: str, workspace: Optional[str] = None) -> UUID:
|
|
1197
|
+
def resolve_notebook_id(notebook: str, workspace: Optional[str | UUID] = None) -> UUID:
|
|
1151
1198
|
"""
|
|
1152
1199
|
Obtains the notebook Id for a given notebook.
|
|
1153
1200
|
|
|
@@ -1155,6 +1202,10 @@ def resolve_notebook_id(notebook: str, workspace: Optional[str] = None) -> UUID:
|
|
|
1155
1202
|
----------
|
|
1156
1203
|
notebook: str
|
|
1157
1204
|
Name of the notebook.
|
|
1205
|
+
workspace : str | uuid.UUID, default=None
|
|
1206
|
+
The Fabric workspace name or ID in which the semantic model resides.
|
|
1207
|
+
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
1208
|
+
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
1158
1209
|
|
|
1159
1210
|
Returns
|
|
1160
1211
|
-------
|
|
@@ -1162,7 +1213,6 @@ def resolve_notebook_id(notebook: str, workspace: Optional[str] = None) -> UUID:
|
|
|
1162
1213
|
The notebook Id.
|
|
1163
1214
|
"""
|
|
1164
1215
|
|
|
1165
|
-
workspace = fabric.resolve_workspace_name(workspace)
|
|
1166
1216
|
return fabric.resolve_item_id(
|
|
1167
1217
|
item_name=notebook, type="Notebook", workspace=workspace
|
|
1168
1218
|
)
|
|
@@ -1173,16 +1223,24 @@ def generate_guid():
|
|
|
1173
1223
|
return str(uuid.uuid4())
|
|
1174
1224
|
|
|
1175
1225
|
|
|
1176
|
-
def
|
|
1226
|
+
def _get_column_aggregate(
|
|
1227
|
+
lakehouse: str,
|
|
1228
|
+
table_name: str,
|
|
1229
|
+
column_name: str = "RunId",
|
|
1230
|
+
function: str = "max",
|
|
1231
|
+
default_value: int = 0,
|
|
1232
|
+
) -> int:
|
|
1177
1233
|
|
|
1178
1234
|
from pyspark.sql import SparkSession
|
|
1179
1235
|
|
|
1180
1236
|
spark = SparkSession.builder.getOrCreate()
|
|
1181
|
-
|
|
1237
|
+
function = function.upper()
|
|
1238
|
+
query = f"SELECT {function}({column_name}) FROM {lakehouse}.{table_name}"
|
|
1239
|
+
if "COUNT" in function and "DISTINCT" in function:
|
|
1240
|
+
query = f"SELECT COUNT(DISTINCT({column_name})) FROM {lakehouse}.{table_name}"
|
|
1182
1241
|
dfSpark = spark.sql(query)
|
|
1183
|
-
max_run_id = dfSpark.collect()[0][0] or 0
|
|
1184
1242
|
|
|
1185
|
-
return
|
|
1243
|
+
return dfSpark.collect()[0][0] or default_value
|
|
1186
1244
|
|
|
1187
1245
|
|
|
1188
1246
|
def _make_list_unique(my_list):
|
|
@@ -1190,22 +1248,21 @@ def _make_list_unique(my_list):
|
|
|
1190
1248
|
return list(set(my_list))
|
|
1191
1249
|
|
|
1192
1250
|
|
|
1193
|
-
def _get_partition_map(
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
(dataset_name, dataset_id) = resolve_dataset_name_and_id(dataset, workspace_id)
|
|
1251
|
+
def _get_partition_map(
|
|
1252
|
+
dataset: str, workspace: Optional[str | UUID] = None
|
|
1253
|
+
) -> pd.DataFrame:
|
|
1197
1254
|
|
|
1198
1255
|
partitions = fabric.evaluate_dax(
|
|
1199
|
-
dataset=
|
|
1200
|
-
workspace=
|
|
1256
|
+
dataset=dataset,
|
|
1257
|
+
workspace=workspace,
|
|
1201
1258
|
dax_string="""
|
|
1202
1259
|
select [ID] AS [PartitionID], [TableID], [Name] AS [PartitionName] from $system.tmschema_partitions
|
|
1203
1260
|
""",
|
|
1204
1261
|
)
|
|
1205
1262
|
|
|
1206
1263
|
tables = fabric.evaluate_dax(
|
|
1207
|
-
dataset=
|
|
1208
|
-
workspace=
|
|
1264
|
+
dataset=dataset,
|
|
1265
|
+
workspace=workspace,
|
|
1209
1266
|
dax_string="""
|
|
1210
1267
|
select [ID] AS [TableID], [Name] AS [TableName] from $system.tmschema_tables
|
|
1211
1268
|
""",
|