semantic-link-labs 0.8.2__py3-none-any.whl → 0.8.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of semantic-link-labs might be problematic. Click here for more details.
- {semantic_link_labs-0.8.2.dist-info → semantic_link_labs-0.8.4.dist-info}/METADATA +37 -8
- {semantic_link_labs-0.8.2.dist-info → semantic_link_labs-0.8.4.dist-info}/RECORD +108 -104
- {semantic_link_labs-0.8.2.dist-info → semantic_link_labs-0.8.4.dist-info}/WHEEL +1 -1
- sempy_labs/__init__.py +38 -0
- sempy_labs/_bpa_translation/_model/_translations_am-ET.po +24 -5
- sempy_labs/_bpa_translation/_model/_translations_ar-AE.po +28 -4
- sempy_labs/_bpa_translation/_model/_translations_bg-BG.po +34 -4
- sempy_labs/_bpa_translation/_model/_translations_ca-ES.po +33 -4
- sempy_labs/_bpa_translation/_model/_translations_cs-CZ.po +31 -4
- sempy_labs/_bpa_translation/_model/_translations_da-DK.po +31 -4
- sempy_labs/_bpa_translation/_model/_translations_de-DE.po +34 -4
- sempy_labs/_bpa_translation/_model/_translations_el-GR.po +36 -4
- sempy_labs/_bpa_translation/_model/_translations_es-ES.po +90 -58
- sempy_labs/_bpa_translation/_model/_translations_fa-IR.po +31 -5
- sempy_labs/_bpa_translation/_model/_translations_fi-FI.po +31 -4
- sempy_labs/_bpa_translation/_model/_translations_fr-FR.po +34 -4
- sempy_labs/_bpa_translation/_model/_translations_ga-IE.po +34 -4
- sempy_labs/_bpa_translation/_model/_translations_he-IL.po +28 -4
- sempy_labs/_bpa_translation/_model/_translations_hi-IN.po +32 -4
- sempy_labs/_bpa_translation/_model/_translations_hu-HU.po +32 -4
- sempy_labs/_bpa_translation/_model/_translations_id-ID.po +32 -4
- sempy_labs/_bpa_translation/_model/_translations_is-IS.po +31 -4
- sempy_labs/_bpa_translation/_model/_translations_it-IT.po +34 -4
- sempy_labs/_bpa_translation/_model/_translations_ja-JP.po +24 -4
- sempy_labs/_bpa_translation/_model/_translations_ko-KR.po +72 -56
- sempy_labs/_bpa_translation/_model/_translations_mt-MT.po +34 -4
- sempy_labs/_bpa_translation/_model/_translations_nl-NL.po +34 -4
- sempy_labs/_bpa_translation/_model/_translations_pl-PL.po +95 -71
- sempy_labs/_bpa_translation/_model/_translations_pt-BR.po +32 -4
- sempy_labs/_bpa_translation/_model/_translations_pt-PT.po +32 -4
- sempy_labs/_bpa_translation/_model/_translations_ro-RO.po +33 -4
- sempy_labs/_bpa_translation/_model/_translations_ru-RU.po +34 -4
- sempy_labs/_bpa_translation/_model/_translations_sk-SK.po +31 -4
- sempy_labs/_bpa_translation/_model/_translations_sl-SL.po +32 -4
- sempy_labs/_bpa_translation/_model/_translations_sv-SE.po +32 -4
- sempy_labs/_bpa_translation/_model/_translations_ta-IN.po +32 -4
- sempy_labs/_bpa_translation/_model/_translations_te-IN.po +31 -4
- sempy_labs/_bpa_translation/_model/_translations_th-TH.po +31 -4
- sempy_labs/_bpa_translation/_model/_translations_tr-TR.po +32 -4
- sempy_labs/_bpa_translation/_model/_translations_uk-UA.po +100 -72
- sempy_labs/_bpa_translation/_model/_translations_zh-CN.po +23 -5
- sempy_labs/_bpa_translation/_model/_translations_zu-ZA.po +32 -4
- sempy_labs/_capacities.py +138 -25
- sempy_labs/_capacity_migration.py +161 -60
- sempy_labs/_clear_cache.py +3 -3
- sempy_labs/_data_pipelines.py +54 -0
- sempy_labs/_dataflows.py +4 -0
- sempy_labs/_deployment_pipelines.py +13 -7
- sempy_labs/_environments.py +6 -0
- sempy_labs/_eventhouses.py +6 -0
- sempy_labs/_eventstreams.py +6 -0
- sempy_labs/_external_data_shares.py +190 -0
- sempy_labs/_generate_semantic_model.py +26 -4
- sempy_labs/_git.py +15 -15
- sempy_labs/_helper_functions.py +186 -11
- sempy_labs/_icons.py +55 -22
- sempy_labs/_kql_databases.py +6 -0
- sempy_labs/_kql_querysets.py +6 -0
- sempy_labs/_list_functions.py +6 -3
- sempy_labs/_managed_private_endpoints.py +166 -0
- sempy_labs/_mirrored_warehouses.py +2 -0
- sempy_labs/_ml_experiments.py +6 -0
- sempy_labs/_ml_models.py +6 -0
- sempy_labs/_model_bpa.py +11 -6
- sempy_labs/_model_bpa_bulk.py +14 -30
- sempy_labs/_model_bpa_rules.py +8 -3
- sempy_labs/_notebooks.py +111 -15
- sempy_labs/_query_scale_out.py +8 -6
- sempy_labs/_refresh_semantic_model.py +299 -49
- sempy_labs/_spark.py +12 -5
- sempy_labs/_sql.py +2 -2
- sempy_labs/_translations.py +16 -14
- sempy_labs/_vertipaq.py +127 -116
- sempy_labs/_warehouses.py +90 -1
- sempy_labs/_workloads.py +128 -0
- sempy_labs/_workspace_identity.py +4 -4
- sempy_labs/_workspaces.py +14 -1
- sempy_labs/admin/__init__.py +2 -0
- sempy_labs/admin/_basic_functions.py +203 -58
- sempy_labs/admin/_domains.py +18 -18
- sempy_labs/directlake/__init__.py +2 -0
- sempy_labs/directlake/_directlake_schema_sync.py +2 -6
- sempy_labs/directlake/_dl_helper.py +4 -1
- sempy_labs/directlake/_generate_shared_expression.py +1 -1
- sempy_labs/directlake/_get_shared_expression.py +7 -1
- sempy_labs/directlake/_guardrails.py +3 -2
- sempy_labs/directlake/_show_unsupported_directlake_objects.py +2 -8
- sempy_labs/directlake/_update_directlake_model_lakehouse_connection.py +78 -0
- sempy_labs/directlake/_update_directlake_partition_entity.py +13 -32
- sempy_labs/lakehouse/_get_lakehouse_tables.py +6 -2
- sempy_labs/lakehouse/_shortcuts.py +4 -0
- sempy_labs/migration/_create_pqt_file.py +2 -2
- sempy_labs/migration/_migrate_calctables_to_lakehouse.py +3 -2
- sempy_labs/migration/_migrate_calctables_to_semantic_model.py +2 -0
- sempy_labs/migration/_migrate_model_objects_to_semantic_model.py +2 -8
- sempy_labs/migration/_migrate_tables_columns_to_semantic_model.py +17 -0
- sempy_labs/migration/_migration_validation.py +2 -0
- sempy_labs/migration/_refresh_calc_tables.py +1 -0
- sempy_labs/report/__init__.py +4 -1
- sempy_labs/report/_generate_report.py +16 -14
- sempy_labs/report/_paginated.py +74 -0
- sempy_labs/report/_report_bpa.py +8 -10
- sempy_labs/report/_report_functions.py +19 -19
- sempy_labs/report/_report_rebind.py +6 -1
- sempy_labs/report/_reportwrapper.py +3 -3
- sempy_labs/tom/_model.py +173 -67
- {semantic_link_labs-0.8.2.dist-info → semantic_link_labs-0.8.4.dist-info}/LICENSE +0 -0
- {semantic_link_labs-0.8.2.dist-info → semantic_link_labs-0.8.4.dist-info}/top_level.txt +0 -0
sempy_labs/_vertipaq.py
CHANGED
|
@@ -13,7 +13,7 @@ from sempy_labs._helper_functions import (
|
|
|
13
13
|
resolve_dataset_id,
|
|
14
14
|
save_as_delta_table,
|
|
15
15
|
resolve_workspace_capacity,
|
|
16
|
-
|
|
16
|
+
_get_max_run_id,
|
|
17
17
|
)
|
|
18
18
|
from sempy_labs._list_functions import list_relationships, list_tables
|
|
19
19
|
from sempy_labs.lakehouse import lakehouse_attached, get_lakehouse_tables
|
|
@@ -69,73 +69,67 @@ def vertipaq_analyzer(
|
|
|
69
69
|
|
|
70
70
|
workspace = fabric.resolve_workspace_name(workspace)
|
|
71
71
|
|
|
72
|
-
data_type_string = "string"
|
|
73
|
-
data_type_long = "long"
|
|
74
|
-
data_type_timestamp = "timestamp"
|
|
75
|
-
data_type_double = "double"
|
|
76
|
-
data_type_bool = "bool"
|
|
77
|
-
|
|
78
72
|
vertipaq_map = {
|
|
79
73
|
"Model": {
|
|
80
|
-
"Dataset Name": data_type_string,
|
|
81
|
-
"Total Size": data_type_long,
|
|
82
|
-
"Table Count": data_type_long,
|
|
83
|
-
"Column Count": data_type_long,
|
|
84
|
-
"Compatibility Level": data_type_long,
|
|
85
|
-
"Default Mode": data_type_string,
|
|
74
|
+
"Dataset Name": [icons.data_type_string, icons.no_format],
|
|
75
|
+
"Total Size": [icons.data_type_long, icons.int_format],
|
|
76
|
+
"Table Count": [icons.data_type_long, icons.int_format],
|
|
77
|
+
"Column Count": [icons.data_type_long, icons.int_format],
|
|
78
|
+
"Compatibility Level": [icons.data_type_long, icons.no_format],
|
|
79
|
+
"Default Mode": [icons.data_type_string, icons.no_format],
|
|
86
80
|
},
|
|
87
81
|
"Tables": {
|
|
88
|
-
"Table Name": data_type_string,
|
|
89
|
-
"Type": data_type_string,
|
|
90
|
-
"Row Count": data_type_long,
|
|
91
|
-
"Total Size": data_type_long,
|
|
92
|
-
"Dictionary Size": data_type_long,
|
|
93
|
-
"Data Size": data_type_long,
|
|
94
|
-
"Hierarchy Size": data_type_long,
|
|
95
|
-
"Relationship Size": data_type_long,
|
|
96
|
-
"User Hierarchy Size": data_type_long,
|
|
97
|
-
"Partitions": data_type_long,
|
|
98
|
-
"Columns": data_type_long,
|
|
99
|
-
"% DB": data_type_double,
|
|
82
|
+
"Table Name": [icons.data_type_string, icons.no_format],
|
|
83
|
+
"Type": [icons.data_type_string, icons.no_format],
|
|
84
|
+
"Row Count": [icons.data_type_long, icons.int_format],
|
|
85
|
+
"Total Size": [icons.data_type_long, icons.int_format],
|
|
86
|
+
"Dictionary Size": [icons.data_type_long, icons.int_format],
|
|
87
|
+
"Data Size": [icons.data_type_long, icons.int_format],
|
|
88
|
+
"Hierarchy Size": [icons.data_type_long, icons.int_format],
|
|
89
|
+
"Relationship Size": [icons.data_type_long, icons.int_format],
|
|
90
|
+
"User Hierarchy Size": [icons.data_type_long, icons.int_format],
|
|
91
|
+
"Partitions": [icons.data_type_long, icons.int_format],
|
|
92
|
+
"Columns": [icons.data_type_long, icons.int_format],
|
|
93
|
+
"% DB": [icons.data_type_double, icons.pct_format],
|
|
100
94
|
},
|
|
101
95
|
"Partitions": {
|
|
102
|
-
"Table Name": data_type_string,
|
|
103
|
-
"Partition Name": data_type_string,
|
|
104
|
-
"Mode": data_type_string,
|
|
105
|
-
"Record Count": data_type_long,
|
|
106
|
-
"Segment Count": data_type_long,
|
|
107
|
-
"Records per Segment": data_type_double,
|
|
96
|
+
"Table Name": [icons.data_type_string, icons.no_format],
|
|
97
|
+
"Partition Name": [icons.data_type_string, icons.no_format],
|
|
98
|
+
"Mode": [icons.data_type_string, icons.no_format],
|
|
99
|
+
"Record Count": [icons.data_type_long, icons.int_format],
|
|
100
|
+
"Segment Count": [icons.data_type_long, icons.int_format],
|
|
101
|
+
"Records per Segment": [icons.data_type_double, icons.int_format],
|
|
108
102
|
},
|
|
109
103
|
"Columns": {
|
|
110
|
-
"Table Name": data_type_string,
|
|
111
|
-
"Column Name": data_type_string,
|
|
112
|
-
"Type": data_type_string,
|
|
113
|
-
"Cardinality": data_type_long,
|
|
114
|
-
"Total Size": data_type_long,
|
|
115
|
-
"Data Size": data_type_long,
|
|
116
|
-
"Dictionary Size": data_type_long,
|
|
117
|
-
"Hierarchy Size": data_type_long,
|
|
118
|
-
"% Table": data_type_double,
|
|
119
|
-
"% DB": data_type_double,
|
|
120
|
-
"Data Type": data_type_string,
|
|
121
|
-
"Encoding": data_type_string,
|
|
122
|
-
"Is Resident": data_type_bool,
|
|
123
|
-
"Temperature": data_type_double,
|
|
124
|
-
"Last Accessed": data_type_timestamp,
|
|
104
|
+
"Table Name": [icons.data_type_string, icons.no_format],
|
|
105
|
+
"Column Name": [icons.data_type_string, icons.no_format],
|
|
106
|
+
"Type": [icons.data_type_string, icons.no_format],
|
|
107
|
+
"Cardinality": [icons.data_type_long, icons.int_format],
|
|
108
|
+
"Total Size": [icons.data_type_long, icons.int_format],
|
|
109
|
+
"Data Size": [icons.data_type_long, icons.int_format],
|
|
110
|
+
"Dictionary Size": [icons.data_type_long, icons.int_format],
|
|
111
|
+
"Hierarchy Size": [icons.data_type_long, icons.int_format],
|
|
112
|
+
"% Table": [icons.data_type_double, icons.pct_format],
|
|
113
|
+
"% DB": [icons.data_type_double, icons.pct_format],
|
|
114
|
+
"Data Type": [icons.data_type_string, icons.no_format],
|
|
115
|
+
"Encoding": [icons.data_type_string, icons.no_format],
|
|
116
|
+
"Is Resident": [icons.data_type_bool, icons.no_format],
|
|
117
|
+
"Temperature": [icons.data_type_double, icons.int_format],
|
|
118
|
+
"Last Accessed": [icons.data_type_timestamp, icons.no_format],
|
|
125
119
|
},
|
|
126
120
|
"Hierarchies": {
|
|
127
|
-
"Table Name": data_type_string,
|
|
128
|
-
"Hierarchy Name": data_type_string,
|
|
129
|
-
"Used Size": data_type_long,
|
|
121
|
+
"Table Name": [icons.data_type_string, icons.no_format],
|
|
122
|
+
"Hierarchy Name": [icons.data_type_string, icons.no_format],
|
|
123
|
+
"Used Size": [icons.data_type_long, icons.int_format],
|
|
130
124
|
},
|
|
131
125
|
"Relationships": {
|
|
132
|
-
"From Object": data_type_string,
|
|
133
|
-
"To Object": data_type_string,
|
|
134
|
-
"Multiplicity": data_type_string,
|
|
135
|
-
"Used Size": data_type_long,
|
|
136
|
-
"Max From Cardinality": data_type_long,
|
|
137
|
-
"Max To Cardinality": data_type_long,
|
|
138
|
-
"Missing Rows": data_type_long,
|
|
126
|
+
"From Object": [icons.data_type_string, icons.no_format],
|
|
127
|
+
"To Object": [icons.data_type_string, icons.no_format],
|
|
128
|
+
"Multiplicity": [icons.data_type_string, icons.no_format],
|
|
129
|
+
"Used Size": [icons.data_type_long, icons.int_format],
|
|
130
|
+
"Max From Cardinality": [icons.data_type_long, icons.int_format],
|
|
131
|
+
"Max To Cardinality": [icons.data_type_long, icons.int_format],
|
|
132
|
+
"Missing Rows": [icons.data_type_long, icons.int_format],
|
|
139
133
|
},
|
|
140
134
|
}
|
|
141
135
|
|
|
@@ -163,7 +157,8 @@ def vertipaq_analyzer(
|
|
|
163
157
|
table_count = tom.model.Tables.Count
|
|
164
158
|
column_count = len(list(tom.all_columns()))
|
|
165
159
|
|
|
166
|
-
dfR["Missing Rows"] =
|
|
160
|
+
dfR["Missing Rows"] = 0
|
|
161
|
+
dfR["Missing Rows"] = dfR["Missing Rows"].astype(int)
|
|
167
162
|
|
|
168
163
|
# Direct Lake
|
|
169
164
|
if read_stats_from_data:
|
|
@@ -323,38 +318,16 @@ def vertipaq_analyzer(
|
|
|
323
318
|
dfC["% DB"] = round((dfC["Total Size"] / db_total_size) * 100, 2)
|
|
324
319
|
columnList = list(vertipaq_map["Columns"].keys())
|
|
325
320
|
|
|
321
|
+
dfC = dfC[dfC["Type"] != "RowNumber"].reset_index(drop=True)
|
|
322
|
+
|
|
326
323
|
colSize = dfC[columnList].sort_values(by="Total Size", ascending=False)
|
|
327
324
|
temp = dfC[columnList].sort_values(by="Temperature", ascending=False)
|
|
328
325
|
colSize.reset_index(drop=True, inplace=True)
|
|
329
326
|
temp.reset_index(drop=True, inplace=True)
|
|
330
327
|
|
|
331
328
|
export_Col = colSize.copy()
|
|
332
|
-
|
|
333
|
-
int_cols = []
|
|
334
|
-
pct_cols = []
|
|
335
|
-
for k, v in vertipaq_map["Columns"].items():
|
|
336
|
-
if v in ["int", "long"]:
|
|
337
|
-
int_cols.append(k)
|
|
338
|
-
elif v in ["float", "double"] and k != "Temperature":
|
|
339
|
-
pct_cols.append(k)
|
|
340
|
-
colSize[int_cols] = colSize[int_cols].map("{:,}".format)
|
|
341
|
-
temp[int_cols] = temp[int_cols].map("{:,}".format)
|
|
342
|
-
colSize[pct_cols] = colSize[pct_cols].map("{:.2f}%".format)
|
|
343
|
-
temp[pct_cols] = temp[pct_cols].map("{:.2f}%".format)
|
|
344
|
-
|
|
345
|
-
# Tables
|
|
346
|
-
int_cols = []
|
|
347
|
-
pct_cols = []
|
|
348
|
-
for k, v in vertipaq_map["Tables"].items():
|
|
349
|
-
if v in ["int", "long"]:
|
|
350
|
-
int_cols.append(k)
|
|
351
|
-
elif v in ["float", "double"]:
|
|
352
|
-
pct_cols.append(k)
|
|
353
329
|
export_Table = dfT.copy()
|
|
354
330
|
|
|
355
|
-
dfT[int_cols] = dfT[int_cols].map("{:,}".format)
|
|
356
|
-
dfT[pct_cols] = dfT[pct_cols].map("{:.2f}%".format)
|
|
357
|
-
|
|
358
331
|
# Relationships
|
|
359
332
|
dfR = pd.merge(
|
|
360
333
|
dfR,
|
|
@@ -386,14 +359,6 @@ def vertipaq_analyzer(
|
|
|
386
359
|
dfR.reset_index(drop=True, inplace=True)
|
|
387
360
|
export_Rel = dfR.copy()
|
|
388
361
|
|
|
389
|
-
int_cols = []
|
|
390
|
-
for k, v in vertipaq_map["Relationships"].items():
|
|
391
|
-
if v in ["int", "long"]:
|
|
392
|
-
int_cols.append(k)
|
|
393
|
-
if not read_stats_from_data:
|
|
394
|
-
int_cols.remove("Missing Rows")
|
|
395
|
-
dfR[int_cols] = dfR[int_cols].map("{:,}".format)
|
|
396
|
-
|
|
397
362
|
# Partitions
|
|
398
363
|
dfP = dfP[
|
|
399
364
|
[
|
|
@@ -410,12 +375,6 @@ def vertipaq_analyzer(
|
|
|
410
375
|
) # Remove after records per segment is fixed
|
|
411
376
|
dfP.reset_index(drop=True, inplace=True)
|
|
412
377
|
export_Part = dfP.copy()
|
|
413
|
-
int_cols = []
|
|
414
|
-
for k, v in vertipaq_map["Partitions"].items():
|
|
415
|
-
if v in ["int", "long", "double", "float"]:
|
|
416
|
-
int_cols.append(k)
|
|
417
|
-
intList = ["Record Count", "Segment Count", "Records per Segment"]
|
|
418
|
-
dfP[intList] = dfP[intList].map("{:,}".format)
|
|
419
378
|
|
|
420
379
|
# Hierarchies
|
|
421
380
|
dfH_filt = dfH[dfH["Level Ordinal"] == 0]
|
|
@@ -426,8 +385,6 @@ def vertipaq_analyzer(
|
|
|
426
385
|
dfH_filt.fillna({"Used Size": 0}, inplace=True)
|
|
427
386
|
dfH_filt["Used Size"] = dfH_filt["Used Size"].astype(int)
|
|
428
387
|
export_Hier = dfH_filt.copy()
|
|
429
|
-
intList = ["Used Size"]
|
|
430
|
-
dfH_filt[intList] = dfH_filt[intList].map("{:,}".format)
|
|
431
388
|
|
|
432
389
|
# Model
|
|
433
390
|
# Converting to KB/MB/GB necessitates division by 1024 * 1000.
|
|
@@ -453,11 +410,63 @@ def vertipaq_analyzer(
|
|
|
453
410
|
dfModel.reset_index(drop=True, inplace=True)
|
|
454
411
|
dfModel["Default Mode"] = dfModel["Default Mode"].astype(str)
|
|
455
412
|
export_Model = dfModel.copy()
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
413
|
+
|
|
414
|
+
def _style_columns_based_on_types(dataframe: pd.DataFrame, column_type_mapping):
|
|
415
|
+
|
|
416
|
+
format_mapping = {
|
|
417
|
+
"int": "{:,}",
|
|
418
|
+
"pct": "{:.2f}%",
|
|
419
|
+
"": "{}",
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
format_dict = {
|
|
423
|
+
col: format_mapping[dt] for col, dt in column_type_mapping.items()
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
return dataframe.style.format(format_dict)
|
|
427
|
+
|
|
428
|
+
dfModel = _style_columns_based_on_types(
|
|
429
|
+
dfModel,
|
|
430
|
+
column_type_mapping={
|
|
431
|
+
key: values[1] for key, values in vertipaq_map["Model"].items()
|
|
432
|
+
},
|
|
433
|
+
)
|
|
434
|
+
dfT = _style_columns_based_on_types(
|
|
435
|
+
dfT,
|
|
436
|
+
column_type_mapping={
|
|
437
|
+
key: values[1] for key, values in vertipaq_map["Tables"].items()
|
|
438
|
+
},
|
|
439
|
+
)
|
|
440
|
+
dfP = _style_columns_based_on_types(
|
|
441
|
+
dfP,
|
|
442
|
+
column_type_mapping={
|
|
443
|
+
key: values[1] for key, values in vertipaq_map["Partitions"].items()
|
|
444
|
+
},
|
|
445
|
+
)
|
|
446
|
+
colSize = _style_columns_based_on_types(
|
|
447
|
+
colSize,
|
|
448
|
+
column_type_mapping={
|
|
449
|
+
key: values[1] for key, values in vertipaq_map["Columns"].items()
|
|
450
|
+
},
|
|
451
|
+
)
|
|
452
|
+
temp = _style_columns_based_on_types(
|
|
453
|
+
temp,
|
|
454
|
+
column_type_mapping={
|
|
455
|
+
key: values[1] for key, values in vertipaq_map["Columns"].items()
|
|
456
|
+
},
|
|
457
|
+
)
|
|
458
|
+
dfR = _style_columns_based_on_types(
|
|
459
|
+
dfR,
|
|
460
|
+
column_type_mapping={
|
|
461
|
+
key: values[1] for key, values in vertipaq_map["Relationships"].items()
|
|
462
|
+
},
|
|
463
|
+
)
|
|
464
|
+
dfH_filt = _style_columns_based_on_types(
|
|
465
|
+
dfH_filt,
|
|
466
|
+
column_type_mapping={
|
|
467
|
+
key: values[1] for key, values in vertipaq_map["Hierarchies"].items()
|
|
468
|
+
},
|
|
469
|
+
)
|
|
461
470
|
|
|
462
471
|
dataFrames = {
|
|
463
472
|
"dfModel": dfModel,
|
|
@@ -484,8 +493,6 @@ def vertipaq_analyzer(
|
|
|
484
493
|
)
|
|
485
494
|
|
|
486
495
|
if export == "table":
|
|
487
|
-
# spark = SparkSession.builder.getOrCreate()
|
|
488
|
-
|
|
489
496
|
lakehouse_id = fabric.get_lakehouse_id()
|
|
490
497
|
lake_workspace = fabric.resolve_workspace_name()
|
|
491
498
|
lakehouse = resolve_lakehouse_name(
|
|
@@ -499,7 +506,7 @@ def vertipaq_analyzer(
|
|
|
499
506
|
if len(lakeT_filt) == 0:
|
|
500
507
|
runId = 1
|
|
501
508
|
else:
|
|
502
|
-
max_run_id =
|
|
509
|
+
max_run_id = _get_max_run_id(lakehouse=lakehouse, table_name=lakeTName)
|
|
503
510
|
runId = max_run_id + 1
|
|
504
511
|
|
|
505
512
|
dfMap = {
|
|
@@ -549,23 +556,23 @@ def vertipaq_analyzer(
|
|
|
549
556
|
df.columns = df.columns.str.replace(" ", "_")
|
|
550
557
|
|
|
551
558
|
schema = {
|
|
552
|
-
"Capacity_Name": data_type_string,
|
|
553
|
-
"Capacity_Id": data_type_string,
|
|
554
|
-
"Workspace_Name": data_type_string,
|
|
555
|
-
"Workspace_Id": data_type_string,
|
|
556
|
-
"Dataset_Name": data_type_string,
|
|
557
|
-
"Dataset_Id": data_type_string,
|
|
558
|
-
"Configured_By": data_type_string,
|
|
559
|
+
"Capacity_Name": icons.data_type_string,
|
|
560
|
+
"Capacity_Id": icons.data_type_string,
|
|
561
|
+
"Workspace_Name": icons.data_type_string,
|
|
562
|
+
"Workspace_Id": icons.data_type_string,
|
|
563
|
+
"Dataset_Name": icons.data_type_string,
|
|
564
|
+
"Dataset_Id": icons.data_type_string,
|
|
565
|
+
"Configured_By": icons.data_type_string,
|
|
559
566
|
}
|
|
560
567
|
|
|
561
568
|
schema.update(
|
|
562
569
|
{
|
|
563
|
-
key.replace(" ", "_"): value
|
|
570
|
+
key.replace(" ", "_"): value[0]
|
|
564
571
|
for key, value in vertipaq_map[key_name].items()
|
|
565
572
|
}
|
|
566
573
|
)
|
|
567
|
-
schema["RunId"] = data_type_long
|
|
568
|
-
schema["Timestamp"] = data_type_timestamp
|
|
574
|
+
schema["RunId"] = icons.data_type_long
|
|
575
|
+
schema["Timestamp"] = icons.data_type_timestamp
|
|
569
576
|
|
|
570
577
|
delta_table_name = f"VertipaqAnalyzer_{obj}".lower()
|
|
571
578
|
save_as_delta_table(
|
|
@@ -739,7 +746,11 @@ def visualize_vertipaq(dataframes):
|
|
|
739
746
|
"ColumnName": "Column Name",
|
|
740
747
|
"Tooltip": "The name of the column",
|
|
741
748
|
},
|
|
742
|
-
{
|
|
749
|
+
{
|
|
750
|
+
"ViewName": "Column",
|
|
751
|
+
"ColumnName": "Type",
|
|
752
|
+
"Tooltip": "The type of column",
|
|
753
|
+
},
|
|
743
754
|
{
|
|
744
755
|
"ViewName": "Column",
|
|
745
756
|
"ColumnName": "Cardinality",
|
sempy_labs/_warehouses.py
CHANGED
|
@@ -11,17 +11,24 @@ from sempy.fabric.exceptions import FabricHTTPException
|
|
|
11
11
|
|
|
12
12
|
|
|
13
13
|
def create_warehouse(
|
|
14
|
-
warehouse: str,
|
|
14
|
+
warehouse: str,
|
|
15
|
+
description: Optional[str] = None,
|
|
16
|
+
case_insensitive_collation: bool = False,
|
|
17
|
+
workspace: Optional[str] = None,
|
|
15
18
|
):
|
|
16
19
|
"""
|
|
17
20
|
Creates a Fabric warehouse.
|
|
18
21
|
|
|
22
|
+
This is a wrapper function for the following API: `Items - Create Warehouse <https://learn.microsoft.com/rest/api/fabric/warehouse/items/create-warehouse`_.
|
|
23
|
+
|
|
19
24
|
Parameters
|
|
20
25
|
----------
|
|
21
26
|
warehouse: str
|
|
22
27
|
Name of the warehouse.
|
|
23
28
|
description : str, default=None
|
|
24
29
|
A description of the warehouse.
|
|
30
|
+
case_insensitive_collation: bool, default=False
|
|
31
|
+
If True, creates the warehouse with case-insensitive collation.
|
|
25
32
|
workspace : str, default=None
|
|
26
33
|
The Fabric workspace name.
|
|
27
34
|
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
@@ -34,6 +41,11 @@ def create_warehouse(
|
|
|
34
41
|
|
|
35
42
|
if description:
|
|
36
43
|
request_body["description"] = description
|
|
44
|
+
if case_insensitive_collation:
|
|
45
|
+
request_body.setdefault("creationPayload", {})
|
|
46
|
+
request_body["creationPayload"][
|
|
47
|
+
"defaultCollation"
|
|
48
|
+
] = "Latin1_General_100_CI_AS_KS_WS_SC_UTF8"
|
|
37
49
|
|
|
38
50
|
client = fabric.FabricRestClient()
|
|
39
51
|
response = client.post(
|
|
@@ -51,6 +63,8 @@ def list_warehouses(workspace: Optional[str] = None) -> pd.DataFrame:
|
|
|
51
63
|
"""
|
|
52
64
|
Shows the warehouses within a workspace.
|
|
53
65
|
|
|
66
|
+
This is a wrapper function for the following API: `Items - List Warehouses <https://learn.microsoft.com/rest/api/fabric/warehouse/items/list-warehouses`_.
|
|
67
|
+
|
|
54
68
|
Parameters
|
|
55
69
|
----------
|
|
56
70
|
workspace : str, default=None
|
|
@@ -105,6 +119,8 @@ def delete_warehouse(name: str, workspace: Optional[str] = None):
|
|
|
105
119
|
"""
|
|
106
120
|
Deletes a Fabric warehouse.
|
|
107
121
|
|
|
122
|
+
This is a wrapper function for the following API: `Items - Delete Warehouse <https://learn.microsoft.com/rest/api/fabric/warehouse/items/delete-warehouse`_.
|
|
123
|
+
|
|
108
124
|
Parameters
|
|
109
125
|
----------
|
|
110
126
|
name: str
|
|
@@ -130,3 +146,76 @@ def delete_warehouse(name: str, workspace: Optional[str] = None):
|
|
|
130
146
|
print(
|
|
131
147
|
f"{icons.green_dot} The '{name}' warehouse within the '{workspace}' workspace has been deleted."
|
|
132
148
|
)
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def get_warehouse_tables(
|
|
152
|
+
warehouse: str, workspace: Optional[str] = None
|
|
153
|
+
) -> pd.DataFrame:
|
|
154
|
+
"""
|
|
155
|
+
Shows a list of the tables in the Fabric warehouse. This function is based on INFORMATION_SCHEMA.TABLES.
|
|
156
|
+
|
|
157
|
+
Parameters
|
|
158
|
+
----------
|
|
159
|
+
warehouse : str
|
|
160
|
+
Name of the Fabric warehouse.
|
|
161
|
+
workspace : str, default=None
|
|
162
|
+
The Fabric workspace name.
|
|
163
|
+
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
164
|
+
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
165
|
+
|
|
166
|
+
Returns
|
|
167
|
+
-------
|
|
168
|
+
pandas.DataFrame
|
|
169
|
+
A pandas dataframe showing a list of the tables in the Fabric warehouse.
|
|
170
|
+
"""
|
|
171
|
+
|
|
172
|
+
from sempy_labs._sql import ConnectWarehouse
|
|
173
|
+
|
|
174
|
+
with ConnectWarehouse(warehouse=warehouse, workspace=workspace) as sql:
|
|
175
|
+
df = sql.query(
|
|
176
|
+
"""
|
|
177
|
+
SELECT TABLE_SCHEMA AS [Schema], TABLE_NAME AS [Table Name], TABLE_TYPE AS [Table Type]
|
|
178
|
+
FROM INFORMATION_SCHEMA.TABLES
|
|
179
|
+
WHERE TABLE_TYPE = 'BASE TABLE'
|
|
180
|
+
"""
|
|
181
|
+
)
|
|
182
|
+
|
|
183
|
+
return df
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def get_warehouse_columns(
|
|
187
|
+
warehouse: str, workspace: Optional[str] = None
|
|
188
|
+
) -> pd.DataFrame:
|
|
189
|
+
"""
|
|
190
|
+
Shows a list of the columns in each table within the Fabric warehouse. This function is based on INFORMATION_SCHEMA.COLUMNS.
|
|
191
|
+
|
|
192
|
+
Parameters
|
|
193
|
+
----------
|
|
194
|
+
warehouse : str
|
|
195
|
+
Name of the Fabric warehouse.
|
|
196
|
+
workspace : str, default=None
|
|
197
|
+
The Fabric workspace name.
|
|
198
|
+
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
199
|
+
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
200
|
+
|
|
201
|
+
Returns
|
|
202
|
+
-------
|
|
203
|
+
pandas.DataFrame
|
|
204
|
+
A pandas dataframe showing a list of the columns in each table within the Fabric warehouse.
|
|
205
|
+
"""
|
|
206
|
+
|
|
207
|
+
from sempy_labs._sql import ConnectWarehouse
|
|
208
|
+
|
|
209
|
+
with ConnectWarehouse(warehouse=warehouse, workspace=workspace) as sql:
|
|
210
|
+
df = sql.query(
|
|
211
|
+
"""
|
|
212
|
+
SELECT t.TABLE_SCHEMA AS [Schema], t.TABLE_NAME AS [Table Name], c.COLUMN_NAME AS [Column Name], c.DATA_TYPE AS [Data Type], c.IS_NULLABLE AS [Is Nullable], c.CHARACTER_MAXIMUM_LENGTH AS [Character Max Length]
|
|
213
|
+
FROM INFORMATION_SCHEMA.TABLES AS t
|
|
214
|
+
LEFT JOIN INFORMATION_SCHEMA.COLUMNS AS c
|
|
215
|
+
ON t.TABLE_NAME = c.TABLE_NAME
|
|
216
|
+
AND t.TABLE_SCHEMA = c.TABLE_SCHEMA
|
|
217
|
+
WHERE t.TABLE_TYPE = 'BASE TABLE'
|
|
218
|
+
"""
|
|
219
|
+
)
|
|
220
|
+
|
|
221
|
+
return df
|
sempy_labs/_workloads.py
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import sempy.fabric as fabric
|
|
2
|
+
import pandas as pd
|
|
3
|
+
from typing import Optional
|
|
4
|
+
import sempy_labs._icons as icons
|
|
5
|
+
from sempy.fabric.exceptions import FabricHTTPException
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def list_workloads(capacity_name: str) -> pd.DataFrame:
|
|
9
|
+
"""
|
|
10
|
+
Returns the current state of the specified capacity workloads.
|
|
11
|
+
If a workload is enabled, the percentage of maximum memory that the workload can consume is also returned.
|
|
12
|
+
|
|
13
|
+
This is a wrapper function for the following API: `Capacities - Get Workloads <https://learn.microsoft.com/rest/api/power-bi/capacities/get-workloads>`_.
|
|
14
|
+
|
|
15
|
+
Parameters
|
|
16
|
+
----------
|
|
17
|
+
capacity_name : str
|
|
18
|
+
The capacity name.
|
|
19
|
+
|
|
20
|
+
Returns
|
|
21
|
+
-------
|
|
22
|
+
pandas.DataFrame
|
|
23
|
+
A pandas dataframe showing the current state of the specified capacity workloads.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from sempy_labs._helper_functions import resolve_capacity_id
|
|
27
|
+
|
|
28
|
+
df = pd.DataFrame(
|
|
29
|
+
columns=["Workload Name", "State", "Max Memory Percentage Set By User"]
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
capacity_id = resolve_capacity_id(capacity_name=capacity_name)
|
|
33
|
+
|
|
34
|
+
client = fabric.PowerBIRestClient()
|
|
35
|
+
response = client.get(f"/v1.0/myorg/capacities/{capacity_id}/Workloads")
|
|
36
|
+
|
|
37
|
+
if response.status_code != 200:
|
|
38
|
+
raise FabricHTTPException(response)
|
|
39
|
+
|
|
40
|
+
for v in response.json().get("value", []):
|
|
41
|
+
new_data = {
|
|
42
|
+
"Workload Name": v.get("name"),
|
|
43
|
+
"State": v.get("state"),
|
|
44
|
+
"Max Memory Percentage Set By User": v.get("maxMemoryPercentageSetByUser"),
|
|
45
|
+
}
|
|
46
|
+
df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
|
|
47
|
+
|
|
48
|
+
int_cols = ["Max Memory Percentage Set By User"]
|
|
49
|
+
df[int_cols] = df[int_cols].astype(int)
|
|
50
|
+
|
|
51
|
+
return df
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def patch_workload(
|
|
55
|
+
capacity_name: str,
|
|
56
|
+
workload_name: str,
|
|
57
|
+
state: Optional[str] = None,
|
|
58
|
+
max_memory_percentage: Optional[int] = None,
|
|
59
|
+
):
|
|
60
|
+
"""
|
|
61
|
+
Changes the state of a specific workload to Enabled or Disabled.
|
|
62
|
+
When enabling a workload, specify the percentage of maximum memory that the workload can consume.
|
|
63
|
+
|
|
64
|
+
This is a wrapper function for the following API: `Capacities - Patch Workload <https://learn.microsoft.com/rest/api/power-bi/capacities/patch-workload>`_.
|
|
65
|
+
|
|
66
|
+
Parameters
|
|
67
|
+
----------
|
|
68
|
+
capacity_name : str
|
|
69
|
+
The capacity name.
|
|
70
|
+
workload_name : str
|
|
71
|
+
The workload name.
|
|
72
|
+
state : str, default=None
|
|
73
|
+
The capacity workload state.
|
|
74
|
+
max_memory_percentage : int, default=None
|
|
75
|
+
The percentage of the maximum memory that a workload can consume (set by the user).
|
|
76
|
+
"""
|
|
77
|
+
|
|
78
|
+
from sempy_labs._helper_functions import resolve_capacity_id
|
|
79
|
+
|
|
80
|
+
capacity_id = resolve_capacity_id(capacity_name=capacity_name)
|
|
81
|
+
|
|
82
|
+
states = ["Disabled", "Enabled", "Unsupported"]
|
|
83
|
+
state = state.capitalize()
|
|
84
|
+
if state is not None and state not in states:
|
|
85
|
+
raise ValueError(
|
|
86
|
+
f"{icons.red_dot} Invalid 'state' parameter. Please choose from these options: {states}."
|
|
87
|
+
)
|
|
88
|
+
if max_memory_percentage is not None and (
|
|
89
|
+
max_memory_percentage < 0 or max_memory_percentage > 100
|
|
90
|
+
):
|
|
91
|
+
raise ValueError(
|
|
92
|
+
f"{icons.red_dot} Invalid max memory percentage. Must be a value between 0-100."
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
client = fabric.PowerBIRestClient()
|
|
96
|
+
url = f"/v1.0/myorg/capacities/{capacity_id}/Workloads/{workload_name}"
|
|
97
|
+
get_response = client.get(url)
|
|
98
|
+
if get_response.status_code != 200:
|
|
99
|
+
raise FabricHTTPException(get_response)
|
|
100
|
+
|
|
101
|
+
get_json = get_response.json().get("value")
|
|
102
|
+
current_state = get_json.get("state")
|
|
103
|
+
current_max_memory = get_json.get("maxMemoryPercentageSetByUser")
|
|
104
|
+
|
|
105
|
+
if current_state == state and str(current_max_memory) == str(max_memory_percentage):
|
|
106
|
+
print(
|
|
107
|
+
f"{icons.info} The current workload settings are the same as those specified in the parameters of this function. The workload has not been updated."
|
|
108
|
+
)
|
|
109
|
+
return
|
|
110
|
+
|
|
111
|
+
payload = {}
|
|
112
|
+
if state is not None:
|
|
113
|
+
payload["state"] = state
|
|
114
|
+
else:
|
|
115
|
+
payload["state"] = current_state
|
|
116
|
+
if max_memory_percentage is not None:
|
|
117
|
+
payload["maxMemoryPercentageSetByUser"] = max_memory_percentage
|
|
118
|
+
else:
|
|
119
|
+
payload["maxMemoryPercentageSetByUser"] = current_max_memory
|
|
120
|
+
|
|
121
|
+
response = client.patch(url, json=payload)
|
|
122
|
+
|
|
123
|
+
if response.status_code != 200:
|
|
124
|
+
raise FabricHTTPException(response)
|
|
125
|
+
|
|
126
|
+
print(
|
|
127
|
+
f"The '{workload_name}' workload within the '{capacity_name}' capacity has been updated accordingly."
|
|
128
|
+
)
|
|
@@ -12,6 +12,8 @@ def provision_workspace_identity(workspace: Optional[str] = None):
|
|
|
12
12
|
"""
|
|
13
13
|
Provisions a workspace identity for a workspace.
|
|
14
14
|
|
|
15
|
+
This is a wrapper function for the following API: `Workspaces - Provision Identity <https://learn.microsoft.com/rest/api/fabric/core/workspaces/provision-identity`_.
|
|
16
|
+
|
|
15
17
|
Parameters
|
|
16
18
|
----------
|
|
17
19
|
workspace : str, default=None
|
|
@@ -20,8 +22,6 @@ def provision_workspace_identity(workspace: Optional[str] = None):
|
|
|
20
22
|
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
21
23
|
"""
|
|
22
24
|
|
|
23
|
-
# https://learn.microsoft.com/en-us/rest/api/fabric/core/workspaces/provision-identity?tabs=HTTP
|
|
24
|
-
|
|
25
25
|
workspace, workspace_id = resolve_workspace_name_and_id(workspace)
|
|
26
26
|
|
|
27
27
|
client = fabric.FabricRestClient()
|
|
@@ -41,6 +41,8 @@ def deprovision_workspace_identity(workspace: Optional[str] = None):
|
|
|
41
41
|
"""
|
|
42
42
|
Deprovisions a workspace identity for a workspace.
|
|
43
43
|
|
|
44
|
+
This is a wrapper function for the following API: `Workspaces - Derovision Identity <https://learn.microsoft.com/rest/api/fabric/core/workspaces/deprovision-identity`_.
|
|
45
|
+
|
|
44
46
|
Parameters
|
|
45
47
|
----------
|
|
46
48
|
workspace : str, default=None
|
|
@@ -49,8 +51,6 @@ def deprovision_workspace_identity(workspace: Optional[str] = None):
|
|
|
49
51
|
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
50
52
|
"""
|
|
51
53
|
|
|
52
|
-
# https://learn.microsoft.com/en-us/rest/api/fabric/core/workspaces/deprovision-identity?tabs=HTTP
|
|
53
|
-
|
|
54
54
|
workspace, workspace_id = resolve_workspace_name_and_id(workspace)
|
|
55
55
|
|
|
56
56
|
client = fabric.FabricRestClient()
|