semantic-link-labs 0.4.1__py3-none-any.whl → 0.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of semantic-link-labs might be problematic. Click here for more details.
- {semantic_link_labs-0.4.1.dist-info → semantic_link_labs-0.5.0.dist-info}/METADATA +1 -1
- semantic_link_labs-0.5.0.dist-info/RECORD +53 -0
- {semantic_link_labs-0.4.1.dist-info → semantic_link_labs-0.5.0.dist-info}/WHEEL +1 -1
- sempy_labs/__init__.py +51 -27
- sempy_labs/_ai.py +32 -51
- sempy_labs/_clear_cache.py +2 -3
- sempy_labs/_connections.py +39 -38
- sempy_labs/_dax.py +5 -9
- sempy_labs/_generate_semantic_model.py +15 -21
- sempy_labs/_helper_functions.py +20 -25
- sempy_labs/_icons.py +6 -0
- sempy_labs/_list_functions.py +1172 -392
- sempy_labs/_model_auto_build.py +3 -5
- sempy_labs/_model_bpa.py +20 -24
- sempy_labs/_model_dependencies.py +7 -14
- sempy_labs/_one_lake_integration.py +14 -24
- sempy_labs/_query_scale_out.py +13 -31
- sempy_labs/_refresh_semantic_model.py +8 -18
- sempy_labs/_translations.py +5 -5
- sempy_labs/_vertipaq.py +11 -18
- sempy_labs/directlake/_directlake_schema_compare.py +11 -15
- sempy_labs/directlake/_directlake_schema_sync.py +35 -40
- sempy_labs/directlake/_fallback.py +3 -7
- sempy_labs/directlake/_get_directlake_lakehouse.py +3 -4
- sempy_labs/directlake/_get_shared_expression.py +5 -11
- sempy_labs/directlake/_guardrails.py +5 -7
- sempy_labs/directlake/_list_directlake_model_calc_tables.py +28 -26
- sempy_labs/directlake/_show_unsupported_directlake_objects.py +3 -4
- sempy_labs/directlake/_update_directlake_model_lakehouse_connection.py +11 -16
- sempy_labs/directlake/_update_directlake_partition_entity.py +25 -15
- sempy_labs/directlake/_warm_cache.py +10 -15
- sempy_labs/lakehouse/__init__.py +0 -2
- sempy_labs/lakehouse/_get_lakehouse_columns.py +4 -3
- sempy_labs/lakehouse/_get_lakehouse_tables.py +12 -11
- sempy_labs/lakehouse/_lakehouse.py +6 -7
- sempy_labs/lakehouse/_shortcuts.py +10 -111
- sempy_labs/migration/__init__.py +4 -2
- sempy_labs/migration/_create_pqt_file.py +5 -14
- sempy_labs/migration/_migrate_calctables_to_lakehouse.py +7 -7
- sempy_labs/migration/_migrate_calctables_to_semantic_model.py +4 -4
- sempy_labs/migration/_migrate_model_objects_to_semantic_model.py +3 -8
- sempy_labs/migration/_migrate_tables_columns_to_semantic_model.py +6 -6
- sempy_labs/migration/_migration_validation.py +5 -164
- sempy_labs/migration/_refresh_calc_tables.py +5 -5
- sempy_labs/report/__init__.py +2 -2
- sempy_labs/report/_generate_report.py +14 -19
- sempy_labs/report/_report_functions.py +41 -83
- sempy_labs/report/_report_rebind.py +43 -44
- sempy_labs/tom/__init__.py +6 -0
- sempy_labs/{_tom.py → tom/_model.py} +274 -337
- semantic_link_labs-0.4.1.dist-info/RECORD +0 -52
- {semantic_link_labs-0.4.1.dist-info → semantic_link_labs-0.5.0.dist-info}/LICENSE +0 -0
- {semantic_link_labs-0.4.1.dist-info → semantic_link_labs-0.5.0.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
sempy_labs/__init__.py,sha256=Xpq66PKo8KK7QIP0QYme_X_FyfZnpOBsli2V4bnMNYo,4843
|
|
2
|
+
sempy_labs/_ai.py,sha256=WNCLh8wzZ7brXJWe2CNC79D9NHeS-2KpAWCIAlvZD7U,17784
|
|
3
|
+
sempy_labs/_clear_cache.py,sha256=ELHcD4smBS3EiHEO1Ux97yV_Q7j_zB8TJJ4-kS1ylCU,1394
|
|
4
|
+
sempy_labs/_connections.py,sha256=ghYtHBLTaTmYwgllh6I3zfFjG5lcxM2BAQR52B3z-t0,7795
|
|
5
|
+
sempy_labs/_dax.py,sha256=u4qVxsu2dVaOJmso-ErScNZ5yI4lGQTlon_jmrAzvGs,2148
|
|
6
|
+
sempy_labs/_generate_semantic_model.py,sha256=rBi1jmLQJNq9NOt56AqqYzAXBJ2DX2bpG7FYrHXyiEA,9180
|
|
7
|
+
sempy_labs/_helper_functions.py,sha256=RlLtpEmBpoX285sSFKBBjkPfXqp8_FSIwYcoR4rZG58,14251
|
|
8
|
+
sempy_labs/_icons.py,sha256=WkmhtLcQPO1PlcwFklb253dBLpwGUyXLwxn9_-nu3s0,215
|
|
9
|
+
sempy_labs/_list_functions.py,sha256=Gc49_Q5e9CmdsVntvq0wFv1qdGInhhejxcvn-Vokyds,80791
|
|
10
|
+
sempy_labs/_model_auto_build.py,sha256=SGz3ASfKJBpYdgFRPoofiU7kpsjPDBkeB3qCntKb6qs,5083
|
|
11
|
+
sempy_labs/_model_bpa.py,sha256=xEp5LdqoMLBsUDzGWnfNrFCwr9tz9-2QDSi0NwnvXFI,63191
|
|
12
|
+
sempy_labs/_model_dependencies.py,sha256=S8u0f7AAVL6Zk1Jm35EqbLsIEGD8KwBGc_kH1AlBI1A,12948
|
|
13
|
+
sempy_labs/_one_lake_integration.py,sha256=XlGKghnYtXIprUdUI5fQj0ddshxE_AvUowb9YIrL2CE,6184
|
|
14
|
+
sempy_labs/_query_scale_out.py,sha256=xkyCFz7vchxB6c6cMIhZXUWifOduanSbv1KGQbkmVls,15214
|
|
15
|
+
sempy_labs/_refresh_semantic_model.py,sha256=bTVUNEdQiJDxQh1T0g4aYU3-VHaN9__qFAOYbx85-O0,6622
|
|
16
|
+
sempy_labs/_translations.py,sha256=sH_-W8vdQ632lY68RxM5a2lCmy4MRDFXLxjvMuw4xQg,18054
|
|
17
|
+
sempy_labs/_vertipaq.py,sha256=04hQ-A4wuW6gi4zIccmVoDDByk34Rp-QTkRXSo6XLfI,33266
|
|
18
|
+
sempy_labs/directlake/__init__.py,sha256=HbfHvDvGE4H-xSbV6JO7TUb4HoLGJf2AeuqeQxIuuJ4,1689
|
|
19
|
+
sempy_labs/directlake/_directlake_schema_compare.py,sha256=axqSYQHMs3zAhFAF3DMNdm2SK1k-95j_Zh-pZZXv8SQ,4636
|
|
20
|
+
sempy_labs/directlake/_directlake_schema_sync.py,sha256=EMDPAVn53EN1PM769AxKspb_cVDcCazz4kHMKvWqJMQ,5119
|
|
21
|
+
sempy_labs/directlake/_fallback.py,sha256=2dX5MRU2d04_jA799TaPezSQJWW-NrflDtdURge6Ceo,1995
|
|
22
|
+
sempy_labs/directlake/_get_directlake_lakehouse.py,sha256=vEY1QBU7gAdoVQIGl59T_1TTYuXbHgk3pZA41EkVkl8,2358
|
|
23
|
+
sempy_labs/directlake/_get_shared_expression.py,sha256=ngZCnoOjj278n6Yql7TVZ36z89HDej9JSRuoRWALxDs,1926
|
|
24
|
+
sempy_labs/directlake/_guardrails.py,sha256=iReOycR6caBuBWpEOIpicZS_7kkSGJLTUDf2Ch0QUCE,2280
|
|
25
|
+
sempy_labs/directlake/_list_directlake_model_calc_tables.py,sha256=C6C8WycGV4NQOQifh3XexBZx9hm30-Ac56sCo4MfqgI,2082
|
|
26
|
+
sempy_labs/directlake/_show_unsupported_directlake_objects.py,sha256=oumzh7mXghkFFqFDDK7bBmNMRWnOZeE1DxuMsEPSBzo,3365
|
|
27
|
+
sempy_labs/directlake/_update_directlake_model_lakehouse_connection.py,sha256=lY4JlbKwow664sXK4cZ22PFTy9Gw79R-6TYx36uz8SY,3183
|
|
28
|
+
sempy_labs/directlake/_update_directlake_partition_entity.py,sha256=KzUvP4r0-UOylXgM46cVKoUlbIFEpxO3ThMQl9JU-Gw,3140
|
|
29
|
+
sempy_labs/directlake/_warm_cache.py,sha256=codsJhifbn8yO8Cjo40syRtFnIArIvpWQ726WZ88ZsQ,8211
|
|
30
|
+
sempy_labs/lakehouse/__init__.py,sha256=i6VRx4dR1SIN-1GxioiNwhC4FxbozRCIz5TfXjb9rKc,587
|
|
31
|
+
sempy_labs/lakehouse/_get_lakehouse_columns.py,sha256=QGmuH3uFuxv_mIjm4HTXaX-s1UFb9BbHnaCx9kTHgy8,2594
|
|
32
|
+
sempy_labs/lakehouse/_get_lakehouse_tables.py,sha256=g_kKtntm5uPbIxNrunOVhzuKvJfC-9g5awxul2KSmUI,9085
|
|
33
|
+
sempy_labs/lakehouse/_lakehouse.py,sha256=ovm5S4hk3aLChFCzUGWIJmL3wJ47wRycoh0mbBd8pRA,2774
|
|
34
|
+
sempy_labs/lakehouse/_shortcuts.py,sha256=voKile93krzWK7ccmKVk2_fv1lioyAq5rk8YrJ6qy6k,6895
|
|
35
|
+
sempy_labs/migration/__init__.py,sha256=l5v8pC2INdNwbAKVmvWpuVxs6lpb6omim_4BPOmNo4E,1042
|
|
36
|
+
sempy_labs/migration/_create_pqt_file.py,sha256=PGC21nhIsE9TKyz110tYaf4Nle6-eV1xqvBOqUSEDQY,8986
|
|
37
|
+
sempy_labs/migration/_migrate_calctables_to_lakehouse.py,sha256=ajFvHauFdEsbgxvn9JXj2kiXaRtJLEjwX4hWQG7FQy0,20609
|
|
38
|
+
sempy_labs/migration/_migrate_calctables_to_semantic_model.py,sha256=Co2f579vSwkWZ95SlBStS-XJ73YwgcdfAMlJbUv_pkk,6343
|
|
39
|
+
sempy_labs/migration/_migrate_model_objects_to_semantic_model.py,sha256=q4oYBDHXvcTExeufmcOOeK3jv_9A2Xef5ksyEwG0PfA,23801
|
|
40
|
+
sempy_labs/migration/_migrate_tables_columns_to_semantic_model.py,sha256=qohJC6ARjM8NiMH7nZEKqUEXMrh-IqhdeUzgrBNZ1DQ,7028
|
|
41
|
+
sempy_labs/migration/_migration_validation.py,sha256=R7xz_OFmYRO4fxFWxvdl_ORZQKzqflsSBnGapmS067c,2508
|
|
42
|
+
sempy_labs/migration/_refresh_calc_tables.py,sha256=VumJaTiA3bTfm8jWwyIl7gxW4-a7W_3auGjWRcvd65g,6043
|
|
43
|
+
sempy_labs/report/__init__.py,sha256=fkjbkAXZuH7VnAn-k3iB4dngWZKaX-k0bxS6mBa9iAs,846
|
|
44
|
+
sempy_labs/report/_generate_report.py,sha256=uFbTO_7GSItt5wM_yZrKybEQ3kQoqXKbY1bGzlOW5c0,8502
|
|
45
|
+
sempy_labs/report/_report_functions.py,sha256=5rzYaS_8Gj9FEyCgysWIsLxiE-DaCgPNArI9YIe4Jvo,29515
|
|
46
|
+
sempy_labs/report/_report_rebind.py,sha256=hsXXOl6C06hH-SU_TaKBInjIXHf-uRZru7uCHpbRJYA,4756
|
|
47
|
+
sempy_labs/tom/__init__.py,sha256=hFwkmWk5AZ7GK1LWqoqaK1g4gDmu9mZMkfLQvLsR_eE,130
|
|
48
|
+
sempy_labs/tom/_model.py,sha256=a3EXOj_yXxl0aQQfWa5bolC2q-KdCgQMF8CrrQ6aPrU,136859
|
|
49
|
+
semantic_link_labs-0.5.0.dist-info/LICENSE,sha256=ws_MuBL-SCEBqPBFl9_FqZkaaydIJmxHrJG2parhU4M,1141
|
|
50
|
+
semantic_link_labs-0.5.0.dist-info/METADATA,sha256=TINv_gW59FHx2uvyUwYSH9MxKls6eO7XYji03ab-kYk,764
|
|
51
|
+
semantic_link_labs-0.5.0.dist-info/WHEEL,sha256=y4mX-SOX4fYIkonsAGA5N0Oy-8_gI4FXw5HNI1xqvWg,91
|
|
52
|
+
semantic_link_labs-0.5.0.dist-info/top_level.txt,sha256=kiQX1y42Dbein1l3Q8jMUYyRulDjdlc2tMepvtrvixQ,11
|
|
53
|
+
semantic_link_labs-0.5.0.dist-info/RECORD,,
|
sempy_labs/__init__.py
CHANGED
|
@@ -5,7 +5,7 @@ from sempy_labs._clear_cache import clear_cache
|
|
|
5
5
|
# create_connection_vnet,
|
|
6
6
|
# create_connection_on_prem
|
|
7
7
|
# )
|
|
8
|
-
from sempy_labs._dax import
|
|
8
|
+
from sempy_labs._dax import evaluate_dax_impersonation
|
|
9
9
|
from sempy_labs._generate_semantic_model import (
|
|
10
10
|
create_blank_semantic_model,
|
|
11
11
|
create_semantic_model_from_bim,
|
|
@@ -13,6 +13,8 @@ from sempy_labs._generate_semantic_model import (
|
|
|
13
13
|
get_semantic_model_bim,
|
|
14
14
|
)
|
|
15
15
|
from sempy_labs._list_functions import (
|
|
16
|
+
list_semantic_model_objects,
|
|
17
|
+
list_shortcuts,
|
|
16
18
|
get_object_level_security,
|
|
17
19
|
# list_annotations,
|
|
18
20
|
# list_columns,
|
|
@@ -32,9 +34,21 @@ from sempy_labs._list_functions import (
|
|
|
32
34
|
# list_sqlendpoints,
|
|
33
35
|
# list_tables,
|
|
34
36
|
list_warehouses,
|
|
35
|
-
|
|
37
|
+
list_workspace_role_assignments,
|
|
36
38
|
create_warehouse,
|
|
37
39
|
update_item,
|
|
40
|
+
list_custom_pools,
|
|
41
|
+
create_custom_pool,
|
|
42
|
+
update_custom_pool,
|
|
43
|
+
assign_workspace_to_capacity,
|
|
44
|
+
unassign_workspace_from_capacity,
|
|
45
|
+
get_spark_settings,
|
|
46
|
+
update_spark_settings,
|
|
47
|
+
add_user_to_workspace,
|
|
48
|
+
delete_user_from_workspace,
|
|
49
|
+
update_workspace_user,
|
|
50
|
+
list_workspace_users,
|
|
51
|
+
assign_workspace_to_dataflow_storage,
|
|
38
52
|
)
|
|
39
53
|
|
|
40
54
|
from sempy_labs._helper_functions import (
|
|
@@ -50,9 +64,9 @@ from sempy_labs._helper_functions import (
|
|
|
50
64
|
resolve_dataset_name,
|
|
51
65
|
resolve_report_id,
|
|
52
66
|
resolve_report_name,
|
|
53
|
-
#
|
|
67
|
+
# language_validate
|
|
54
68
|
)
|
|
55
|
-
from sempy_labs._model_auto_build import model_auto_build
|
|
69
|
+
# from sempy_labs._model_auto_build import model_auto_build
|
|
56
70
|
from sempy_labs._model_bpa import model_bpa_rules, run_model_bpa
|
|
57
71
|
from sempy_labs._model_dependencies import (
|
|
58
72
|
measure_dependency_tree,
|
|
@@ -62,16 +76,15 @@ from sempy_labs._model_dependencies import (
|
|
|
62
76
|
from sempy_labs._one_lake_integration import (
|
|
63
77
|
export_model_to_onelake,
|
|
64
78
|
)
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
# )
|
|
79
|
+
from sempy_labs._query_scale_out import (
|
|
80
|
+
qso_sync,
|
|
81
|
+
qso_sync_status,
|
|
82
|
+
set_qso,
|
|
83
|
+
list_qso_settings,
|
|
84
|
+
disable_qso,
|
|
85
|
+
set_semantic_model_storage_format,
|
|
86
|
+
set_workspace_default_storage_format,
|
|
87
|
+
)
|
|
75
88
|
from sempy_labs._refresh_semantic_model import (
|
|
76
89
|
refresh_semantic_model,
|
|
77
90
|
cancel_dataset_refresh,
|
|
@@ -82,14 +95,13 @@ from sempy_labs._vertipaq import (
|
|
|
82
95
|
# visualize_vertipaq,
|
|
83
96
|
import_vertipaq_analyzer,
|
|
84
97
|
)
|
|
85
|
-
from sempy_labs._tom import TOMWrapper, connect_semantic_model
|
|
86
98
|
|
|
87
99
|
__all__ = [
|
|
88
100
|
"clear_cache",
|
|
89
101
|
# create_connection_cloud,
|
|
90
102
|
# create_connection_vnet,
|
|
91
103
|
# create_connection_on_prem,
|
|
92
|
-
"
|
|
104
|
+
"evaluate_dax_impersonation",
|
|
93
105
|
"create_blank_semantic_model",
|
|
94
106
|
"create_semantic_model_from_bim",
|
|
95
107
|
#'deploy_semantic_model',
|
|
@@ -113,7 +125,7 @@ __all__ = [
|
|
|
113
125
|
#'list_sqlendpoints',
|
|
114
126
|
#'list_tables',
|
|
115
127
|
"list_warehouses",
|
|
116
|
-
|
|
128
|
+
'list_workspace_role_assignments',
|
|
117
129
|
"create_warehouse",
|
|
118
130
|
"update_item",
|
|
119
131
|
"create_abfss_path",
|
|
@@ -129,26 +141,38 @@ __all__ = [
|
|
|
129
141
|
"resolve_report_id",
|
|
130
142
|
"resolve_report_name",
|
|
131
143
|
#'language_validate',
|
|
132
|
-
"model_auto_build",
|
|
144
|
+
#"model_auto_build",
|
|
133
145
|
"model_bpa_rules",
|
|
134
146
|
"run_model_bpa",
|
|
135
147
|
"measure_dependency_tree",
|
|
136
148
|
"get_measure_dependencies",
|
|
137
149
|
"get_model_calc_dependencies",
|
|
138
150
|
"export_model_to_onelake",
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
151
|
+
'qso_sync',
|
|
152
|
+
'qso_sync_status',
|
|
153
|
+
'set_qso',
|
|
154
|
+
'list_qso_settings',
|
|
155
|
+
'disable_qso',
|
|
156
|
+
'set_semantic_model_storage_format',
|
|
157
|
+
'set_workspace_default_storage_format',
|
|
146
158
|
"refresh_semantic_model",
|
|
147
159
|
"cancel_dataset_refresh",
|
|
148
160
|
"translate_semantic_model",
|
|
149
161
|
"vertipaq_analyzer",
|
|
150
162
|
#'visualize_vertipaq',
|
|
151
163
|
"import_vertipaq_analyzer",
|
|
152
|
-
"
|
|
153
|
-
"
|
|
164
|
+
"list_semantic_model_objects",
|
|
165
|
+
"list_shortcuts",
|
|
166
|
+
"list_custom_pools",
|
|
167
|
+
"create_custom_pool",
|
|
168
|
+
"update_custom_pool",
|
|
169
|
+
"assign_workspace_to_capacity",
|
|
170
|
+
"unassign_workspace_from_capacity",
|
|
171
|
+
"get_spark_settings",
|
|
172
|
+
"update_spark_settings",
|
|
173
|
+
"add_user_to_workspace",
|
|
174
|
+
"delete_user_from_workspace",
|
|
175
|
+
"update_workspace_user",
|
|
176
|
+
"list_workspace_users",
|
|
177
|
+
"assign_workspace_to_dataflow_storage"
|
|
154
178
|
]
|
sempy_labs/_ai.py
CHANGED
|
@@ -6,6 +6,7 @@ from pyspark.sql.functions import col
|
|
|
6
6
|
from pyspark.sql import SparkSession
|
|
7
7
|
from typing import List, Optional, Union
|
|
8
8
|
from IPython.display import display
|
|
9
|
+
import sempy_labs._icons as icons
|
|
9
10
|
|
|
10
11
|
|
|
11
12
|
def optimize_semantic_model(dataset: str, workspace: Optional[str] = None):
|
|
@@ -13,6 +14,7 @@ def optimize_semantic_model(dataset: str, workspace: Optional[str] = None):
|
|
|
13
14
|
from ._model_bpa import run_model_bpa
|
|
14
15
|
from .directlake._fallback import check_fallback_reason
|
|
15
16
|
from ._helper_functions import format_dax_object_name
|
|
17
|
+
from sempy_labs.tom import connect_semantic_model
|
|
16
18
|
|
|
17
19
|
modelBPA = run_model_bpa(
|
|
18
20
|
dataset=dataset, workspace=workspace, return_dataframe=True
|
|
@@ -39,7 +41,7 @@ def optimize_semantic_model(dataset: str, workspace: Optional[str] = None):
|
|
|
39
41
|
|
|
40
42
|
if len(fallback_filt) > 0:
|
|
41
43
|
print(
|
|
42
|
-
f"The '{dataset}' semantic model is a Direct Lake semantic model which contains views. Since views always fall back to DirectQuery, it is recommended to only use lakehouse tables and not views."
|
|
44
|
+
f"{icons.yellow_dot} The '{dataset}' semantic model is a Direct Lake semantic model which contains views. Since views always fall back to DirectQuery, it is recommended to only use lakehouse tables and not views."
|
|
43
45
|
)
|
|
44
46
|
|
|
45
47
|
# Potential model reduction estimate
|
|
@@ -56,11 +58,11 @@ def optimize_semantic_model(dataset: str, workspace: Optional[str] = None):
|
|
|
56
58
|
totSize = df["Total Size"].sum()
|
|
57
59
|
if len(df_filt) > 0:
|
|
58
60
|
print(
|
|
59
|
-
f"Potential savings of {totSize} bytes from following the '{rule}' rule."
|
|
61
|
+
f"{icons.yellow_dot} Potential savings of {totSize} bytes from following the '{rule}' rule."
|
|
60
62
|
)
|
|
61
63
|
display(df_filt)
|
|
62
64
|
else:
|
|
63
|
-
print(f"The '{rule}' rule has been followed.")
|
|
65
|
+
print(f"{icons.green_dot} The '{rule}' rule has been followed.")
|
|
64
66
|
|
|
65
67
|
|
|
66
68
|
def generate_measure_descriptions(
|
|
@@ -77,10 +79,7 @@ def generate_measure_descriptions(
|
|
|
77
79
|
|
|
78
80
|
validModels = ["gpt-35-turbo", "gpt-35-turbo-16k", "gpt-4"]
|
|
79
81
|
if gpt_model not in validModels:
|
|
80
|
-
|
|
81
|
-
f"The '{gpt_model}' model is not a valid model. Enter a gpt_model from this list: {validModels}."
|
|
82
|
-
)
|
|
83
|
-
return
|
|
82
|
+
raise ValueError(f"{icons.red_dot} The '{gpt_model}' model is not a valid model. Enter a gpt_model from this list: {validModels}.")
|
|
84
83
|
|
|
85
84
|
dfM = fabric.list_measures(dataset=dataset, workspace=workspace)
|
|
86
85
|
|
|
@@ -92,7 +91,7 @@ def generate_measure_descriptions(
|
|
|
92
91
|
df = dfM_filt[["Table Name", "Measure Name", "Measure Expression"]]
|
|
93
92
|
|
|
94
93
|
df["prompt"] = (
|
|
95
|
-
|
|
94
|
+
"The following is DAX code used by Microsoft Power BI. Please explain this code in simple terms:"
|
|
96
95
|
+ df["Measure Expression"]
|
|
97
96
|
)
|
|
98
97
|
|
|
@@ -115,8 +114,8 @@ def generate_measure_descriptions(
|
|
|
115
114
|
)
|
|
116
115
|
|
|
117
116
|
# Update the model to use the new descriptions
|
|
118
|
-
|
|
119
|
-
|
|
117
|
+
#with connect_semantic_model(dataset=dataset, workspace=workspace, readonly=False) as tom:
|
|
118
|
+
|
|
120
119
|
|
|
121
120
|
# for t in m.Tables:
|
|
122
121
|
# tName = t.Name
|
|
@@ -152,11 +151,11 @@ def generate_aggs(
|
|
|
152
151
|
#'OrderDateKey': 'GroupBy'
|
|
153
152
|
# }
|
|
154
153
|
|
|
155
|
-
if workspace
|
|
154
|
+
if workspace is None:
|
|
156
155
|
workspace_id = fabric.get_workspace_id()
|
|
157
156
|
workspace = fabric.resolve_workspace_name(workspace_id)
|
|
158
157
|
|
|
159
|
-
if lakehouse_workspace
|
|
158
|
+
if lakehouse_workspace is None:
|
|
160
159
|
lakehouse_workspace = workspace
|
|
161
160
|
lakehouse_workspace_id = workspace_id
|
|
162
161
|
else:
|
|
@@ -172,48 +171,33 @@ def generate_aggs(
|
|
|
172
171
|
numericTypes = ["Int64", "Double", "Decimal"]
|
|
173
172
|
|
|
174
173
|
if any(value not in aggTypes for value in columns.values()):
|
|
175
|
-
|
|
176
|
-
f"Invalid aggregation type(s) have been specified in the 'columns' parameter. Valid aggregation types: {aggTypes}."
|
|
177
|
-
)
|
|
178
|
-
return
|
|
174
|
+
raise ValueError(f"{icons.red_dot} Invalid aggregation type(s) have been specified in the 'columns' parameter. Valid aggregation types: {aggTypes}.")
|
|
179
175
|
|
|
180
176
|
dfC = fabric.list_columns(dataset=dataset, workspace=workspace)
|
|
181
177
|
dfP = fabric.list_partitions(dataset=dataset, workspace=workspace)
|
|
182
178
|
dfM = fabric.list_measures(dataset=dataset, workspace=workspace)
|
|
183
179
|
dfR = fabric.list_relationships(dataset=dataset, workspace=workspace)
|
|
184
180
|
if not any(r["Mode"] == "DirectLake" for i, r in dfP.iterrows()):
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
)
|
|
188
|
-
return
|
|
189
|
-
|
|
181
|
+
raise ValueError(f"{icons.red_dot} The '{dataset}' semantic model within the '{workspace}' workspace is not in Direct Lake mode. This function is only relevant for Direct Lake semantic models.")
|
|
182
|
+
|
|
190
183
|
dfC_filtT = dfC[dfC["Table Name"] == table_name]
|
|
191
184
|
|
|
192
185
|
if len(dfC_filtT) == 0:
|
|
193
|
-
|
|
194
|
-
f"The '{table_name}' table does not exist in the '{dataset}' semantic model within the '{workspace}' workspace."
|
|
195
|
-
)
|
|
196
|
-
return
|
|
186
|
+
raise ValueError(f"{icons.red_dot} The '{table_name}' table does not exist in the '{dataset}' semantic model within the '{workspace}' workspace.")
|
|
197
187
|
|
|
198
188
|
dfC_filt = dfC[
|
|
199
189
|
(dfC["Table Name"] == table_name) & (dfC["Column Name"].isin(columnValues))
|
|
200
190
|
]
|
|
201
191
|
|
|
202
192
|
if len(columns) != len(dfC_filt):
|
|
203
|
-
|
|
204
|
-
f"Columns listed in '{columnValues}' do not exist in the '{table_name}' table in the '{dataset}' semantic model within the '{workspace}' workspace."
|
|
205
|
-
)
|
|
206
|
-
return
|
|
193
|
+
raise ValueError(f"{icons.red_dot} Columns listed in '{columnValues}' do not exist in the '{table_name}' table in the '{dataset}' semantic model within the '{workspace}' workspace.")
|
|
207
194
|
|
|
208
195
|
# Check if doing sum/count/min/max etc. on a non-number column
|
|
209
196
|
for col, agg in columns.items():
|
|
210
197
|
dfC_col = dfC_filt[dfC_filt["Column Name"] == col]
|
|
211
198
|
dataType = dfC_col["Data Type"].iloc[0]
|
|
212
199
|
if agg in aggTypesAggregate and dataType not in numericTypes:
|
|
213
|
-
|
|
214
|
-
f"The '{col}' column in the '{table_name}' table is of '{dataType}' data type. Only columns of '{numericTypes}' data types can be aggregated as '{aggTypesAggregate}' aggregation types."
|
|
215
|
-
)
|
|
216
|
-
return
|
|
200
|
+
raise ValueError(f"{icons.red_dot} The '{col}' column in the '{table_name}' table is of '{dataType}' data type. Only columns of '{numericTypes}' data types can be aggregated as '{aggTypesAggregate}' aggregation types.")
|
|
217
201
|
|
|
218
202
|
# Create/update lakehouse delta agg table
|
|
219
203
|
aggSuffix = "_agg"
|
|
@@ -229,10 +213,7 @@ def generate_aggs(
|
|
|
229
213
|
dfI_filt = dfI[(dfI["Id"] == sqlEndpointId)]
|
|
230
214
|
|
|
231
215
|
if len(dfI_filt) == 0:
|
|
232
|
-
|
|
233
|
-
f"The lakehouse (SQL Endpoint) used by the '{dataset}' semantic model does not reside in the '{lakehouse_workspace}' workspace. Please update the lakehouse_workspace parameter."
|
|
234
|
-
)
|
|
235
|
-
return
|
|
216
|
+
raise ValueError(f"{icons.red_dot} The lakehouse (SQL Endpoint) used by the '{dataset}' semantic model does not reside in the '{lakehouse_workspace}' workspace. Please update the lakehouse_workspace parameter.")
|
|
236
217
|
|
|
237
218
|
lakehouseName = dfI_filt["Display Name"].iloc[0]
|
|
238
219
|
lakehouse_id = resolve_lakehouse_id(
|
|
@@ -278,16 +259,16 @@ def generate_aggs(
|
|
|
278
259
|
delta_table_name=aggLakeTName,
|
|
279
260
|
)
|
|
280
261
|
spark_df.write.mode("overwrite").format("delta").save(aggFilePath)
|
|
281
|
-
f"The '{aggLakeTName}' table has been created/updated in the lakehouse."
|
|
262
|
+
f"{icons.green_dot} The '{aggLakeTName}' table has been created/updated in the lakehouse."
|
|
282
263
|
|
|
283
264
|
# Create/update semantic model agg table
|
|
284
265
|
tom_server = fabric.create_tom_server(readonly=False, workspace=workspace)
|
|
285
266
|
m = tom_server.Databases.GetByName(dataset).Model
|
|
286
|
-
f"\
|
|
267
|
+
print(f"\n{icons.in_progress} Updating the '{dataset}' semantic model...")
|
|
287
268
|
dfC_agg = dfC[dfC["Table Name"] == aggTableName]
|
|
288
269
|
|
|
289
270
|
if len(dfC_agg) == 0:
|
|
290
|
-
print(f"Creating the '{aggTableName}' table...")
|
|
271
|
+
print(f"{icons.in_progress} Creating the '{aggTableName}' table...")
|
|
291
272
|
exp = m.Expressions["DatabaseQuery"]
|
|
292
273
|
tbl = TOM.Table()
|
|
293
274
|
tbl.Name = aggTableName
|
|
@@ -318,15 +299,15 @@ def generate_aggs(
|
|
|
318
299
|
|
|
319
300
|
tbl.Columns.Add(col)
|
|
320
301
|
print(
|
|
321
|
-
f"The '{aggTableName}'[{cName}] column has been added to the '{dataset}' semantic model."
|
|
302
|
+
f"{icons.green_dot} The '{aggTableName}'[{cName}] column has been added to the '{dataset}' semantic model."
|
|
322
303
|
)
|
|
323
304
|
|
|
324
305
|
m.Tables.Add(tbl)
|
|
325
306
|
print(
|
|
326
|
-
f"The '{aggTableName}' table has been added to the '{dataset}' semantic model."
|
|
307
|
+
f"{icons.green_dot} The '{aggTableName}' table has been added to the '{dataset}' semantic model."
|
|
327
308
|
)
|
|
328
309
|
else:
|
|
329
|
-
print(f"Updating the '{aggTableName}' table's columns...")
|
|
310
|
+
print(f"{icons.in_progress} Updating the '{aggTableName}' table's columns...")
|
|
330
311
|
# Remove existing columns
|
|
331
312
|
for t in m.Tables:
|
|
332
313
|
tName = t.Name
|
|
@@ -347,12 +328,12 @@ def generate_aggs(
|
|
|
347
328
|
col.DataType = System.Enum.Parse(TOM.DataType, dType)
|
|
348
329
|
|
|
349
330
|
m.Tables[aggTableName].Columns.Add(col)
|
|
350
|
-
print(f"The '{aggTableName}'[{cName}] column has been added.")
|
|
331
|
+
print(f"{icons.green_dot} The '{aggTableName}'[{cName}] column has been added.")
|
|
351
332
|
|
|
352
333
|
# Create relationships
|
|
353
334
|
relMap = {"m": "Many", "1": "One", "0": "None"}
|
|
354
335
|
|
|
355
|
-
print(f"\
|
|
336
|
+
print(f"\n{icons.in_progress} Generating necessary relationships...")
|
|
356
337
|
for i, r in dfR.iterrows():
|
|
357
338
|
fromTable = r["From Table"]
|
|
358
339
|
fromColumn = r["From Column"]
|
|
@@ -384,27 +365,27 @@ def generate_aggs(
|
|
|
384
365
|
rel.FromColumn = m.Tables[aggTableName].Columns[fromColumn]
|
|
385
366
|
m.Relationships.Add(rel)
|
|
386
367
|
print(
|
|
387
|
-
f"'{aggTableName}'[{fromColumn}] -> '{toTable}'[{toColumn}] relationship has been added."
|
|
368
|
+
f"{icons.green_dot} '{aggTableName}'[{fromColumn}] -> '{toTable}'[{toColumn}] relationship has been added."
|
|
388
369
|
)
|
|
389
370
|
except:
|
|
390
371
|
print(
|
|
391
|
-
f"'{aggTableName}'[{fromColumn}] -> '{toTable}'[{toColumn}] relationship has not been created."
|
|
372
|
+
f"{icons.red_dot} '{aggTableName}'[{fromColumn}] -> '{toTable}'[{toColumn}] relationship has not been created."
|
|
392
373
|
)
|
|
393
374
|
elif toTable == table_name:
|
|
394
375
|
try:
|
|
395
376
|
rel.ToColumn = m.Tables[aggTableName].Columns[toColumn]
|
|
396
377
|
m.Relationships.Add(rel)
|
|
397
378
|
print(
|
|
398
|
-
f"'{fromTable}'[{fromColumn}] -> '{aggTableName}'[{toColumn}] relationship has been added."
|
|
379
|
+
f"{icons.green_dot} '{fromTable}'[{fromColumn}] -> '{aggTableName}'[{toColumn}] relationship has been added."
|
|
399
380
|
)
|
|
400
381
|
except:
|
|
401
382
|
print(
|
|
402
|
-
f"'{fromTable}'[{fromColumn}] -> '{aggTableName}'[{toColumn}] relationship has not been created."
|
|
383
|
+
f"{icons.red_dot} '{fromTable}'[{fromColumn}] -> '{aggTableName}'[{toColumn}] relationship has not been created."
|
|
403
384
|
)
|
|
404
385
|
f"Relationship creation is complete."
|
|
405
386
|
|
|
406
387
|
# Create IF measure
|
|
407
|
-
f"\
|
|
388
|
+
f"\n{icons.in_progress} Creating measure to check if the agg table can be used..."
|
|
408
389
|
aggChecker = "IF("
|
|
409
390
|
dfR_filt = dfR[
|
|
410
391
|
(dfR["From Table"] == table_name) & (~dfR["From Column"].isin(columnValues))
|
|
@@ -419,7 +400,7 @@ def generate_aggs(
|
|
|
419
400
|
print(aggChecker)
|
|
420
401
|
|
|
421
402
|
# Todo: add IFISFILTERED clause for columns
|
|
422
|
-
f"\n Creating the base measures in the agg table..."
|
|
403
|
+
f"\n{icons.in_progress} Creating the base measures in the agg table..."
|
|
423
404
|
# Create base agg measures
|
|
424
405
|
dep = fabric.evaluate_dax(
|
|
425
406
|
dataset=dataset,
|
sempy_labs/_clear_cache.py
CHANGED
|
@@ -8,6 +8,7 @@ import sempy_labs._icons as icons
|
|
|
8
8
|
def clear_cache(dataset: str, workspace: Optional[str] = None):
|
|
9
9
|
"""
|
|
10
10
|
Clears the cache of a semantic model.
|
|
11
|
+
See `here <https://learn.microsoft.com/analysis-services/instances/clear-the-analysis-services-caches?view=asallproducts-allversions>`_ for documentation.
|
|
11
12
|
|
|
12
13
|
Parameters
|
|
13
14
|
----------
|
|
@@ -19,9 +20,7 @@ def clear_cache(dataset: str, workspace: Optional[str] = None):
|
|
|
19
20
|
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
20
21
|
"""
|
|
21
22
|
|
|
22
|
-
|
|
23
|
-
workspace_id = fabric.get_workspace_id()
|
|
24
|
-
workspace = fabric.resolve_workspace_name(workspace_id)
|
|
23
|
+
workspace = fabric.resolve_workspace_name(workspace)
|
|
25
24
|
|
|
26
25
|
datasetID = resolve_dataset_id(dataset=dataset, workspace=workspace)
|
|
27
26
|
|
sempy_labs/_connections.py
CHANGED
|
@@ -2,6 +2,7 @@ import sempy
|
|
|
2
2
|
import sempy.fabric as fabric
|
|
3
3
|
import pandas as pd
|
|
4
4
|
from typing import List, Optional, Union
|
|
5
|
+
import sempy_labs._icons as icons
|
|
5
6
|
|
|
6
7
|
|
|
7
8
|
def create_connection_cloud(
|
|
@@ -11,7 +12,7 @@ def create_connection_cloud(
|
|
|
11
12
|
user_name: str,
|
|
12
13
|
password: str,
|
|
13
14
|
privacy_level: str,
|
|
14
|
-
):
|
|
15
|
+
) -> pd.DataFrame:
|
|
15
16
|
|
|
16
17
|
# https://review.learn.microsoft.com/en-us/rest/api/fabric/core/connections/create-connection?branch=features%2Fdmts&tabs=HTTP
|
|
17
18
|
|
|
@@ -60,16 +61,16 @@ def create_connection_cloud(
|
|
|
60
61
|
if response.status_code == 200:
|
|
61
62
|
o = response.json()
|
|
62
63
|
new_data = {
|
|
63
|
-
"Connection Id": o
|
|
64
|
-
"Connection Name": o
|
|
65
|
-
"Connectivity Type": o
|
|
66
|
-
"Connection Type": o
|
|
67
|
-
"Connection Path": o
|
|
68
|
-
"Privacy Level": o
|
|
69
|
-
"Credential Type": o
|
|
70
|
-
"Single Sign On Type": o
|
|
71
|
-
"Connection Encryption": o
|
|
72
|
-
"Skip Test Connection": o
|
|
64
|
+
"Connection Id": o.get("id"),
|
|
65
|
+
"Connection Name": o.get("name"),
|
|
66
|
+
"Connectivity Type": o.get("connectivityType"),
|
|
67
|
+
"Connection Type": o.get("connectionDetails",{}).get("type"),
|
|
68
|
+
"Connection Path": o.get("connectionDetails",{}).get("path"),
|
|
69
|
+
"Privacy Level": o.get("privacyLevel"),
|
|
70
|
+
"Credential Type": o.get("credentialDetails",{}).get("credentialType"),
|
|
71
|
+
"Single Sign On Type": o.get("credentialDetails",{}).get("singleSignOnType"),
|
|
72
|
+
"Connection Encryption": o.get("credentialDetails",{}).get("connectionEncryption"),
|
|
73
|
+
"Skip Test Connection": o.get("credentialDetails",{}).get("skipTestConnection"),
|
|
73
74
|
}
|
|
74
75
|
df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
|
|
75
76
|
|
|
@@ -77,7 +78,7 @@ def create_connection_cloud(
|
|
|
77
78
|
|
|
78
79
|
return df
|
|
79
80
|
else:
|
|
80
|
-
print(response.status_code)
|
|
81
|
+
print(f"{icons.red_dot} {response.status_code}")
|
|
81
82
|
|
|
82
83
|
|
|
83
84
|
def create_connection_on_prem(
|
|
@@ -87,7 +88,7 @@ def create_connection_on_prem(
|
|
|
87
88
|
database_name: str,
|
|
88
89
|
credentials: str,
|
|
89
90
|
privacy_level: str,
|
|
90
|
-
):
|
|
91
|
+
) -> pd.DataFrame:
|
|
91
92
|
|
|
92
93
|
df = pd.DataFrame(
|
|
93
94
|
columns=[
|
|
@@ -135,17 +136,17 @@ def create_connection_on_prem(
|
|
|
135
136
|
if response.status_code == 200:
|
|
136
137
|
o = response.json()
|
|
137
138
|
new_data = {
|
|
138
|
-
"Connection Id": o
|
|
139
|
-
"Connection Name": o
|
|
140
|
-
"Gateway ID": o
|
|
141
|
-
"Connectivity Type": o
|
|
142
|
-
"Connection Type": o
|
|
143
|
-
"Connection Path": o
|
|
144
|
-
"Privacy Level": o
|
|
145
|
-
"Credential Type": o
|
|
146
|
-
"Single Sign On Type": o
|
|
147
|
-
"Connection Encryption": o
|
|
148
|
-
"Skip Test Connection": o
|
|
139
|
+
"Connection Id": o.get("id"),
|
|
140
|
+
"Connection Name": o.get("name"),
|
|
141
|
+
"Gateway ID": o.get("gatewayId"),
|
|
142
|
+
"Connectivity Type": o.get("connectivityType"),
|
|
143
|
+
"Connection Type": o.get("connectionDetails",{}).get("type"),
|
|
144
|
+
"Connection Path": o.get("connectionDetails",{}).get("path"),
|
|
145
|
+
"Privacy Level": o.get("privacyLevel"),
|
|
146
|
+
"Credential Type": o.get("credentialDetails",{}).get("credentialType"),
|
|
147
|
+
"Single Sign On Type": o.get("credentialDetails",{}).get("singleSignOnType"),
|
|
148
|
+
"Connection Encryption": o.get("credentialDetails",{}).get("connectionEncryption"),
|
|
149
|
+
"Skip Test Connection": o.get("credentialDetails",{}).get("skipTestConnection"),
|
|
149
150
|
}
|
|
150
151
|
df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
|
|
151
152
|
|
|
@@ -153,7 +154,7 @@ def create_connection_on_prem(
|
|
|
153
154
|
|
|
154
155
|
return df
|
|
155
156
|
else:
|
|
156
|
-
print(response.status_code)
|
|
157
|
+
print(f"{icons.red_dot} {response.status_code}")
|
|
157
158
|
|
|
158
159
|
|
|
159
160
|
def create_connection_vnet(
|
|
@@ -164,7 +165,7 @@ def create_connection_vnet(
|
|
|
164
165
|
user_name: str,
|
|
165
166
|
password: str,
|
|
166
167
|
privacy_level: str,
|
|
167
|
-
):
|
|
168
|
+
) -> pd.DataFrame:
|
|
168
169
|
|
|
169
170
|
df = pd.DataFrame(
|
|
170
171
|
columns=[
|
|
@@ -213,17 +214,17 @@ def create_connection_vnet(
|
|
|
213
214
|
if response.status_code == 200:
|
|
214
215
|
o = response.json()
|
|
215
216
|
new_data = {
|
|
216
|
-
"Connection Id": o
|
|
217
|
-
"Connection Name": o
|
|
218
|
-
"Gateway ID": o
|
|
219
|
-
"Connectivity Type": o
|
|
220
|
-
"Connection Type": o
|
|
221
|
-
"Connection Path": o
|
|
222
|
-
"Privacy Level": o
|
|
223
|
-
"Credential Type": o
|
|
224
|
-
"Single Sign On Type": o
|
|
225
|
-
"Connection Encryption": o
|
|
226
|
-
"Skip Test Connection": o
|
|
217
|
+
"Connection Id": o.get("id"),
|
|
218
|
+
"Connection Name": o.get("name"),
|
|
219
|
+
"Gateway ID": o.get("gatewayId"),
|
|
220
|
+
"Connectivity Type": o.get("connectivityType"),
|
|
221
|
+
"Connection Type": o.get("connectionDetails",{}).get("type"),
|
|
222
|
+
"Connection Path": o.get("connectionDetails",{}).get("path"),
|
|
223
|
+
"Privacy Level": o.get("privacyLevel"),
|
|
224
|
+
"Credential Type": o.get("credentialDetails",{}).get("credentialType"),
|
|
225
|
+
"Single Sign On Type": o.get("credentialDetails",{}).get("singleSignOnType"),
|
|
226
|
+
"Connection Encryption": o.get("credentialDetails",{}).get("connectionEncryption"),
|
|
227
|
+
"Skip Test Connection": o.get("credentialDetails",{}).get("skipTestConnection"),
|
|
227
228
|
}
|
|
228
229
|
df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
|
|
229
230
|
|
|
@@ -231,4 +232,4 @@ def create_connection_vnet(
|
|
|
231
232
|
|
|
232
233
|
return df
|
|
233
234
|
else:
|
|
234
|
-
print(response.status_code)
|
|
235
|
+
print(f"{icons.red_dot} {response.status_code}")
|