semantic-link-labs 0.12.8__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (243) hide show
  1. semantic_link_labs-0.12.8.dist-info/METADATA +354 -0
  2. semantic_link_labs-0.12.8.dist-info/RECORD +243 -0
  3. semantic_link_labs-0.12.8.dist-info/WHEEL +5 -0
  4. semantic_link_labs-0.12.8.dist-info/licenses/LICENSE +21 -0
  5. semantic_link_labs-0.12.8.dist-info/top_level.txt +1 -0
  6. sempy_labs/__init__.py +606 -0
  7. sempy_labs/_a_lib_info.py +2 -0
  8. sempy_labs/_ai.py +437 -0
  9. sempy_labs/_authentication.py +264 -0
  10. sempy_labs/_bpa_translation/_model/_translations_am-ET.po +869 -0
  11. sempy_labs/_bpa_translation/_model/_translations_ar-AE.po +908 -0
  12. sempy_labs/_bpa_translation/_model/_translations_bg-BG.po +968 -0
  13. sempy_labs/_bpa_translation/_model/_translations_ca-ES.po +963 -0
  14. sempy_labs/_bpa_translation/_model/_translations_cs-CZ.po +943 -0
  15. sempy_labs/_bpa_translation/_model/_translations_da-DK.po +945 -0
  16. sempy_labs/_bpa_translation/_model/_translations_de-DE.po +988 -0
  17. sempy_labs/_bpa_translation/_model/_translations_el-GR.po +993 -0
  18. sempy_labs/_bpa_translation/_model/_translations_es-ES.po +971 -0
  19. sempy_labs/_bpa_translation/_model/_translations_fa-IR.po +933 -0
  20. sempy_labs/_bpa_translation/_model/_translations_fi-FI.po +942 -0
  21. sempy_labs/_bpa_translation/_model/_translations_fr-FR.po +994 -0
  22. sempy_labs/_bpa_translation/_model/_translations_ga-IE.po +967 -0
  23. sempy_labs/_bpa_translation/_model/_translations_he-IL.po +902 -0
  24. sempy_labs/_bpa_translation/_model/_translations_hi-IN.po +944 -0
  25. sempy_labs/_bpa_translation/_model/_translations_hu-HU.po +963 -0
  26. sempy_labs/_bpa_translation/_model/_translations_id-ID.po +946 -0
  27. sempy_labs/_bpa_translation/_model/_translations_is-IS.po +939 -0
  28. sempy_labs/_bpa_translation/_model/_translations_it-IT.po +986 -0
  29. sempy_labs/_bpa_translation/_model/_translations_ja-JP.po +846 -0
  30. sempy_labs/_bpa_translation/_model/_translations_ko-KR.po +839 -0
  31. sempy_labs/_bpa_translation/_model/_translations_mt-MT.po +967 -0
  32. sempy_labs/_bpa_translation/_model/_translations_nl-NL.po +978 -0
  33. sempy_labs/_bpa_translation/_model/_translations_pl-PL.po +962 -0
  34. sempy_labs/_bpa_translation/_model/_translations_pt-BR.po +962 -0
  35. sempy_labs/_bpa_translation/_model/_translations_pt-PT.po +957 -0
  36. sempy_labs/_bpa_translation/_model/_translations_ro-RO.po +968 -0
  37. sempy_labs/_bpa_translation/_model/_translations_ru-RU.po +964 -0
  38. sempy_labs/_bpa_translation/_model/_translations_sk-SK.po +952 -0
  39. sempy_labs/_bpa_translation/_model/_translations_sl-SL.po +950 -0
  40. sempy_labs/_bpa_translation/_model/_translations_sv-SE.po +942 -0
  41. sempy_labs/_bpa_translation/_model/_translations_ta-IN.po +976 -0
  42. sempy_labs/_bpa_translation/_model/_translations_te-IN.po +947 -0
  43. sempy_labs/_bpa_translation/_model/_translations_th-TH.po +924 -0
  44. sempy_labs/_bpa_translation/_model/_translations_tr-TR.po +953 -0
  45. sempy_labs/_bpa_translation/_model/_translations_uk-UA.po +961 -0
  46. sempy_labs/_bpa_translation/_model/_translations_zh-CN.po +804 -0
  47. sempy_labs/_bpa_translation/_model/_translations_zu-ZA.po +969 -0
  48. sempy_labs/_capacities.py +1198 -0
  49. sempy_labs/_capacity_migration.py +660 -0
  50. sempy_labs/_clear_cache.py +351 -0
  51. sempy_labs/_connections.py +610 -0
  52. sempy_labs/_dashboards.py +69 -0
  53. sempy_labs/_data_access_security.py +98 -0
  54. sempy_labs/_data_pipelines.py +162 -0
  55. sempy_labs/_dataflows.py +668 -0
  56. sempy_labs/_dax.py +501 -0
  57. sempy_labs/_daxformatter.py +80 -0
  58. sempy_labs/_delta_analyzer.py +467 -0
  59. sempy_labs/_delta_analyzer_history.py +301 -0
  60. sempy_labs/_dictionary_diffs.py +221 -0
  61. sempy_labs/_documentation.py +147 -0
  62. sempy_labs/_domains.py +51 -0
  63. sempy_labs/_eventhouses.py +182 -0
  64. sempy_labs/_external_data_shares.py +230 -0
  65. sempy_labs/_gateways.py +521 -0
  66. sempy_labs/_generate_semantic_model.py +521 -0
  67. sempy_labs/_get_connection_string.py +84 -0
  68. sempy_labs/_git.py +543 -0
  69. sempy_labs/_graphQL.py +90 -0
  70. sempy_labs/_helper_functions.py +2833 -0
  71. sempy_labs/_icons.py +149 -0
  72. sempy_labs/_job_scheduler.py +609 -0
  73. sempy_labs/_kql_databases.py +149 -0
  74. sempy_labs/_kql_querysets.py +124 -0
  75. sempy_labs/_kusto.py +137 -0
  76. sempy_labs/_labels.py +124 -0
  77. sempy_labs/_list_functions.py +1720 -0
  78. sempy_labs/_managed_private_endpoints.py +253 -0
  79. sempy_labs/_mirrored_databases.py +416 -0
  80. sempy_labs/_mirrored_warehouses.py +60 -0
  81. sempy_labs/_ml_experiments.py +113 -0
  82. sempy_labs/_model_auto_build.py +140 -0
  83. sempy_labs/_model_bpa.py +557 -0
  84. sempy_labs/_model_bpa_bulk.py +378 -0
  85. sempy_labs/_model_bpa_rules.py +859 -0
  86. sempy_labs/_model_dependencies.py +343 -0
  87. sempy_labs/_mounted_data_factories.py +123 -0
  88. sempy_labs/_notebooks.py +441 -0
  89. sempy_labs/_one_lake_integration.py +151 -0
  90. sempy_labs/_onelake.py +131 -0
  91. sempy_labs/_query_scale_out.py +433 -0
  92. sempy_labs/_refresh_semantic_model.py +435 -0
  93. sempy_labs/_semantic_models.py +468 -0
  94. sempy_labs/_spark.py +455 -0
  95. sempy_labs/_sql.py +241 -0
  96. sempy_labs/_sql_audit_settings.py +207 -0
  97. sempy_labs/_sql_endpoints.py +214 -0
  98. sempy_labs/_tags.py +201 -0
  99. sempy_labs/_translations.py +43 -0
  100. sempy_labs/_user_delegation_key.py +44 -0
  101. sempy_labs/_utils.py +79 -0
  102. sempy_labs/_vertipaq.py +1021 -0
  103. sempy_labs/_vpax.py +388 -0
  104. sempy_labs/_warehouses.py +234 -0
  105. sempy_labs/_workloads.py +140 -0
  106. sempy_labs/_workspace_identity.py +72 -0
  107. sempy_labs/_workspaces.py +595 -0
  108. sempy_labs/admin/__init__.py +170 -0
  109. sempy_labs/admin/_activities.py +167 -0
  110. sempy_labs/admin/_apps.py +145 -0
  111. sempy_labs/admin/_artifacts.py +65 -0
  112. sempy_labs/admin/_basic_functions.py +463 -0
  113. sempy_labs/admin/_capacities.py +508 -0
  114. sempy_labs/admin/_dataflows.py +45 -0
  115. sempy_labs/admin/_datasets.py +186 -0
  116. sempy_labs/admin/_domains.py +522 -0
  117. sempy_labs/admin/_external_data_share.py +100 -0
  118. sempy_labs/admin/_git.py +72 -0
  119. sempy_labs/admin/_items.py +265 -0
  120. sempy_labs/admin/_labels.py +211 -0
  121. sempy_labs/admin/_reports.py +241 -0
  122. sempy_labs/admin/_scanner.py +118 -0
  123. sempy_labs/admin/_shared.py +82 -0
  124. sempy_labs/admin/_sharing_links.py +110 -0
  125. sempy_labs/admin/_tags.py +131 -0
  126. sempy_labs/admin/_tenant.py +503 -0
  127. sempy_labs/admin/_tenant_keys.py +89 -0
  128. sempy_labs/admin/_users.py +140 -0
  129. sempy_labs/admin/_workspaces.py +236 -0
  130. sempy_labs/deployment_pipeline/__init__.py +23 -0
  131. sempy_labs/deployment_pipeline/_items.py +580 -0
  132. sempy_labs/directlake/__init__.py +57 -0
  133. sempy_labs/directlake/_autosync.py +58 -0
  134. sempy_labs/directlake/_directlake_schema_compare.py +120 -0
  135. sempy_labs/directlake/_directlake_schema_sync.py +161 -0
  136. sempy_labs/directlake/_dl_helper.py +274 -0
  137. sempy_labs/directlake/_generate_shared_expression.py +94 -0
  138. sempy_labs/directlake/_get_directlake_lakehouse.py +62 -0
  139. sempy_labs/directlake/_get_shared_expression.py +34 -0
  140. sempy_labs/directlake/_guardrails.py +96 -0
  141. sempy_labs/directlake/_list_directlake_model_calc_tables.py +70 -0
  142. sempy_labs/directlake/_show_unsupported_directlake_objects.py +90 -0
  143. sempy_labs/directlake/_update_directlake_model_lakehouse_connection.py +239 -0
  144. sempy_labs/directlake/_update_directlake_partition_entity.py +259 -0
  145. sempy_labs/directlake/_warm_cache.py +236 -0
  146. sempy_labs/dotnet_lib/dotnet.runtime.config.json +10 -0
  147. sempy_labs/environment/__init__.py +23 -0
  148. sempy_labs/environment/_items.py +212 -0
  149. sempy_labs/environment/_pubstage.py +223 -0
  150. sempy_labs/eventstream/__init__.py +37 -0
  151. sempy_labs/eventstream/_items.py +263 -0
  152. sempy_labs/eventstream/_topology.py +652 -0
  153. sempy_labs/graph/__init__.py +59 -0
  154. sempy_labs/graph/_groups.py +651 -0
  155. sempy_labs/graph/_sensitivity_labels.py +120 -0
  156. sempy_labs/graph/_teams.py +125 -0
  157. sempy_labs/graph/_user_licenses.py +96 -0
  158. sempy_labs/graph/_users.py +516 -0
  159. sempy_labs/graph_model/__init__.py +15 -0
  160. sempy_labs/graph_model/_background_jobs.py +63 -0
  161. sempy_labs/graph_model/_items.py +149 -0
  162. sempy_labs/lakehouse/__init__.py +67 -0
  163. sempy_labs/lakehouse/_blobs.py +247 -0
  164. sempy_labs/lakehouse/_get_lakehouse_columns.py +102 -0
  165. sempy_labs/lakehouse/_get_lakehouse_tables.py +274 -0
  166. sempy_labs/lakehouse/_helper.py +250 -0
  167. sempy_labs/lakehouse/_lakehouse.py +351 -0
  168. sempy_labs/lakehouse/_livy_sessions.py +143 -0
  169. sempy_labs/lakehouse/_materialized_lake_views.py +157 -0
  170. sempy_labs/lakehouse/_partitioning.py +165 -0
  171. sempy_labs/lakehouse/_schemas.py +217 -0
  172. sempy_labs/lakehouse/_shortcuts.py +440 -0
  173. sempy_labs/migration/__init__.py +35 -0
  174. sempy_labs/migration/_create_pqt_file.py +238 -0
  175. sempy_labs/migration/_direct_lake_to_import.py +105 -0
  176. sempy_labs/migration/_migrate_calctables_to_lakehouse.py +398 -0
  177. sempy_labs/migration/_migrate_calctables_to_semantic_model.py +148 -0
  178. sempy_labs/migration/_migrate_model_objects_to_semantic_model.py +533 -0
  179. sempy_labs/migration/_migrate_tables_columns_to_semantic_model.py +172 -0
  180. sempy_labs/migration/_migration_validation.py +71 -0
  181. sempy_labs/migration/_refresh_calc_tables.py +131 -0
  182. sempy_labs/mirrored_azure_databricks_catalog/__init__.py +15 -0
  183. sempy_labs/mirrored_azure_databricks_catalog/_discover.py +213 -0
  184. sempy_labs/mirrored_azure_databricks_catalog/_refresh_catalog_metadata.py +45 -0
  185. sempy_labs/ml_model/__init__.py +23 -0
  186. sempy_labs/ml_model/_functions.py +427 -0
  187. sempy_labs/report/_BPAReportTemplate.json +232 -0
  188. sempy_labs/report/__init__.py +55 -0
  189. sempy_labs/report/_bpareporttemplate/.pbi/localSettings.json +9 -0
  190. sempy_labs/report/_bpareporttemplate/.platform +11 -0
  191. sempy_labs/report/_bpareporttemplate/StaticResources/SharedResources/BaseThemes/CY24SU06.json +710 -0
  192. sempy_labs/report/_bpareporttemplate/definition/pages/01d72098bda5055bd500/page.json +11 -0
  193. sempy_labs/report/_bpareporttemplate/definition/pages/01d72098bda5055bd500/visuals/1b08bce3bebabb0a27a8/visual.json +191 -0
  194. sempy_labs/report/_bpareporttemplate/definition/pages/01d72098bda5055bd500/visuals/2f22ddb70c301693c165/visual.json +438 -0
  195. sempy_labs/report/_bpareporttemplate/definition/pages/01d72098bda5055bd500/visuals/3b1182230aa6c600b43a/visual.json +127 -0
  196. sempy_labs/report/_bpareporttemplate/definition/pages/01d72098bda5055bd500/visuals/58577ba6380c69891500/visual.json +576 -0
  197. sempy_labs/report/_bpareporttemplate/definition/pages/01d72098bda5055bd500/visuals/a2a8fa5028b3b776c96c/visual.json +207 -0
  198. sempy_labs/report/_bpareporttemplate/definition/pages/01d72098bda5055bd500/visuals/adfd47ef30652707b987/visual.json +506 -0
  199. sempy_labs/report/_bpareporttemplate/definition/pages/01d72098bda5055bd500/visuals/b6a80ee459e716e170b1/visual.json +127 -0
  200. sempy_labs/report/_bpareporttemplate/definition/pages/01d72098bda5055bd500/visuals/ce3130a721c020cc3d81/visual.json +513 -0
  201. sempy_labs/report/_bpareporttemplate/definition/pages/92735ae19b31712208ad/page.json +8 -0
  202. sempy_labs/report/_bpareporttemplate/definition/pages/92735ae19b31712208ad/visuals/66e60dfb526437cd78d1/visual.json +112 -0
  203. sempy_labs/report/_bpareporttemplate/definition/pages/c597da16dc7e63222a82/page.json +11 -0
  204. sempy_labs/report/_bpareporttemplate/definition/pages/c597da16dc7e63222a82/visuals/07deb8bce824e1be37d7/visual.json +513 -0
  205. sempy_labs/report/_bpareporttemplate/definition/pages/c597da16dc7e63222a82/visuals/0b1c68838818b32ad03b/visual.json +352 -0
  206. sempy_labs/report/_bpareporttemplate/definition/pages/c597da16dc7e63222a82/visuals/0c171de9d2683d10b930/visual.json +37 -0
  207. sempy_labs/report/_bpareporttemplate/definition/pages/c597da16dc7e63222a82/visuals/0efa01be0510e40a645e/visual.json +542 -0
  208. sempy_labs/report/_bpareporttemplate/definition/pages/c597da16dc7e63222a82/visuals/6bf2f0eb830ab53cc668/visual.json +221 -0
  209. sempy_labs/report/_bpareporttemplate/definition/pages/c597da16dc7e63222a82/visuals/88d8141cb8500b60030c/visual.json +127 -0
  210. sempy_labs/report/_bpareporttemplate/definition/pages/c597da16dc7e63222a82/visuals/a753273590beed656a03/visual.json +576 -0
  211. sempy_labs/report/_bpareporttemplate/definition/pages/c597da16dc7e63222a82/visuals/b8fdc82cddd61ac447bc/visual.json +127 -0
  212. sempy_labs/report/_bpareporttemplate/definition/pages/d37dce724a0ccc30044b/page.json +9 -0
  213. sempy_labs/report/_bpareporttemplate/definition/pages/d37dce724a0ccc30044b/visuals/ce8532a7e25020271077/visual.json +38 -0
  214. sempy_labs/report/_bpareporttemplate/definition/pages/pages.json +10 -0
  215. sempy_labs/report/_bpareporttemplate/definition/report.json +176 -0
  216. sempy_labs/report/_bpareporttemplate/definition/version.json +4 -0
  217. sempy_labs/report/_bpareporttemplate/definition.pbir +14 -0
  218. sempy_labs/report/_download_report.py +76 -0
  219. sempy_labs/report/_export_report.py +257 -0
  220. sempy_labs/report/_generate_report.py +427 -0
  221. sempy_labs/report/_paginated.py +76 -0
  222. sempy_labs/report/_report_bpa.py +354 -0
  223. sempy_labs/report/_report_bpa_rules.py +115 -0
  224. sempy_labs/report/_report_functions.py +581 -0
  225. sempy_labs/report/_report_helper.py +227 -0
  226. sempy_labs/report/_report_list_functions.py +110 -0
  227. sempy_labs/report/_report_rebind.py +149 -0
  228. sempy_labs/report/_reportwrapper.py +3100 -0
  229. sempy_labs/report/_save_report.py +147 -0
  230. sempy_labs/snowflake_database/__init__.py +10 -0
  231. sempy_labs/snowflake_database/_items.py +105 -0
  232. sempy_labs/sql_database/__init__.py +21 -0
  233. sempy_labs/sql_database/_items.py +201 -0
  234. sempy_labs/sql_database/_mirroring.py +79 -0
  235. sempy_labs/theme/__init__.py +12 -0
  236. sempy_labs/theme/_org_themes.py +129 -0
  237. sempy_labs/tom/__init__.py +3 -0
  238. sempy_labs/tom/_model.py +5977 -0
  239. sempy_labs/variable_library/__init__.py +19 -0
  240. sempy_labs/variable_library/_functions.py +403 -0
  241. sempy_labs/warehouse/__init__.py +28 -0
  242. sempy_labs/warehouse/_items.py +234 -0
  243. sempy_labs/warehouse/_restore_points.py +309 -0
sempy_labs/_ai.py ADDED
@@ -0,0 +1,437 @@
1
+ import sempy
2
+ import sempy.fabric as fabric
3
+ import pandas as pd
4
+ from typing import List, Optional, Union
5
+ from IPython.display import display
6
+ import sempy_labs._icons as icons
7
+ from .._helper_functions import (
8
+ _read_delta_table,
9
+ _run_spark_sql_query,
10
+ )
11
+
12
+
13
+ def optimize_semantic_model(dataset: str, workspace: Optional[str] = None):
14
+
15
+ from ._model_bpa import run_model_bpa
16
+ from .directlake._dl_helper import check_fallback_reason
17
+ from ._helper_functions import format_dax_object_name
18
+
19
+ modelBPA = run_model_bpa(
20
+ dataset=dataset, workspace=workspace, return_dataframe=True
21
+ )
22
+ dfC = fabric.list_columns(dataset=dataset, workspace=workspace, extended=True)
23
+ dfC["Column Object"] = format_dax_object_name(dfC["Table Name"], dfC["Column Name"])
24
+ dfC["Total Size"] = dfC["Total Size"].astype("int")
25
+ dfP = fabric.list_partitions(dataset=dataset, workspace=workspace)
26
+
27
+ modelBPA_col = modelBPA[modelBPA["Object Type"] == "Column"]
28
+ modelBPA_col = pd.merge(
29
+ modelBPA_col,
30
+ dfC[["Column Object", "Total Size"]],
31
+ left_on="Object Name",
32
+ right_on="Column Object",
33
+ how="left",
34
+ )
35
+
36
+ isDirectLake = any(r["Mode"] == "DirectLake" for i, r in dfP.iterrows())
37
+
38
+ if isDirectLake:
39
+ fallback = check_fallback_reason(dataset=dataset, workspace=workspace)
40
+ fallback_filt = fallback[fallback["FallbackReasonID"] == 2]
41
+
42
+ if len(fallback_filt) > 0:
43
+ print(
44
+ f"{icons.yellow_dot} The '{dataset}' semantic model is a Direct Lake semantic model which contains views. "
45
+ "Since views always fall back to DirectQuery, it is recommended to only use lakehouse tables and not views."
46
+ )
47
+
48
+ # Potential model reduction estimate
49
+ ruleNames = [
50
+ "Remove unnecessary columns",
51
+ "Set IsAvailableInMdx to false on non-attribute columns",
52
+ ]
53
+
54
+ for rule in ruleNames:
55
+ df = modelBPA_col[modelBPA_col["Rule Name"] == rule]
56
+ df_filt = df[["Object Name", "Total Size"]].sort_values(
57
+ by="Total Size", ascending=False
58
+ )
59
+ totSize = df["Total Size"].sum()
60
+ if len(df_filt) > 0:
61
+ print(
62
+ f"{icons.yellow_dot} Potential savings of {totSize} bytes from following the '{rule}' rule."
63
+ )
64
+ display(df_filt)
65
+ else:
66
+ print(f"{icons.green_dot} The '{rule}' rule has been followed.")
67
+
68
+
69
+ def generate_aggs(
70
+ dataset: str,
71
+ table_name: str,
72
+ columns: Union[str, List[str]],
73
+ workspace: Optional[str] = None,
74
+ lakehouse_workspace: Optional[str] = None,
75
+ ):
76
+
77
+ from ._helper_functions import (
78
+ get_direct_lake_sql_endpoint,
79
+ create_abfss_path,
80
+ format_dax_object_name,
81
+ resolve_lakehouse_id,
82
+ )
83
+
84
+ sempy.fabric._client._utils._init_analysis_services()
85
+ import Microsoft.AnalysisServices.Tabular as TOM
86
+ import System
87
+
88
+ # columns = {
89
+ # 'SalesAmount': 'Sum',
90
+ # 'ProductKey': 'GroupBy',
91
+ # 'OrderDateKey': 'GroupBy'
92
+ # }
93
+
94
+ if workspace is None:
95
+ workspace_id = fabric.get_workspace_id()
96
+ workspace = fabric.resolve_workspace_name(workspace_id)
97
+
98
+ if lakehouse_workspace is None:
99
+ lakehouse_workspace = workspace
100
+ lakehouse_workspace_id = workspace_id
101
+ else:
102
+ lakehouse_workspace_id = fabric.resolve_workspace_id(lakehouse_workspace)
103
+
104
+ if isinstance(columns, str):
105
+ columns = [columns]
106
+
107
+ columnValues = columns.keys()
108
+
109
+ aggTypes = ["Sum", "Count", "Min", "Max", "GroupBy"]
110
+ aggTypesAggregate = ["Sum", "Count", "Min", "Max"]
111
+ numericTypes = ["Int64", "Double", "Decimal"]
112
+
113
+ if any(value not in aggTypes for value in columns.values()):
114
+ raise ValueError(
115
+ f"{icons.red_dot} Invalid aggregation type(s) have been specified in the 'columns' parameter. Valid aggregation types: {aggTypes}."
116
+ )
117
+
118
+ dfC = fabric.list_columns(dataset=dataset, workspace=workspace)
119
+ dfP = fabric.list_partitions(dataset=dataset, workspace=workspace)
120
+ dfM = fabric.list_measures(dataset=dataset, workspace=workspace)
121
+ dfR = fabric.list_relationships(dataset=dataset, workspace=workspace)
122
+ if not any(r["Mode"] == "DirectLake" for i, r in dfP.iterrows()):
123
+ raise ValueError(
124
+ f"{icons.red_dot} The '{dataset}' semantic model within the '{workspace}' workspace is not in Direct Lake mode. This function is only relevant for Direct Lake semantic models."
125
+ )
126
+
127
+ dfC_filtT = dfC[dfC["Table Name"] == table_name]
128
+
129
+ if len(dfC_filtT) == 0:
130
+ raise ValueError(
131
+ f"{icons.red_dot} The '{table_name}' table does not exist in the '{dataset}' semantic model within the '{workspace}' workspace."
132
+ )
133
+
134
+ dfC_filt = dfC[
135
+ (dfC["Table Name"] == table_name) & (dfC["Column Name"].isin(columnValues))
136
+ ]
137
+
138
+ if len(columns) != len(dfC_filt):
139
+ raise ValueError(
140
+ f"{icons.red_dot} Columns listed in '{columnValues}' do not exist in the '{table_name}' table in the '{dataset}' semantic model within the '{workspace}' workspace."
141
+ )
142
+
143
+ # Check if doing sum/count/min/max etc. on a non-number column
144
+ for cm, agg in columns.items():
145
+ dfC_col = dfC_filt[dfC_filt["Column Name"] == cm]
146
+ dataType = dfC_col["Data Type"].iloc[0]
147
+ if agg in aggTypesAggregate and dataType not in numericTypes:
148
+ raise ValueError(
149
+ f"{icons.red_dot} The '{cm}' column in the '{table_name}' table is of '{dataType}' data type. Only columns of '{numericTypes}' data types"
150
+ f" can be aggregated as '{aggTypesAggregate}' aggregation types."
151
+ )
152
+
153
+ # Create/update lakehouse delta agg table
154
+ aggSuffix = "_agg"
155
+ aggTableName = f"{table_name}{aggSuffix}"
156
+ aggLakeTName = aggTableName.lower().replace(" ", "_")
157
+ dfP = fabric.list_partitions(dataset=dataset, workspace=workspace)
158
+ dfP_filt = dfP[dfP["Table Name"] == table_name]
159
+ lakeTName = dfP_filt["Query"].iloc[0]
160
+
161
+ sqlEndpointId = get_direct_lake_sql_endpoint(dataset=dataset, workspace=workspace)
162
+
163
+ dfI = fabric.list_items(workspace=lakehouse_workspace, type="SQLEndpoint")
164
+ dfI_filt = dfI[(dfI["Id"] == sqlEndpointId)]
165
+
166
+ if len(dfI_filt) == 0:
167
+ raise ValueError(
168
+ f"{icons.red_dot} The lakehouse (SQL Endpoint) used by the '{dataset}' semantic model does not reside in"
169
+ f" the '{lakehouse_workspace}' workspace. Please update the lakehouse_workspace parameter."
170
+ )
171
+
172
+ lakehouseName = dfI_filt["Display Name"].iloc[0]
173
+ lakehouse_id = resolve_lakehouse_id(
174
+ lakehouse=lakehouseName, workspace=lakehouse_workspace
175
+ )
176
+
177
+ # Generate SQL query
178
+ query = "SELECT"
179
+ groupBy = "\nGROUP BY"
180
+ for cm, agg in columns.items():
181
+ colFilt = dfC_filt[dfC_filt["Column Name"] == cm]
182
+ sourceCol = colFilt["Source"].iloc[0]
183
+
184
+ if agg == "GroupBy":
185
+ query = f"{query}\n{sourceCol},"
186
+ groupBy = f"{groupBy}\n{sourceCol},"
187
+ else:
188
+ query = f"{query}\n{agg}({sourceCol}) AS {sourceCol},"
189
+
190
+ query = query[:-1]
191
+
192
+ fromTablePath = create_abfss_path(
193
+ lakehouse_id=lakehouse_id,
194
+ lakehouse_workspace_id=lakehouse_workspace_id,
195
+ delta_table_name=lakeTName,
196
+ )
197
+
198
+ df = _read_delta_table(fromTablePath)
199
+ tempTableName = f"delta_table_{lakeTName}"
200
+ df.createOrReplaceTempView(tempTableName)
201
+ sqlQuery = f"{query} \n FROM {tempTableName} {groupBy}"
202
+
203
+ sqlQuery = sqlQuery[:-1]
204
+ print(sqlQuery)
205
+
206
+ # Save query to spark dataframe
207
+ spark_df = _run_spark_sql_query(sqlQuery)
208
+ f"\nCreating/updating the '{aggLakeTName}' table in the lakehouse..."
209
+ # Write spark dataframe to delta table
210
+ aggFilePath = create_abfss_path(
211
+ lakehouse_id=lakehouse_id,
212
+ lakehouse_workspace_id=lakehouse_workspace_id,
213
+ delta_table_name=aggLakeTName,
214
+ )
215
+ spark_df.write.mode("overwrite").format("delta").save(aggFilePath)
216
+ f"{icons.green_dot} The '{aggLakeTName}' table has been created/updated in the lakehouse."
217
+
218
+ # Create/update semantic model agg table
219
+ tom_server = fabric.create_tom_server(
220
+ dataset=dataset, readonly=False, workspace=workspace
221
+ )
222
+ m = tom_server.Databases.GetByName(dataset).Model
223
+ print(f"\n{icons.in_progress} Updating the '{dataset}' semantic model...")
224
+ dfC_agg = dfC[dfC["Table Name"] == aggTableName]
225
+
226
+ if len(dfC_agg) == 0:
227
+ print(f"{icons.in_progress} Creating the '{aggTableName}' table...")
228
+ exp = m.Expressions["DatabaseQuery"]
229
+ tbl = TOM.Table()
230
+ tbl.Name = aggTableName
231
+ tbl.IsHidden = True
232
+
233
+ ep = TOM.EntityPartitionSource()
234
+ ep.Name = aggTableName
235
+ ep.EntityName = aggLakeTName
236
+ ep.ExpressionSource = exp
237
+
238
+ part = TOM.Partition()
239
+ part.Name = aggTableName
240
+ part.Source = ep
241
+ part.Mode = TOM.ModeType.DirectLake
242
+
243
+ tbl.Partitions.Add(part)
244
+
245
+ for i, r in dfC_filt.iterrows():
246
+ scName = r["Source"]
247
+ cName = r["Column Name"]
248
+ dType = r["Data Type"]
249
+
250
+ col = TOM.DataColumn()
251
+ col.Name = cName
252
+ col.IsHidden = True
253
+ col.SourceColumn = scName
254
+ col.DataType = System.Enum.Parse(TOM.DataType, dType)
255
+
256
+ tbl.Columns.Add(col)
257
+ print(
258
+ f"{icons.green_dot} The '{aggTableName}'[{cName}] column has been added to the '{dataset}' semantic model."
259
+ )
260
+
261
+ m.Tables.Add(tbl)
262
+ print(
263
+ f"{icons.green_dot} The '{aggTableName}' table has been added to the '{dataset}' semantic model."
264
+ )
265
+ else:
266
+ print(f"{icons.in_progress} Updating the '{aggTableName}' table's columns...")
267
+ # Remove existing columns
268
+ for t in m.Tables:
269
+ tName = t.Name
270
+ for c in t.Columns:
271
+ cName = c.Name
272
+ if t.Name == aggTableName:
273
+ m.Tables[tName].Columns.Remove(cName)
274
+ # Add columns
275
+ for i, r in dfC_filt.iterrows():
276
+ scName = r["Source"]
277
+ cName = r["Column Name"]
278
+ dType = r["Data Type"]
279
+
280
+ col = TOM.DataColumn()
281
+ col.Name = cName
282
+ col.IsHidden = True
283
+ col.SourceColumn = scName
284
+ col.DataType = System.Enum.Parse(TOM.DataType, dType)
285
+
286
+ m.Tables[aggTableName].Columns.Add(col)
287
+ print(
288
+ f"{icons.green_dot} The '{aggTableName}'[{cName}] column has been added."
289
+ )
290
+
291
+ # Create relationships
292
+ relMap = {"m": "Many", "1": "One", "0": "None"}
293
+
294
+ print(f"\n{icons.in_progress} Generating necessary relationships...")
295
+ for i, r in dfR.iterrows():
296
+ fromTable = r["From Table"]
297
+ fromColumn = r["From Column"]
298
+ toTable = r["To Table"]
299
+ toColumn = r["To Column"]
300
+ cfb = r["Cross Filtering Behavior"]
301
+ sfb = r["Security Filtering Behavior"]
302
+ mult = r["Multiplicity"]
303
+
304
+ crossFB = System.Enum.Parse(TOM.CrossFilteringBehavior, cfb)
305
+ secFB = System.Enum.Parse(TOM.SecurityFilteringBehavior, sfb)
306
+ fromCardinality = System.Enum.Parse(
307
+ TOM.RelationshipEndCardinality, relMap.get(mult[0])
308
+ )
309
+ toCardinality = System.Enum.Parse(
310
+ TOM.RelationshipEndCardinality, relMap.get(mult[-1])
311
+ )
312
+
313
+ rel = TOM.SingleColumnRelationship()
314
+ rel.FromCardinality = fromCardinality
315
+ rel.ToCardinality = toCardinality
316
+ rel.IsActive = r["Active"]
317
+ rel.CrossFilteringBehavior = crossFB
318
+ rel.SecurityFilteringBehavior = secFB
319
+ rel.RelyOnReferentialIntegrity = r["Rely On Referential Integrity"]
320
+
321
+ if fromTable == table_name:
322
+ try:
323
+ rel.FromColumn = m.Tables[aggTableName].Columns[fromColumn]
324
+ m.Relationships.Add(rel)
325
+ print(
326
+ f"{icons.green_dot} '{aggTableName}'[{fromColumn}] -> '{toTable}'[{toColumn}] relationship has been added."
327
+ )
328
+ except Exception as e:
329
+ print(
330
+ f"{icons.red_dot} '{aggTableName}'[{fromColumn}] -> '{toTable}'[{toColumn}] relationship has not been created."
331
+ )
332
+ print(f"Exception occured: {e}")
333
+ elif toTable == table_name:
334
+ try:
335
+ rel.ToColumn = m.Tables[aggTableName].Columns[toColumn]
336
+ m.Relationships.Add(rel)
337
+ print(
338
+ f"{icons.green_dot} '{fromTable}'[{fromColumn}] -> '{aggTableName}'[{toColumn}] relationship has been added."
339
+ )
340
+ except Exception as e:
341
+ print(
342
+ f"{icons.red_dot} '{fromTable}'[{fromColumn}] -> '{aggTableName}'[{toColumn}] relationship has not been created."
343
+ )
344
+ print(f"Exception occured: {e}")
345
+ "Relationship creation is complete."
346
+
347
+ # Create IF measure
348
+ f"\n{icons.in_progress} Creating measure to check if the agg table can be used..."
349
+ aggChecker = "IF("
350
+ dfR_filt = dfR[
351
+ (dfR["From Table"] == table_name) & (~dfR["From Column"].isin(columnValues))
352
+ ]
353
+
354
+ for i, r in dfR_filt.iterrows():
355
+ toTable = r["To Table"]
356
+ aggChecker = f"{aggChecker}\nISCROSSFILTERED('{toTable}') ||"
357
+
358
+ aggChecker = aggChecker[:-3]
359
+ aggChecker = f"{aggChecker},1,0)"
360
+ print(aggChecker)
361
+
362
+ # Todo: add IFISFILTERED clause for columns
363
+ f"\n{icons.in_progress} Creating the base measures in the agg table..."
364
+ # Create base agg measures
365
+ dep = fabric.evaluate_dax(
366
+ dataset=dataset,
367
+ workspace=workspace,
368
+ dax_string="""
369
+ SELECT
370
+ [TABLE] AS [Table Name]
371
+ ,[OBJECT] AS [Object Name]
372
+ ,[OBJECT_TYPE] AS [Object Type]
373
+ ,[REFERENCED_TABLE] AS [Referenced Table]
374
+ ,[REFERENCED_OBJECT] AS [Referenced Object]
375
+ ,[REFERENCED_OBJECT_TYPE] AS [Referenced Object Type]
376
+ FROM $SYSTEM.DISCOVER_CALC_DEPENDENCY
377
+ WHERE [OBJECT_TYPE] = 'MEASURE'
378
+ """,
379
+ )
380
+
381
+ baseMeasures = dep[
382
+ (dep["Referenced Object Type"] == "COLUMN")
383
+ & (dep["Referenced Table"] == table_name)
384
+ & (dep["Referenced Object"].isin(columnValues))
385
+ ]
386
+ for i, r in baseMeasures.iterrows():
387
+ tName = r["Table Name"]
388
+ mName = r["Object Name"]
389
+ cName = r["Referenced Object"]
390
+ dfM_filt = dfM[dfM["Measure Name"] == mName]
391
+ expr = dfM_filt["Measure Expression"].iloc[0]
392
+
393
+ colFQNonAgg = format_dax_object_name(tName, cName)
394
+ colFQAgg = format_dax_object_name(aggTableName, cName)
395
+ colNQNonAgg = f"{tName}[{cName}]"
396
+
397
+ if " " in tName:
398
+ newExpr = expr.replace(colFQNonAgg, colFQAgg)
399
+ else:
400
+ newExpr = expr.replace(colFQNonAgg, colFQAgg).replace(colNQNonAgg, colFQAgg)
401
+ print(expr)
402
+ print(newExpr)
403
+
404
+ aggMName = f"{mName}{aggSuffix}"
405
+ measure = TOM.Measure()
406
+ measure.Name = aggMName
407
+ measure.IsHidden = True
408
+ measure.Expression = newExpr
409
+ m.Tables[aggTableName].Measures.Add(measure)
410
+ f"The '{aggMName}' measure has been created in the '{aggTableName}' table."
411
+
412
+ # Update base detail measures
413
+
414
+ # m.SaveChanges()
415
+
416
+
417
+ # Identify views used within Direct Lake model
418
+ # workspace = 'MK Demo 6'
419
+ # lakehouse = 'MyLakehouse'
420
+ # dataset = 'MigrationTest'
421
+ # lakehouse_workspace = workspace
422
+
423
+ # dfView = pd.DataFrame(columns=['Workspace Name', 'Lakehouse Name', 'View Name'])
424
+ # dfP = fabric.list_partitions(dataset = dataset, workspace = workspace)
425
+ # isDirectLake = any(r['Mode'] == 'DirectLake' for i, r in dfP.iterrows())
426
+
427
+ # spark = _create_spark_session()
428
+ # views = spark.sql(f"SHOW VIEWS IN {lakehouse}").collect()
429
+ # for view in views:
430
+ # viewName = view['viewName']
431
+ # isTemporary = view['isTemporary']
432
+ # new_data = {'Workspace Name': workspace, 'Lakehouse Name': lakehouse, 'View Name': viewName}
433
+ # dfView = pd.concat([dfView, pd.DataFrame(new_data, index=[0])], ignore_index=True)
434
+ # dfView
435
+ # lakeT = get_lakehouse_tables(lakehouse, lakehouse_workspace)
436
+ # if not dfP['Query'].isin(lakeT['Table Name'].values):
437
+ # if
@@ -0,0 +1,264 @@
1
+ from typing import Dict, Literal, Optional
2
+ from azure.core.credentials import AccessToken, TokenCredential
3
+ from azure.identity import ClientSecretCredential
4
+ from sempy._utils._log import log
5
+ from contextlib import contextmanager
6
+ import contextvars
7
+
8
+
9
+ class ServicePrincipalTokenProvider(TokenCredential):
10
+ """
11
+ A class to acquire authentication token with Service Principal.
12
+
13
+ For more information on Service Principal see: `Application and service principal objects in Microsoft Entra ID <https://learn.microsoft.com/en-us/entra/identity-platform/app-objects-and-service-principals?tabs=browser#service-principal-object>`_
14
+ """
15
+
16
+ _shorthand_scopes: Dict[str, str] = {
17
+ "pbi": "https://analysis.windows.net/powerbi/api/.default",
18
+ "storage": "https://storage.azure.com/.default",
19
+ "azure": "https://management.azure.com/.default",
20
+ "graph": "https://graph.microsoft.com/.default",
21
+ "asazure": "https://{region}.asazure.windows.net/.default",
22
+ "keyvault": "https://vault.azure.net/.default",
23
+ }
24
+
25
+ def __init__(self, credential: ClientSecretCredential):
26
+
27
+ self.credential = credential
28
+
29
+ @classmethod
30
+ def from_aad_application_key_authentication(
31
+ cls, tenant_id: str, client_id: str, client_secret: str
32
+ ) -> "ServicePrincipalTokenProvider":
33
+ """
34
+ Generates the ServicePrincipalTokenProvider, providing the Service Principal information.
35
+
36
+ ***USE THIS ONE ONLY FOR TEST PURPOSE. FOR PRODUCTION WE RECOMMEND CALLING ServicePrincipalTokenProvider.from_azure_key_vault()***
37
+
38
+ Parameters
39
+ ----------
40
+ tenant_id : str
41
+ The Fabric Tenant ID.
42
+ client_id : str
43
+ The Service Principal Application Client ID.
44
+ client_secret : str
45
+ The Service Principal Client Secret.
46
+
47
+ Returns
48
+ -------
49
+ ServicePrincipalTokenProvider
50
+ Token provider to be used with FabricRestClient or PowerBIRestClient.
51
+ """
52
+ credential = ClientSecretCredential(
53
+ tenant_id=tenant_id, client_id=client_id, client_secret=client_secret
54
+ )
55
+
56
+ cls.tenant_id = tenant_id
57
+ cls.client_id = client_id
58
+ cls.client_secret = client_secret
59
+
60
+ return cls(credential)
61
+
62
+ @classmethod
63
+ def from_azure_key_vault(
64
+ cls,
65
+ key_vault_uri: str,
66
+ key_vault_tenant_id: str,
67
+ key_vault_client_id: str,
68
+ key_vault_client_secret: str,
69
+ ) -> "ServicePrincipalTokenProvider":
70
+ """
71
+ Generates the ServicePrincipalTokenProvider, providing the Azure Key Vault details.
72
+
73
+ For more information on Azure Key Vault, `click here <https://learn.microsoft.com/en-us/azure/key-vault/general/overview>`_.
74
+
75
+ Parameters
76
+ ----------
77
+ key_vault_uri : str
78
+ Azure Key Vault URI.
79
+ key_vault_tenant_id : str
80
+ Name of the secret in the Key Vault with the Fabric Tenant ID.
81
+ key_vault_client_id : str
82
+ Name of the secret in the Key Vault with the Service Principal Client ID.
83
+ key_vault_client_secret : str
84
+ Name of the secret in the Key Vault with the Service Principal Client Secret.
85
+
86
+ Returns
87
+ -------
88
+ ServicePrincipalTokenProvider
89
+ Token provider to be used with FabricRestClient or PowerBIRestClient.
90
+ """
91
+
92
+ import notebookutils
93
+
94
+ tenant_id = notebookutils.credentials.getSecret(
95
+ key_vault_uri, key_vault_tenant_id
96
+ )
97
+ client_id = notebookutils.credentials.getSecret(
98
+ key_vault_uri, key_vault_client_id
99
+ )
100
+ client_secret = notebookutils.credentials.getSecret(
101
+ key_vault_uri, key_vault_client_secret
102
+ )
103
+
104
+ credential = ClientSecretCredential(
105
+ tenant_id=tenant_id, client_id=client_id, client_secret=client_secret
106
+ )
107
+
108
+ cls.tenant_id = tenant_id
109
+ cls.client_id = client_id
110
+ cls.client_secret = client_secret
111
+
112
+ return cls(credential)
113
+
114
+ def __call__(
115
+ self,
116
+ audience: Literal[
117
+ "pbi", "storage", "azure", "graph", "asazure", "keyvault"
118
+ ] = "pbi",
119
+ region: Optional[str] = None,
120
+ ) -> str:
121
+ """
122
+ Parameters
123
+ ----------
124
+ audience : Literal["pbi", "storage", "azure", "graph", "asazure", "keyvault"] = "pbi") -> str
125
+ Literal if it's for PBI/Fabric API call or OneLake/Storage Account call.
126
+ region : str, default=None
127
+ The region of the Azure Analysis Services. For example: 'westus2'.
128
+ """
129
+ # Check if audience is supported
130
+ if audience not in self._shorthand_scopes:
131
+ raise NotImplementedError
132
+
133
+ return self.get_token(audience, region=region).token
134
+
135
+ def get_token(self, *scopes, **kwargs) -> AccessToken:
136
+ """
137
+ Gets a token for the specified scopes.
138
+
139
+ Parameters
140
+ ----------
141
+ *scopes : str
142
+ The scopes for which to obtain a token.
143
+ **kwargs : dict
144
+ Additional parameters to pass to the token request.
145
+
146
+ Returns
147
+ -------
148
+ AccessToken
149
+ The access token.
150
+ """
151
+ if len(scopes) == 0:
152
+ scopes = ("pbi",)
153
+
154
+ region = kwargs.pop("region", None)
155
+ scopes = [
156
+ self._get_fully_qualified_scope(scope, region=region) for scope in scopes
157
+ ]
158
+ return self.credential.get_token(*scopes, **kwargs)
159
+
160
+ def _get_fully_qualified_scope(
161
+ self, scope: str, region: Optional[str] = None
162
+ ) -> str:
163
+ """
164
+ Resolve to fully qualified scope if Fabric short-handed scope is given.
165
+ Otherwise, return the original scope.
166
+
167
+ Parameters
168
+ ----------
169
+ scope : str
170
+ The scope to resolve.
171
+ region : str, default=None
172
+ The specific region to use to resolve scope.
173
+ Required if scope is "asazure".
174
+
175
+ Returns
176
+ -------
177
+ str
178
+ The resolved scope.
179
+ """
180
+ fully_qualified_scope = self._shorthand_scopes.get(scope, scope)
181
+
182
+ if scope == "asazure":
183
+ if region is None:
184
+ raise ValueError("Region is required for 'asazure' scope")
185
+ return fully_qualified_scope.format(region=region)
186
+
187
+ return fully_qualified_scope
188
+
189
+
190
+ def _get_headers(
191
+ token_provider: TokenCredential,
192
+ audience: Literal[
193
+ "pbi", "storage", "azure", "graph", "asazure", "keyvault"
194
+ ] = "azure",
195
+ ):
196
+ """
197
+ Generates headers for an API request.
198
+ """
199
+
200
+ token = token_provider.get_token(audience).token
201
+
202
+ headers = {"Authorization": f"Bearer {token}"}
203
+
204
+ if audience == "graph":
205
+ headers["ConsistencyLevel"] = "eventual"
206
+ else:
207
+ headers["Content-Type"] = "application/json"
208
+
209
+ return headers
210
+
211
+
212
+ token_provider = contextvars.ContextVar("token_provider", default=None)
213
+
214
+
215
+ @log
216
+ @contextmanager
217
+ def service_principal_authentication(
218
+ key_vault_uri: str,
219
+ key_vault_tenant_id: str,
220
+ key_vault_client_id: str,
221
+ key_vault_client_secret: str,
222
+ ):
223
+ """
224
+ Establishes an authentication via Service Principal.
225
+
226
+ Parameters
227
+ ----------
228
+ key_vault_uri : str
229
+ Azure Key Vault URI.
230
+ key_vault_tenant_id : str
231
+ Name of the secret in the Key Vault with the Fabric Tenant ID.
232
+ key_vault_client_id : str
233
+ Name of the secret in the Key Vault with the Service Principal Client ID.
234
+ key_vault_client_secret : str
235
+ Name of the secret in the Key Vault with the Service Principal Client Secret.
236
+ """
237
+
238
+ # Save the prior state
239
+ prior_token = token_provider.get()
240
+
241
+ # Set the new token_provider in a thread-safe manner
242
+ token_provider.set(
243
+ ServicePrincipalTokenProvider.from_azure_key_vault(
244
+ key_vault_uri=key_vault_uri,
245
+ key_vault_tenant_id=key_vault_tenant_id,
246
+ key_vault_client_id=key_vault_client_id,
247
+ key_vault_client_secret=key_vault_client_secret,
248
+ )
249
+ )
250
+ try:
251
+ from sempy.fabric import set_service_principal
252
+
253
+ with set_service_principal(
254
+ (key_vault_uri, key_vault_tenant_id),
255
+ (key_vault_uri, key_vault_client_id),
256
+ client_secret=(key_vault_uri, key_vault_client_secret),
257
+ ):
258
+ yield
259
+ finally:
260
+ # Restore the prior state
261
+ if prior_token is None:
262
+ token_provider.set(None)
263
+ else:
264
+ token_provider.set(prior_token)