semantic-link-labs 0.12.8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- semantic_link_labs-0.12.8.dist-info/METADATA +354 -0
- semantic_link_labs-0.12.8.dist-info/RECORD +243 -0
- semantic_link_labs-0.12.8.dist-info/WHEEL +5 -0
- semantic_link_labs-0.12.8.dist-info/licenses/LICENSE +21 -0
- semantic_link_labs-0.12.8.dist-info/top_level.txt +1 -0
- sempy_labs/__init__.py +606 -0
- sempy_labs/_a_lib_info.py +2 -0
- sempy_labs/_ai.py +437 -0
- sempy_labs/_authentication.py +264 -0
- sempy_labs/_bpa_translation/_model/_translations_am-ET.po +869 -0
- sempy_labs/_bpa_translation/_model/_translations_ar-AE.po +908 -0
- sempy_labs/_bpa_translation/_model/_translations_bg-BG.po +968 -0
- sempy_labs/_bpa_translation/_model/_translations_ca-ES.po +963 -0
- sempy_labs/_bpa_translation/_model/_translations_cs-CZ.po +943 -0
- sempy_labs/_bpa_translation/_model/_translations_da-DK.po +945 -0
- sempy_labs/_bpa_translation/_model/_translations_de-DE.po +988 -0
- sempy_labs/_bpa_translation/_model/_translations_el-GR.po +993 -0
- sempy_labs/_bpa_translation/_model/_translations_es-ES.po +971 -0
- sempy_labs/_bpa_translation/_model/_translations_fa-IR.po +933 -0
- sempy_labs/_bpa_translation/_model/_translations_fi-FI.po +942 -0
- sempy_labs/_bpa_translation/_model/_translations_fr-FR.po +994 -0
- sempy_labs/_bpa_translation/_model/_translations_ga-IE.po +967 -0
- sempy_labs/_bpa_translation/_model/_translations_he-IL.po +902 -0
- sempy_labs/_bpa_translation/_model/_translations_hi-IN.po +944 -0
- sempy_labs/_bpa_translation/_model/_translations_hu-HU.po +963 -0
- sempy_labs/_bpa_translation/_model/_translations_id-ID.po +946 -0
- sempy_labs/_bpa_translation/_model/_translations_is-IS.po +939 -0
- sempy_labs/_bpa_translation/_model/_translations_it-IT.po +986 -0
- sempy_labs/_bpa_translation/_model/_translations_ja-JP.po +846 -0
- sempy_labs/_bpa_translation/_model/_translations_ko-KR.po +839 -0
- sempy_labs/_bpa_translation/_model/_translations_mt-MT.po +967 -0
- sempy_labs/_bpa_translation/_model/_translations_nl-NL.po +978 -0
- sempy_labs/_bpa_translation/_model/_translations_pl-PL.po +962 -0
- sempy_labs/_bpa_translation/_model/_translations_pt-BR.po +962 -0
- sempy_labs/_bpa_translation/_model/_translations_pt-PT.po +957 -0
- sempy_labs/_bpa_translation/_model/_translations_ro-RO.po +968 -0
- sempy_labs/_bpa_translation/_model/_translations_ru-RU.po +964 -0
- sempy_labs/_bpa_translation/_model/_translations_sk-SK.po +952 -0
- sempy_labs/_bpa_translation/_model/_translations_sl-SL.po +950 -0
- sempy_labs/_bpa_translation/_model/_translations_sv-SE.po +942 -0
- sempy_labs/_bpa_translation/_model/_translations_ta-IN.po +976 -0
- sempy_labs/_bpa_translation/_model/_translations_te-IN.po +947 -0
- sempy_labs/_bpa_translation/_model/_translations_th-TH.po +924 -0
- sempy_labs/_bpa_translation/_model/_translations_tr-TR.po +953 -0
- sempy_labs/_bpa_translation/_model/_translations_uk-UA.po +961 -0
- sempy_labs/_bpa_translation/_model/_translations_zh-CN.po +804 -0
- sempy_labs/_bpa_translation/_model/_translations_zu-ZA.po +969 -0
- sempy_labs/_capacities.py +1198 -0
- sempy_labs/_capacity_migration.py +660 -0
- sempy_labs/_clear_cache.py +351 -0
- sempy_labs/_connections.py +610 -0
- sempy_labs/_dashboards.py +69 -0
- sempy_labs/_data_access_security.py +98 -0
- sempy_labs/_data_pipelines.py +162 -0
- sempy_labs/_dataflows.py +668 -0
- sempy_labs/_dax.py +501 -0
- sempy_labs/_daxformatter.py +80 -0
- sempy_labs/_delta_analyzer.py +467 -0
- sempy_labs/_delta_analyzer_history.py +301 -0
- sempy_labs/_dictionary_diffs.py +221 -0
- sempy_labs/_documentation.py +147 -0
- sempy_labs/_domains.py +51 -0
- sempy_labs/_eventhouses.py +182 -0
- sempy_labs/_external_data_shares.py +230 -0
- sempy_labs/_gateways.py +521 -0
- sempy_labs/_generate_semantic_model.py +521 -0
- sempy_labs/_get_connection_string.py +84 -0
- sempy_labs/_git.py +543 -0
- sempy_labs/_graphQL.py +90 -0
- sempy_labs/_helper_functions.py +2833 -0
- sempy_labs/_icons.py +149 -0
- sempy_labs/_job_scheduler.py +609 -0
- sempy_labs/_kql_databases.py +149 -0
- sempy_labs/_kql_querysets.py +124 -0
- sempy_labs/_kusto.py +137 -0
- sempy_labs/_labels.py +124 -0
- sempy_labs/_list_functions.py +1720 -0
- sempy_labs/_managed_private_endpoints.py +253 -0
- sempy_labs/_mirrored_databases.py +416 -0
- sempy_labs/_mirrored_warehouses.py +60 -0
- sempy_labs/_ml_experiments.py +113 -0
- sempy_labs/_model_auto_build.py +140 -0
- sempy_labs/_model_bpa.py +557 -0
- sempy_labs/_model_bpa_bulk.py +378 -0
- sempy_labs/_model_bpa_rules.py +859 -0
- sempy_labs/_model_dependencies.py +343 -0
- sempy_labs/_mounted_data_factories.py +123 -0
- sempy_labs/_notebooks.py +441 -0
- sempy_labs/_one_lake_integration.py +151 -0
- sempy_labs/_onelake.py +131 -0
- sempy_labs/_query_scale_out.py +433 -0
- sempy_labs/_refresh_semantic_model.py +435 -0
- sempy_labs/_semantic_models.py +468 -0
- sempy_labs/_spark.py +455 -0
- sempy_labs/_sql.py +241 -0
- sempy_labs/_sql_audit_settings.py +207 -0
- sempy_labs/_sql_endpoints.py +214 -0
- sempy_labs/_tags.py +201 -0
- sempy_labs/_translations.py +43 -0
- sempy_labs/_user_delegation_key.py +44 -0
- sempy_labs/_utils.py +79 -0
- sempy_labs/_vertipaq.py +1021 -0
- sempy_labs/_vpax.py +388 -0
- sempy_labs/_warehouses.py +234 -0
- sempy_labs/_workloads.py +140 -0
- sempy_labs/_workspace_identity.py +72 -0
- sempy_labs/_workspaces.py +595 -0
- sempy_labs/admin/__init__.py +170 -0
- sempy_labs/admin/_activities.py +167 -0
- sempy_labs/admin/_apps.py +145 -0
- sempy_labs/admin/_artifacts.py +65 -0
- sempy_labs/admin/_basic_functions.py +463 -0
- sempy_labs/admin/_capacities.py +508 -0
- sempy_labs/admin/_dataflows.py +45 -0
- sempy_labs/admin/_datasets.py +186 -0
- sempy_labs/admin/_domains.py +522 -0
- sempy_labs/admin/_external_data_share.py +100 -0
- sempy_labs/admin/_git.py +72 -0
- sempy_labs/admin/_items.py +265 -0
- sempy_labs/admin/_labels.py +211 -0
- sempy_labs/admin/_reports.py +241 -0
- sempy_labs/admin/_scanner.py +118 -0
- sempy_labs/admin/_shared.py +82 -0
- sempy_labs/admin/_sharing_links.py +110 -0
- sempy_labs/admin/_tags.py +131 -0
- sempy_labs/admin/_tenant.py +503 -0
- sempy_labs/admin/_tenant_keys.py +89 -0
- sempy_labs/admin/_users.py +140 -0
- sempy_labs/admin/_workspaces.py +236 -0
- sempy_labs/deployment_pipeline/__init__.py +23 -0
- sempy_labs/deployment_pipeline/_items.py +580 -0
- sempy_labs/directlake/__init__.py +57 -0
- sempy_labs/directlake/_autosync.py +58 -0
- sempy_labs/directlake/_directlake_schema_compare.py +120 -0
- sempy_labs/directlake/_directlake_schema_sync.py +161 -0
- sempy_labs/directlake/_dl_helper.py +274 -0
- sempy_labs/directlake/_generate_shared_expression.py +94 -0
- sempy_labs/directlake/_get_directlake_lakehouse.py +62 -0
- sempy_labs/directlake/_get_shared_expression.py +34 -0
- sempy_labs/directlake/_guardrails.py +96 -0
- sempy_labs/directlake/_list_directlake_model_calc_tables.py +70 -0
- sempy_labs/directlake/_show_unsupported_directlake_objects.py +90 -0
- sempy_labs/directlake/_update_directlake_model_lakehouse_connection.py +239 -0
- sempy_labs/directlake/_update_directlake_partition_entity.py +259 -0
- sempy_labs/directlake/_warm_cache.py +236 -0
- sempy_labs/dotnet_lib/dotnet.runtime.config.json +10 -0
- sempy_labs/environment/__init__.py +23 -0
- sempy_labs/environment/_items.py +212 -0
- sempy_labs/environment/_pubstage.py +223 -0
- sempy_labs/eventstream/__init__.py +37 -0
- sempy_labs/eventstream/_items.py +263 -0
- sempy_labs/eventstream/_topology.py +652 -0
- sempy_labs/graph/__init__.py +59 -0
- sempy_labs/graph/_groups.py +651 -0
- sempy_labs/graph/_sensitivity_labels.py +120 -0
- sempy_labs/graph/_teams.py +125 -0
- sempy_labs/graph/_user_licenses.py +96 -0
- sempy_labs/graph/_users.py +516 -0
- sempy_labs/graph_model/__init__.py +15 -0
- sempy_labs/graph_model/_background_jobs.py +63 -0
- sempy_labs/graph_model/_items.py +149 -0
- sempy_labs/lakehouse/__init__.py +67 -0
- sempy_labs/lakehouse/_blobs.py +247 -0
- sempy_labs/lakehouse/_get_lakehouse_columns.py +102 -0
- sempy_labs/lakehouse/_get_lakehouse_tables.py +274 -0
- sempy_labs/lakehouse/_helper.py +250 -0
- sempy_labs/lakehouse/_lakehouse.py +351 -0
- sempy_labs/lakehouse/_livy_sessions.py +143 -0
- sempy_labs/lakehouse/_materialized_lake_views.py +157 -0
- sempy_labs/lakehouse/_partitioning.py +165 -0
- sempy_labs/lakehouse/_schemas.py +217 -0
- sempy_labs/lakehouse/_shortcuts.py +440 -0
- sempy_labs/migration/__init__.py +35 -0
- sempy_labs/migration/_create_pqt_file.py +238 -0
- sempy_labs/migration/_direct_lake_to_import.py +105 -0
- sempy_labs/migration/_migrate_calctables_to_lakehouse.py +398 -0
- sempy_labs/migration/_migrate_calctables_to_semantic_model.py +148 -0
- sempy_labs/migration/_migrate_model_objects_to_semantic_model.py +533 -0
- sempy_labs/migration/_migrate_tables_columns_to_semantic_model.py +172 -0
- sempy_labs/migration/_migration_validation.py +71 -0
- sempy_labs/migration/_refresh_calc_tables.py +131 -0
- sempy_labs/mirrored_azure_databricks_catalog/__init__.py +15 -0
- sempy_labs/mirrored_azure_databricks_catalog/_discover.py +213 -0
- sempy_labs/mirrored_azure_databricks_catalog/_refresh_catalog_metadata.py +45 -0
- sempy_labs/ml_model/__init__.py +23 -0
- sempy_labs/ml_model/_functions.py +427 -0
- sempy_labs/report/_BPAReportTemplate.json +232 -0
- sempy_labs/report/__init__.py +55 -0
- sempy_labs/report/_bpareporttemplate/.pbi/localSettings.json +9 -0
- sempy_labs/report/_bpareporttemplate/.platform +11 -0
- sempy_labs/report/_bpareporttemplate/StaticResources/SharedResources/BaseThemes/CY24SU06.json +710 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/01d72098bda5055bd500/page.json +11 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/01d72098bda5055bd500/visuals/1b08bce3bebabb0a27a8/visual.json +191 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/01d72098bda5055bd500/visuals/2f22ddb70c301693c165/visual.json +438 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/01d72098bda5055bd500/visuals/3b1182230aa6c600b43a/visual.json +127 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/01d72098bda5055bd500/visuals/58577ba6380c69891500/visual.json +576 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/01d72098bda5055bd500/visuals/a2a8fa5028b3b776c96c/visual.json +207 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/01d72098bda5055bd500/visuals/adfd47ef30652707b987/visual.json +506 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/01d72098bda5055bd500/visuals/b6a80ee459e716e170b1/visual.json +127 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/01d72098bda5055bd500/visuals/ce3130a721c020cc3d81/visual.json +513 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/92735ae19b31712208ad/page.json +8 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/92735ae19b31712208ad/visuals/66e60dfb526437cd78d1/visual.json +112 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/c597da16dc7e63222a82/page.json +11 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/c597da16dc7e63222a82/visuals/07deb8bce824e1be37d7/visual.json +513 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/c597da16dc7e63222a82/visuals/0b1c68838818b32ad03b/visual.json +352 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/c597da16dc7e63222a82/visuals/0c171de9d2683d10b930/visual.json +37 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/c597da16dc7e63222a82/visuals/0efa01be0510e40a645e/visual.json +542 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/c597da16dc7e63222a82/visuals/6bf2f0eb830ab53cc668/visual.json +221 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/c597da16dc7e63222a82/visuals/88d8141cb8500b60030c/visual.json +127 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/c597da16dc7e63222a82/visuals/a753273590beed656a03/visual.json +576 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/c597da16dc7e63222a82/visuals/b8fdc82cddd61ac447bc/visual.json +127 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/d37dce724a0ccc30044b/page.json +9 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/d37dce724a0ccc30044b/visuals/ce8532a7e25020271077/visual.json +38 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/pages.json +10 -0
- sempy_labs/report/_bpareporttemplate/definition/report.json +176 -0
- sempy_labs/report/_bpareporttemplate/definition/version.json +4 -0
- sempy_labs/report/_bpareporttemplate/definition.pbir +14 -0
- sempy_labs/report/_download_report.py +76 -0
- sempy_labs/report/_export_report.py +257 -0
- sempy_labs/report/_generate_report.py +427 -0
- sempy_labs/report/_paginated.py +76 -0
- sempy_labs/report/_report_bpa.py +354 -0
- sempy_labs/report/_report_bpa_rules.py +115 -0
- sempy_labs/report/_report_functions.py +581 -0
- sempy_labs/report/_report_helper.py +227 -0
- sempy_labs/report/_report_list_functions.py +110 -0
- sempy_labs/report/_report_rebind.py +149 -0
- sempy_labs/report/_reportwrapper.py +3100 -0
- sempy_labs/report/_save_report.py +147 -0
- sempy_labs/snowflake_database/__init__.py +10 -0
- sempy_labs/snowflake_database/_items.py +105 -0
- sempy_labs/sql_database/__init__.py +21 -0
- sempy_labs/sql_database/_items.py +201 -0
- sempy_labs/sql_database/_mirroring.py +79 -0
- sempy_labs/theme/__init__.py +12 -0
- sempy_labs/theme/_org_themes.py +129 -0
- sempy_labs/tom/__init__.py +3 -0
- sempy_labs/tom/_model.py +5977 -0
- sempy_labs/variable_library/__init__.py +19 -0
- sempy_labs/variable_library/_functions.py +403 -0
- sempy_labs/warehouse/__init__.py +28 -0
- sempy_labs/warehouse/_items.py +234 -0
- sempy_labs/warehouse/_restore_points.py +309 -0
sempy_labs/_ai.py
ADDED
|
@@ -0,0 +1,437 @@
|
|
|
1
|
+
import sempy
|
|
2
|
+
import sempy.fabric as fabric
|
|
3
|
+
import pandas as pd
|
|
4
|
+
from typing import List, Optional, Union
|
|
5
|
+
from IPython.display import display
|
|
6
|
+
import sempy_labs._icons as icons
|
|
7
|
+
from .._helper_functions import (
|
|
8
|
+
_read_delta_table,
|
|
9
|
+
_run_spark_sql_query,
|
|
10
|
+
)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def optimize_semantic_model(dataset: str, workspace: Optional[str] = None):
|
|
14
|
+
|
|
15
|
+
from ._model_bpa import run_model_bpa
|
|
16
|
+
from .directlake._dl_helper import check_fallback_reason
|
|
17
|
+
from ._helper_functions import format_dax_object_name
|
|
18
|
+
|
|
19
|
+
modelBPA = run_model_bpa(
|
|
20
|
+
dataset=dataset, workspace=workspace, return_dataframe=True
|
|
21
|
+
)
|
|
22
|
+
dfC = fabric.list_columns(dataset=dataset, workspace=workspace, extended=True)
|
|
23
|
+
dfC["Column Object"] = format_dax_object_name(dfC["Table Name"], dfC["Column Name"])
|
|
24
|
+
dfC["Total Size"] = dfC["Total Size"].astype("int")
|
|
25
|
+
dfP = fabric.list_partitions(dataset=dataset, workspace=workspace)
|
|
26
|
+
|
|
27
|
+
modelBPA_col = modelBPA[modelBPA["Object Type"] == "Column"]
|
|
28
|
+
modelBPA_col = pd.merge(
|
|
29
|
+
modelBPA_col,
|
|
30
|
+
dfC[["Column Object", "Total Size"]],
|
|
31
|
+
left_on="Object Name",
|
|
32
|
+
right_on="Column Object",
|
|
33
|
+
how="left",
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
isDirectLake = any(r["Mode"] == "DirectLake" for i, r in dfP.iterrows())
|
|
37
|
+
|
|
38
|
+
if isDirectLake:
|
|
39
|
+
fallback = check_fallback_reason(dataset=dataset, workspace=workspace)
|
|
40
|
+
fallback_filt = fallback[fallback["FallbackReasonID"] == 2]
|
|
41
|
+
|
|
42
|
+
if len(fallback_filt) > 0:
|
|
43
|
+
print(
|
|
44
|
+
f"{icons.yellow_dot} The '{dataset}' semantic model is a Direct Lake semantic model which contains views. "
|
|
45
|
+
"Since views always fall back to DirectQuery, it is recommended to only use lakehouse tables and not views."
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
# Potential model reduction estimate
|
|
49
|
+
ruleNames = [
|
|
50
|
+
"Remove unnecessary columns",
|
|
51
|
+
"Set IsAvailableInMdx to false on non-attribute columns",
|
|
52
|
+
]
|
|
53
|
+
|
|
54
|
+
for rule in ruleNames:
|
|
55
|
+
df = modelBPA_col[modelBPA_col["Rule Name"] == rule]
|
|
56
|
+
df_filt = df[["Object Name", "Total Size"]].sort_values(
|
|
57
|
+
by="Total Size", ascending=False
|
|
58
|
+
)
|
|
59
|
+
totSize = df["Total Size"].sum()
|
|
60
|
+
if len(df_filt) > 0:
|
|
61
|
+
print(
|
|
62
|
+
f"{icons.yellow_dot} Potential savings of {totSize} bytes from following the '{rule}' rule."
|
|
63
|
+
)
|
|
64
|
+
display(df_filt)
|
|
65
|
+
else:
|
|
66
|
+
print(f"{icons.green_dot} The '{rule}' rule has been followed.")
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def generate_aggs(
|
|
70
|
+
dataset: str,
|
|
71
|
+
table_name: str,
|
|
72
|
+
columns: Union[str, List[str]],
|
|
73
|
+
workspace: Optional[str] = None,
|
|
74
|
+
lakehouse_workspace: Optional[str] = None,
|
|
75
|
+
):
|
|
76
|
+
|
|
77
|
+
from ._helper_functions import (
|
|
78
|
+
get_direct_lake_sql_endpoint,
|
|
79
|
+
create_abfss_path,
|
|
80
|
+
format_dax_object_name,
|
|
81
|
+
resolve_lakehouse_id,
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
sempy.fabric._client._utils._init_analysis_services()
|
|
85
|
+
import Microsoft.AnalysisServices.Tabular as TOM
|
|
86
|
+
import System
|
|
87
|
+
|
|
88
|
+
# columns = {
|
|
89
|
+
# 'SalesAmount': 'Sum',
|
|
90
|
+
# 'ProductKey': 'GroupBy',
|
|
91
|
+
# 'OrderDateKey': 'GroupBy'
|
|
92
|
+
# }
|
|
93
|
+
|
|
94
|
+
if workspace is None:
|
|
95
|
+
workspace_id = fabric.get_workspace_id()
|
|
96
|
+
workspace = fabric.resolve_workspace_name(workspace_id)
|
|
97
|
+
|
|
98
|
+
if lakehouse_workspace is None:
|
|
99
|
+
lakehouse_workspace = workspace
|
|
100
|
+
lakehouse_workspace_id = workspace_id
|
|
101
|
+
else:
|
|
102
|
+
lakehouse_workspace_id = fabric.resolve_workspace_id(lakehouse_workspace)
|
|
103
|
+
|
|
104
|
+
if isinstance(columns, str):
|
|
105
|
+
columns = [columns]
|
|
106
|
+
|
|
107
|
+
columnValues = columns.keys()
|
|
108
|
+
|
|
109
|
+
aggTypes = ["Sum", "Count", "Min", "Max", "GroupBy"]
|
|
110
|
+
aggTypesAggregate = ["Sum", "Count", "Min", "Max"]
|
|
111
|
+
numericTypes = ["Int64", "Double", "Decimal"]
|
|
112
|
+
|
|
113
|
+
if any(value not in aggTypes for value in columns.values()):
|
|
114
|
+
raise ValueError(
|
|
115
|
+
f"{icons.red_dot} Invalid aggregation type(s) have been specified in the 'columns' parameter. Valid aggregation types: {aggTypes}."
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
dfC = fabric.list_columns(dataset=dataset, workspace=workspace)
|
|
119
|
+
dfP = fabric.list_partitions(dataset=dataset, workspace=workspace)
|
|
120
|
+
dfM = fabric.list_measures(dataset=dataset, workspace=workspace)
|
|
121
|
+
dfR = fabric.list_relationships(dataset=dataset, workspace=workspace)
|
|
122
|
+
if not any(r["Mode"] == "DirectLake" for i, r in dfP.iterrows()):
|
|
123
|
+
raise ValueError(
|
|
124
|
+
f"{icons.red_dot} The '{dataset}' semantic model within the '{workspace}' workspace is not in Direct Lake mode. This function is only relevant for Direct Lake semantic models."
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
dfC_filtT = dfC[dfC["Table Name"] == table_name]
|
|
128
|
+
|
|
129
|
+
if len(dfC_filtT) == 0:
|
|
130
|
+
raise ValueError(
|
|
131
|
+
f"{icons.red_dot} The '{table_name}' table does not exist in the '{dataset}' semantic model within the '{workspace}' workspace."
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
dfC_filt = dfC[
|
|
135
|
+
(dfC["Table Name"] == table_name) & (dfC["Column Name"].isin(columnValues))
|
|
136
|
+
]
|
|
137
|
+
|
|
138
|
+
if len(columns) != len(dfC_filt):
|
|
139
|
+
raise ValueError(
|
|
140
|
+
f"{icons.red_dot} Columns listed in '{columnValues}' do not exist in the '{table_name}' table in the '{dataset}' semantic model within the '{workspace}' workspace."
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
# Check if doing sum/count/min/max etc. on a non-number column
|
|
144
|
+
for cm, agg in columns.items():
|
|
145
|
+
dfC_col = dfC_filt[dfC_filt["Column Name"] == cm]
|
|
146
|
+
dataType = dfC_col["Data Type"].iloc[0]
|
|
147
|
+
if agg in aggTypesAggregate and dataType not in numericTypes:
|
|
148
|
+
raise ValueError(
|
|
149
|
+
f"{icons.red_dot} The '{cm}' column in the '{table_name}' table is of '{dataType}' data type. Only columns of '{numericTypes}' data types"
|
|
150
|
+
f" can be aggregated as '{aggTypesAggregate}' aggregation types."
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
# Create/update lakehouse delta agg table
|
|
154
|
+
aggSuffix = "_agg"
|
|
155
|
+
aggTableName = f"{table_name}{aggSuffix}"
|
|
156
|
+
aggLakeTName = aggTableName.lower().replace(" ", "_")
|
|
157
|
+
dfP = fabric.list_partitions(dataset=dataset, workspace=workspace)
|
|
158
|
+
dfP_filt = dfP[dfP["Table Name"] == table_name]
|
|
159
|
+
lakeTName = dfP_filt["Query"].iloc[0]
|
|
160
|
+
|
|
161
|
+
sqlEndpointId = get_direct_lake_sql_endpoint(dataset=dataset, workspace=workspace)
|
|
162
|
+
|
|
163
|
+
dfI = fabric.list_items(workspace=lakehouse_workspace, type="SQLEndpoint")
|
|
164
|
+
dfI_filt = dfI[(dfI["Id"] == sqlEndpointId)]
|
|
165
|
+
|
|
166
|
+
if len(dfI_filt) == 0:
|
|
167
|
+
raise ValueError(
|
|
168
|
+
f"{icons.red_dot} The lakehouse (SQL Endpoint) used by the '{dataset}' semantic model does not reside in"
|
|
169
|
+
f" the '{lakehouse_workspace}' workspace. Please update the lakehouse_workspace parameter."
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
lakehouseName = dfI_filt["Display Name"].iloc[0]
|
|
173
|
+
lakehouse_id = resolve_lakehouse_id(
|
|
174
|
+
lakehouse=lakehouseName, workspace=lakehouse_workspace
|
|
175
|
+
)
|
|
176
|
+
|
|
177
|
+
# Generate SQL query
|
|
178
|
+
query = "SELECT"
|
|
179
|
+
groupBy = "\nGROUP BY"
|
|
180
|
+
for cm, agg in columns.items():
|
|
181
|
+
colFilt = dfC_filt[dfC_filt["Column Name"] == cm]
|
|
182
|
+
sourceCol = colFilt["Source"].iloc[0]
|
|
183
|
+
|
|
184
|
+
if agg == "GroupBy":
|
|
185
|
+
query = f"{query}\n{sourceCol},"
|
|
186
|
+
groupBy = f"{groupBy}\n{sourceCol},"
|
|
187
|
+
else:
|
|
188
|
+
query = f"{query}\n{agg}({sourceCol}) AS {sourceCol},"
|
|
189
|
+
|
|
190
|
+
query = query[:-1]
|
|
191
|
+
|
|
192
|
+
fromTablePath = create_abfss_path(
|
|
193
|
+
lakehouse_id=lakehouse_id,
|
|
194
|
+
lakehouse_workspace_id=lakehouse_workspace_id,
|
|
195
|
+
delta_table_name=lakeTName,
|
|
196
|
+
)
|
|
197
|
+
|
|
198
|
+
df = _read_delta_table(fromTablePath)
|
|
199
|
+
tempTableName = f"delta_table_{lakeTName}"
|
|
200
|
+
df.createOrReplaceTempView(tempTableName)
|
|
201
|
+
sqlQuery = f"{query} \n FROM {tempTableName} {groupBy}"
|
|
202
|
+
|
|
203
|
+
sqlQuery = sqlQuery[:-1]
|
|
204
|
+
print(sqlQuery)
|
|
205
|
+
|
|
206
|
+
# Save query to spark dataframe
|
|
207
|
+
spark_df = _run_spark_sql_query(sqlQuery)
|
|
208
|
+
f"\nCreating/updating the '{aggLakeTName}' table in the lakehouse..."
|
|
209
|
+
# Write spark dataframe to delta table
|
|
210
|
+
aggFilePath = create_abfss_path(
|
|
211
|
+
lakehouse_id=lakehouse_id,
|
|
212
|
+
lakehouse_workspace_id=lakehouse_workspace_id,
|
|
213
|
+
delta_table_name=aggLakeTName,
|
|
214
|
+
)
|
|
215
|
+
spark_df.write.mode("overwrite").format("delta").save(aggFilePath)
|
|
216
|
+
f"{icons.green_dot} The '{aggLakeTName}' table has been created/updated in the lakehouse."
|
|
217
|
+
|
|
218
|
+
# Create/update semantic model agg table
|
|
219
|
+
tom_server = fabric.create_tom_server(
|
|
220
|
+
dataset=dataset, readonly=False, workspace=workspace
|
|
221
|
+
)
|
|
222
|
+
m = tom_server.Databases.GetByName(dataset).Model
|
|
223
|
+
print(f"\n{icons.in_progress} Updating the '{dataset}' semantic model...")
|
|
224
|
+
dfC_agg = dfC[dfC["Table Name"] == aggTableName]
|
|
225
|
+
|
|
226
|
+
if len(dfC_agg) == 0:
|
|
227
|
+
print(f"{icons.in_progress} Creating the '{aggTableName}' table...")
|
|
228
|
+
exp = m.Expressions["DatabaseQuery"]
|
|
229
|
+
tbl = TOM.Table()
|
|
230
|
+
tbl.Name = aggTableName
|
|
231
|
+
tbl.IsHidden = True
|
|
232
|
+
|
|
233
|
+
ep = TOM.EntityPartitionSource()
|
|
234
|
+
ep.Name = aggTableName
|
|
235
|
+
ep.EntityName = aggLakeTName
|
|
236
|
+
ep.ExpressionSource = exp
|
|
237
|
+
|
|
238
|
+
part = TOM.Partition()
|
|
239
|
+
part.Name = aggTableName
|
|
240
|
+
part.Source = ep
|
|
241
|
+
part.Mode = TOM.ModeType.DirectLake
|
|
242
|
+
|
|
243
|
+
tbl.Partitions.Add(part)
|
|
244
|
+
|
|
245
|
+
for i, r in dfC_filt.iterrows():
|
|
246
|
+
scName = r["Source"]
|
|
247
|
+
cName = r["Column Name"]
|
|
248
|
+
dType = r["Data Type"]
|
|
249
|
+
|
|
250
|
+
col = TOM.DataColumn()
|
|
251
|
+
col.Name = cName
|
|
252
|
+
col.IsHidden = True
|
|
253
|
+
col.SourceColumn = scName
|
|
254
|
+
col.DataType = System.Enum.Parse(TOM.DataType, dType)
|
|
255
|
+
|
|
256
|
+
tbl.Columns.Add(col)
|
|
257
|
+
print(
|
|
258
|
+
f"{icons.green_dot} The '{aggTableName}'[{cName}] column has been added to the '{dataset}' semantic model."
|
|
259
|
+
)
|
|
260
|
+
|
|
261
|
+
m.Tables.Add(tbl)
|
|
262
|
+
print(
|
|
263
|
+
f"{icons.green_dot} The '{aggTableName}' table has been added to the '{dataset}' semantic model."
|
|
264
|
+
)
|
|
265
|
+
else:
|
|
266
|
+
print(f"{icons.in_progress} Updating the '{aggTableName}' table's columns...")
|
|
267
|
+
# Remove existing columns
|
|
268
|
+
for t in m.Tables:
|
|
269
|
+
tName = t.Name
|
|
270
|
+
for c in t.Columns:
|
|
271
|
+
cName = c.Name
|
|
272
|
+
if t.Name == aggTableName:
|
|
273
|
+
m.Tables[tName].Columns.Remove(cName)
|
|
274
|
+
# Add columns
|
|
275
|
+
for i, r in dfC_filt.iterrows():
|
|
276
|
+
scName = r["Source"]
|
|
277
|
+
cName = r["Column Name"]
|
|
278
|
+
dType = r["Data Type"]
|
|
279
|
+
|
|
280
|
+
col = TOM.DataColumn()
|
|
281
|
+
col.Name = cName
|
|
282
|
+
col.IsHidden = True
|
|
283
|
+
col.SourceColumn = scName
|
|
284
|
+
col.DataType = System.Enum.Parse(TOM.DataType, dType)
|
|
285
|
+
|
|
286
|
+
m.Tables[aggTableName].Columns.Add(col)
|
|
287
|
+
print(
|
|
288
|
+
f"{icons.green_dot} The '{aggTableName}'[{cName}] column has been added."
|
|
289
|
+
)
|
|
290
|
+
|
|
291
|
+
# Create relationships
|
|
292
|
+
relMap = {"m": "Many", "1": "One", "0": "None"}
|
|
293
|
+
|
|
294
|
+
print(f"\n{icons.in_progress} Generating necessary relationships...")
|
|
295
|
+
for i, r in dfR.iterrows():
|
|
296
|
+
fromTable = r["From Table"]
|
|
297
|
+
fromColumn = r["From Column"]
|
|
298
|
+
toTable = r["To Table"]
|
|
299
|
+
toColumn = r["To Column"]
|
|
300
|
+
cfb = r["Cross Filtering Behavior"]
|
|
301
|
+
sfb = r["Security Filtering Behavior"]
|
|
302
|
+
mult = r["Multiplicity"]
|
|
303
|
+
|
|
304
|
+
crossFB = System.Enum.Parse(TOM.CrossFilteringBehavior, cfb)
|
|
305
|
+
secFB = System.Enum.Parse(TOM.SecurityFilteringBehavior, sfb)
|
|
306
|
+
fromCardinality = System.Enum.Parse(
|
|
307
|
+
TOM.RelationshipEndCardinality, relMap.get(mult[0])
|
|
308
|
+
)
|
|
309
|
+
toCardinality = System.Enum.Parse(
|
|
310
|
+
TOM.RelationshipEndCardinality, relMap.get(mult[-1])
|
|
311
|
+
)
|
|
312
|
+
|
|
313
|
+
rel = TOM.SingleColumnRelationship()
|
|
314
|
+
rel.FromCardinality = fromCardinality
|
|
315
|
+
rel.ToCardinality = toCardinality
|
|
316
|
+
rel.IsActive = r["Active"]
|
|
317
|
+
rel.CrossFilteringBehavior = crossFB
|
|
318
|
+
rel.SecurityFilteringBehavior = secFB
|
|
319
|
+
rel.RelyOnReferentialIntegrity = r["Rely On Referential Integrity"]
|
|
320
|
+
|
|
321
|
+
if fromTable == table_name:
|
|
322
|
+
try:
|
|
323
|
+
rel.FromColumn = m.Tables[aggTableName].Columns[fromColumn]
|
|
324
|
+
m.Relationships.Add(rel)
|
|
325
|
+
print(
|
|
326
|
+
f"{icons.green_dot} '{aggTableName}'[{fromColumn}] -> '{toTable}'[{toColumn}] relationship has been added."
|
|
327
|
+
)
|
|
328
|
+
except Exception as e:
|
|
329
|
+
print(
|
|
330
|
+
f"{icons.red_dot} '{aggTableName}'[{fromColumn}] -> '{toTable}'[{toColumn}] relationship has not been created."
|
|
331
|
+
)
|
|
332
|
+
print(f"Exception occured: {e}")
|
|
333
|
+
elif toTable == table_name:
|
|
334
|
+
try:
|
|
335
|
+
rel.ToColumn = m.Tables[aggTableName].Columns[toColumn]
|
|
336
|
+
m.Relationships.Add(rel)
|
|
337
|
+
print(
|
|
338
|
+
f"{icons.green_dot} '{fromTable}'[{fromColumn}] -> '{aggTableName}'[{toColumn}] relationship has been added."
|
|
339
|
+
)
|
|
340
|
+
except Exception as e:
|
|
341
|
+
print(
|
|
342
|
+
f"{icons.red_dot} '{fromTable}'[{fromColumn}] -> '{aggTableName}'[{toColumn}] relationship has not been created."
|
|
343
|
+
)
|
|
344
|
+
print(f"Exception occured: {e}")
|
|
345
|
+
"Relationship creation is complete."
|
|
346
|
+
|
|
347
|
+
# Create IF measure
|
|
348
|
+
f"\n{icons.in_progress} Creating measure to check if the agg table can be used..."
|
|
349
|
+
aggChecker = "IF("
|
|
350
|
+
dfR_filt = dfR[
|
|
351
|
+
(dfR["From Table"] == table_name) & (~dfR["From Column"].isin(columnValues))
|
|
352
|
+
]
|
|
353
|
+
|
|
354
|
+
for i, r in dfR_filt.iterrows():
|
|
355
|
+
toTable = r["To Table"]
|
|
356
|
+
aggChecker = f"{aggChecker}\nISCROSSFILTERED('{toTable}') ||"
|
|
357
|
+
|
|
358
|
+
aggChecker = aggChecker[:-3]
|
|
359
|
+
aggChecker = f"{aggChecker},1,0)"
|
|
360
|
+
print(aggChecker)
|
|
361
|
+
|
|
362
|
+
# Todo: add IFISFILTERED clause for columns
|
|
363
|
+
f"\n{icons.in_progress} Creating the base measures in the agg table..."
|
|
364
|
+
# Create base agg measures
|
|
365
|
+
dep = fabric.evaluate_dax(
|
|
366
|
+
dataset=dataset,
|
|
367
|
+
workspace=workspace,
|
|
368
|
+
dax_string="""
|
|
369
|
+
SELECT
|
|
370
|
+
[TABLE] AS [Table Name]
|
|
371
|
+
,[OBJECT] AS [Object Name]
|
|
372
|
+
,[OBJECT_TYPE] AS [Object Type]
|
|
373
|
+
,[REFERENCED_TABLE] AS [Referenced Table]
|
|
374
|
+
,[REFERENCED_OBJECT] AS [Referenced Object]
|
|
375
|
+
,[REFERENCED_OBJECT_TYPE] AS [Referenced Object Type]
|
|
376
|
+
FROM $SYSTEM.DISCOVER_CALC_DEPENDENCY
|
|
377
|
+
WHERE [OBJECT_TYPE] = 'MEASURE'
|
|
378
|
+
""",
|
|
379
|
+
)
|
|
380
|
+
|
|
381
|
+
baseMeasures = dep[
|
|
382
|
+
(dep["Referenced Object Type"] == "COLUMN")
|
|
383
|
+
& (dep["Referenced Table"] == table_name)
|
|
384
|
+
& (dep["Referenced Object"].isin(columnValues))
|
|
385
|
+
]
|
|
386
|
+
for i, r in baseMeasures.iterrows():
|
|
387
|
+
tName = r["Table Name"]
|
|
388
|
+
mName = r["Object Name"]
|
|
389
|
+
cName = r["Referenced Object"]
|
|
390
|
+
dfM_filt = dfM[dfM["Measure Name"] == mName]
|
|
391
|
+
expr = dfM_filt["Measure Expression"].iloc[0]
|
|
392
|
+
|
|
393
|
+
colFQNonAgg = format_dax_object_name(tName, cName)
|
|
394
|
+
colFQAgg = format_dax_object_name(aggTableName, cName)
|
|
395
|
+
colNQNonAgg = f"{tName}[{cName}]"
|
|
396
|
+
|
|
397
|
+
if " " in tName:
|
|
398
|
+
newExpr = expr.replace(colFQNonAgg, colFQAgg)
|
|
399
|
+
else:
|
|
400
|
+
newExpr = expr.replace(colFQNonAgg, colFQAgg).replace(colNQNonAgg, colFQAgg)
|
|
401
|
+
print(expr)
|
|
402
|
+
print(newExpr)
|
|
403
|
+
|
|
404
|
+
aggMName = f"{mName}{aggSuffix}"
|
|
405
|
+
measure = TOM.Measure()
|
|
406
|
+
measure.Name = aggMName
|
|
407
|
+
measure.IsHidden = True
|
|
408
|
+
measure.Expression = newExpr
|
|
409
|
+
m.Tables[aggTableName].Measures.Add(measure)
|
|
410
|
+
f"The '{aggMName}' measure has been created in the '{aggTableName}' table."
|
|
411
|
+
|
|
412
|
+
# Update base detail measures
|
|
413
|
+
|
|
414
|
+
# m.SaveChanges()
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
# Identify views used within Direct Lake model
|
|
418
|
+
# workspace = 'MK Demo 6'
|
|
419
|
+
# lakehouse = 'MyLakehouse'
|
|
420
|
+
# dataset = 'MigrationTest'
|
|
421
|
+
# lakehouse_workspace = workspace
|
|
422
|
+
|
|
423
|
+
# dfView = pd.DataFrame(columns=['Workspace Name', 'Lakehouse Name', 'View Name'])
|
|
424
|
+
# dfP = fabric.list_partitions(dataset = dataset, workspace = workspace)
|
|
425
|
+
# isDirectLake = any(r['Mode'] == 'DirectLake' for i, r in dfP.iterrows())
|
|
426
|
+
|
|
427
|
+
# spark = _create_spark_session()
|
|
428
|
+
# views = spark.sql(f"SHOW VIEWS IN {lakehouse}").collect()
|
|
429
|
+
# for view in views:
|
|
430
|
+
# viewName = view['viewName']
|
|
431
|
+
# isTemporary = view['isTemporary']
|
|
432
|
+
# new_data = {'Workspace Name': workspace, 'Lakehouse Name': lakehouse, 'View Name': viewName}
|
|
433
|
+
# dfView = pd.concat([dfView, pd.DataFrame(new_data, index=[0])], ignore_index=True)
|
|
434
|
+
# dfView
|
|
435
|
+
# lakeT = get_lakehouse_tables(lakehouse, lakehouse_workspace)
|
|
436
|
+
# if not dfP['Query'].isin(lakeT['Table Name'].values):
|
|
437
|
+
# if
|
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
from typing import Dict, Literal, Optional
|
|
2
|
+
from azure.core.credentials import AccessToken, TokenCredential
|
|
3
|
+
from azure.identity import ClientSecretCredential
|
|
4
|
+
from sempy._utils._log import log
|
|
5
|
+
from contextlib import contextmanager
|
|
6
|
+
import contextvars
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class ServicePrincipalTokenProvider(TokenCredential):
|
|
10
|
+
"""
|
|
11
|
+
A class to acquire authentication token with Service Principal.
|
|
12
|
+
|
|
13
|
+
For more information on Service Principal see: `Application and service principal objects in Microsoft Entra ID <https://learn.microsoft.com/en-us/entra/identity-platform/app-objects-and-service-principals?tabs=browser#service-principal-object>`_
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
_shorthand_scopes: Dict[str, str] = {
|
|
17
|
+
"pbi": "https://analysis.windows.net/powerbi/api/.default",
|
|
18
|
+
"storage": "https://storage.azure.com/.default",
|
|
19
|
+
"azure": "https://management.azure.com/.default",
|
|
20
|
+
"graph": "https://graph.microsoft.com/.default",
|
|
21
|
+
"asazure": "https://{region}.asazure.windows.net/.default",
|
|
22
|
+
"keyvault": "https://vault.azure.net/.default",
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
def __init__(self, credential: ClientSecretCredential):
|
|
26
|
+
|
|
27
|
+
self.credential = credential
|
|
28
|
+
|
|
29
|
+
@classmethod
|
|
30
|
+
def from_aad_application_key_authentication(
|
|
31
|
+
cls, tenant_id: str, client_id: str, client_secret: str
|
|
32
|
+
) -> "ServicePrincipalTokenProvider":
|
|
33
|
+
"""
|
|
34
|
+
Generates the ServicePrincipalTokenProvider, providing the Service Principal information.
|
|
35
|
+
|
|
36
|
+
***USE THIS ONE ONLY FOR TEST PURPOSE. FOR PRODUCTION WE RECOMMEND CALLING ServicePrincipalTokenProvider.from_azure_key_vault()***
|
|
37
|
+
|
|
38
|
+
Parameters
|
|
39
|
+
----------
|
|
40
|
+
tenant_id : str
|
|
41
|
+
The Fabric Tenant ID.
|
|
42
|
+
client_id : str
|
|
43
|
+
The Service Principal Application Client ID.
|
|
44
|
+
client_secret : str
|
|
45
|
+
The Service Principal Client Secret.
|
|
46
|
+
|
|
47
|
+
Returns
|
|
48
|
+
-------
|
|
49
|
+
ServicePrincipalTokenProvider
|
|
50
|
+
Token provider to be used with FabricRestClient or PowerBIRestClient.
|
|
51
|
+
"""
|
|
52
|
+
credential = ClientSecretCredential(
|
|
53
|
+
tenant_id=tenant_id, client_id=client_id, client_secret=client_secret
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
cls.tenant_id = tenant_id
|
|
57
|
+
cls.client_id = client_id
|
|
58
|
+
cls.client_secret = client_secret
|
|
59
|
+
|
|
60
|
+
return cls(credential)
|
|
61
|
+
|
|
62
|
+
@classmethod
|
|
63
|
+
def from_azure_key_vault(
|
|
64
|
+
cls,
|
|
65
|
+
key_vault_uri: str,
|
|
66
|
+
key_vault_tenant_id: str,
|
|
67
|
+
key_vault_client_id: str,
|
|
68
|
+
key_vault_client_secret: str,
|
|
69
|
+
) -> "ServicePrincipalTokenProvider":
|
|
70
|
+
"""
|
|
71
|
+
Generates the ServicePrincipalTokenProvider, providing the Azure Key Vault details.
|
|
72
|
+
|
|
73
|
+
For more information on Azure Key Vault, `click here <https://learn.microsoft.com/en-us/azure/key-vault/general/overview>`_.
|
|
74
|
+
|
|
75
|
+
Parameters
|
|
76
|
+
----------
|
|
77
|
+
key_vault_uri : str
|
|
78
|
+
Azure Key Vault URI.
|
|
79
|
+
key_vault_tenant_id : str
|
|
80
|
+
Name of the secret in the Key Vault with the Fabric Tenant ID.
|
|
81
|
+
key_vault_client_id : str
|
|
82
|
+
Name of the secret in the Key Vault with the Service Principal Client ID.
|
|
83
|
+
key_vault_client_secret : str
|
|
84
|
+
Name of the secret in the Key Vault with the Service Principal Client Secret.
|
|
85
|
+
|
|
86
|
+
Returns
|
|
87
|
+
-------
|
|
88
|
+
ServicePrincipalTokenProvider
|
|
89
|
+
Token provider to be used with FabricRestClient or PowerBIRestClient.
|
|
90
|
+
"""
|
|
91
|
+
|
|
92
|
+
import notebookutils
|
|
93
|
+
|
|
94
|
+
tenant_id = notebookutils.credentials.getSecret(
|
|
95
|
+
key_vault_uri, key_vault_tenant_id
|
|
96
|
+
)
|
|
97
|
+
client_id = notebookutils.credentials.getSecret(
|
|
98
|
+
key_vault_uri, key_vault_client_id
|
|
99
|
+
)
|
|
100
|
+
client_secret = notebookutils.credentials.getSecret(
|
|
101
|
+
key_vault_uri, key_vault_client_secret
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
credential = ClientSecretCredential(
|
|
105
|
+
tenant_id=tenant_id, client_id=client_id, client_secret=client_secret
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
cls.tenant_id = tenant_id
|
|
109
|
+
cls.client_id = client_id
|
|
110
|
+
cls.client_secret = client_secret
|
|
111
|
+
|
|
112
|
+
return cls(credential)
|
|
113
|
+
|
|
114
|
+
def __call__(
|
|
115
|
+
self,
|
|
116
|
+
audience: Literal[
|
|
117
|
+
"pbi", "storage", "azure", "graph", "asazure", "keyvault"
|
|
118
|
+
] = "pbi",
|
|
119
|
+
region: Optional[str] = None,
|
|
120
|
+
) -> str:
|
|
121
|
+
"""
|
|
122
|
+
Parameters
|
|
123
|
+
----------
|
|
124
|
+
audience : Literal["pbi", "storage", "azure", "graph", "asazure", "keyvault"] = "pbi") -> str
|
|
125
|
+
Literal if it's for PBI/Fabric API call or OneLake/Storage Account call.
|
|
126
|
+
region : str, default=None
|
|
127
|
+
The region of the Azure Analysis Services. For example: 'westus2'.
|
|
128
|
+
"""
|
|
129
|
+
# Check if audience is supported
|
|
130
|
+
if audience not in self._shorthand_scopes:
|
|
131
|
+
raise NotImplementedError
|
|
132
|
+
|
|
133
|
+
return self.get_token(audience, region=region).token
|
|
134
|
+
|
|
135
|
+
def get_token(self, *scopes, **kwargs) -> AccessToken:
|
|
136
|
+
"""
|
|
137
|
+
Gets a token for the specified scopes.
|
|
138
|
+
|
|
139
|
+
Parameters
|
|
140
|
+
----------
|
|
141
|
+
*scopes : str
|
|
142
|
+
The scopes for which to obtain a token.
|
|
143
|
+
**kwargs : dict
|
|
144
|
+
Additional parameters to pass to the token request.
|
|
145
|
+
|
|
146
|
+
Returns
|
|
147
|
+
-------
|
|
148
|
+
AccessToken
|
|
149
|
+
The access token.
|
|
150
|
+
"""
|
|
151
|
+
if len(scopes) == 0:
|
|
152
|
+
scopes = ("pbi",)
|
|
153
|
+
|
|
154
|
+
region = kwargs.pop("region", None)
|
|
155
|
+
scopes = [
|
|
156
|
+
self._get_fully_qualified_scope(scope, region=region) for scope in scopes
|
|
157
|
+
]
|
|
158
|
+
return self.credential.get_token(*scopes, **kwargs)
|
|
159
|
+
|
|
160
|
+
def _get_fully_qualified_scope(
|
|
161
|
+
self, scope: str, region: Optional[str] = None
|
|
162
|
+
) -> str:
|
|
163
|
+
"""
|
|
164
|
+
Resolve to fully qualified scope if Fabric short-handed scope is given.
|
|
165
|
+
Otherwise, return the original scope.
|
|
166
|
+
|
|
167
|
+
Parameters
|
|
168
|
+
----------
|
|
169
|
+
scope : str
|
|
170
|
+
The scope to resolve.
|
|
171
|
+
region : str, default=None
|
|
172
|
+
The specific region to use to resolve scope.
|
|
173
|
+
Required if scope is "asazure".
|
|
174
|
+
|
|
175
|
+
Returns
|
|
176
|
+
-------
|
|
177
|
+
str
|
|
178
|
+
The resolved scope.
|
|
179
|
+
"""
|
|
180
|
+
fully_qualified_scope = self._shorthand_scopes.get(scope, scope)
|
|
181
|
+
|
|
182
|
+
if scope == "asazure":
|
|
183
|
+
if region is None:
|
|
184
|
+
raise ValueError("Region is required for 'asazure' scope")
|
|
185
|
+
return fully_qualified_scope.format(region=region)
|
|
186
|
+
|
|
187
|
+
return fully_qualified_scope
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def _get_headers(
|
|
191
|
+
token_provider: TokenCredential,
|
|
192
|
+
audience: Literal[
|
|
193
|
+
"pbi", "storage", "azure", "graph", "asazure", "keyvault"
|
|
194
|
+
] = "azure",
|
|
195
|
+
):
|
|
196
|
+
"""
|
|
197
|
+
Generates headers for an API request.
|
|
198
|
+
"""
|
|
199
|
+
|
|
200
|
+
token = token_provider.get_token(audience).token
|
|
201
|
+
|
|
202
|
+
headers = {"Authorization": f"Bearer {token}"}
|
|
203
|
+
|
|
204
|
+
if audience == "graph":
|
|
205
|
+
headers["ConsistencyLevel"] = "eventual"
|
|
206
|
+
else:
|
|
207
|
+
headers["Content-Type"] = "application/json"
|
|
208
|
+
|
|
209
|
+
return headers
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
token_provider = contextvars.ContextVar("token_provider", default=None)
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
@log
|
|
216
|
+
@contextmanager
|
|
217
|
+
def service_principal_authentication(
|
|
218
|
+
key_vault_uri: str,
|
|
219
|
+
key_vault_tenant_id: str,
|
|
220
|
+
key_vault_client_id: str,
|
|
221
|
+
key_vault_client_secret: str,
|
|
222
|
+
):
|
|
223
|
+
"""
|
|
224
|
+
Establishes an authentication via Service Principal.
|
|
225
|
+
|
|
226
|
+
Parameters
|
|
227
|
+
----------
|
|
228
|
+
key_vault_uri : str
|
|
229
|
+
Azure Key Vault URI.
|
|
230
|
+
key_vault_tenant_id : str
|
|
231
|
+
Name of the secret in the Key Vault with the Fabric Tenant ID.
|
|
232
|
+
key_vault_client_id : str
|
|
233
|
+
Name of the secret in the Key Vault with the Service Principal Client ID.
|
|
234
|
+
key_vault_client_secret : str
|
|
235
|
+
Name of the secret in the Key Vault with the Service Principal Client Secret.
|
|
236
|
+
"""
|
|
237
|
+
|
|
238
|
+
# Save the prior state
|
|
239
|
+
prior_token = token_provider.get()
|
|
240
|
+
|
|
241
|
+
# Set the new token_provider in a thread-safe manner
|
|
242
|
+
token_provider.set(
|
|
243
|
+
ServicePrincipalTokenProvider.from_azure_key_vault(
|
|
244
|
+
key_vault_uri=key_vault_uri,
|
|
245
|
+
key_vault_tenant_id=key_vault_tenant_id,
|
|
246
|
+
key_vault_client_id=key_vault_client_id,
|
|
247
|
+
key_vault_client_secret=key_vault_client_secret,
|
|
248
|
+
)
|
|
249
|
+
)
|
|
250
|
+
try:
|
|
251
|
+
from sempy.fabric import set_service_principal
|
|
252
|
+
|
|
253
|
+
with set_service_principal(
|
|
254
|
+
(key_vault_uri, key_vault_tenant_id),
|
|
255
|
+
(key_vault_uri, key_vault_client_id),
|
|
256
|
+
client_secret=(key_vault_uri, key_vault_client_secret),
|
|
257
|
+
):
|
|
258
|
+
yield
|
|
259
|
+
finally:
|
|
260
|
+
# Restore the prior state
|
|
261
|
+
if prior_token is None:
|
|
262
|
+
token_provider.set(None)
|
|
263
|
+
else:
|
|
264
|
+
token_provider.set(prior_token)
|