semantic-link-labs 0.5.0__py3-none-any.whl → 0.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of semantic-link-labs might be problematic. Click here for more details.
- semantic_link_labs-0.7.0.dist-info/METADATA +148 -0
- semantic_link_labs-0.7.0.dist-info/RECORD +111 -0
- {semantic_link_labs-0.5.0.dist-info → semantic_link_labs-0.7.0.dist-info}/WHEEL +1 -1
- sempy_labs/__init__.py +45 -15
- sempy_labs/_ai.py +42 -85
- sempy_labs/_bpa_translation/_translations_am-ET.po +828 -0
- sempy_labs/_bpa_translation/_translations_ar-AE.po +860 -0
- sempy_labs/_bpa_translation/_translations_cs-CZ.po +894 -0
- sempy_labs/_bpa_translation/_translations_da-DK.po +894 -0
- sempy_labs/_bpa_translation/_translations_de-DE.po +933 -0
- sempy_labs/_bpa_translation/_translations_el-GR.po +936 -0
- sempy_labs/_bpa_translation/_translations_es-ES.po +915 -0
- sempy_labs/_bpa_translation/_translations_fa-IR.po +883 -0
- sempy_labs/_bpa_translation/_translations_fr-FR.po +938 -0
- sempy_labs/_bpa_translation/_translations_ga-IE.po +912 -0
- sempy_labs/_bpa_translation/_translations_he-IL.po +855 -0
- sempy_labs/_bpa_translation/_translations_hi-IN.po +892 -0
- sempy_labs/_bpa_translation/_translations_hu-HU.po +910 -0
- sempy_labs/_bpa_translation/_translations_is-IS.po +887 -0
- sempy_labs/_bpa_translation/_translations_it-IT.po +931 -0
- sempy_labs/_bpa_translation/_translations_ja-JP.po +805 -0
- sempy_labs/_bpa_translation/_translations_nl-NL.po +924 -0
- sempy_labs/_bpa_translation/_translations_pl-PL.po +913 -0
- sempy_labs/_bpa_translation/_translations_pt-BR.po +909 -0
- sempy_labs/_bpa_translation/_translations_pt-PT.po +904 -0
- sempy_labs/_bpa_translation/_translations_ru-RU.po +909 -0
- sempy_labs/_bpa_translation/_translations_ta-IN.po +922 -0
- sempy_labs/_bpa_translation/_translations_te-IN.po +896 -0
- sempy_labs/_bpa_translation/_translations_th-TH.po +873 -0
- sempy_labs/_bpa_translation/_translations_zh-CN.po +767 -0
- sempy_labs/_bpa_translation/_translations_zu-ZA.po +916 -0
- sempy_labs/_clear_cache.py +12 -8
- sempy_labs/_connections.py +77 -70
- sempy_labs/_dax.py +7 -9
- sempy_labs/_generate_semantic_model.py +75 -90
- sempy_labs/_helper_functions.py +371 -20
- sempy_labs/_icons.py +23 -0
- sempy_labs/_list_functions.py +855 -427
- sempy_labs/_model_auto_build.py +4 -3
- sempy_labs/_model_bpa.py +307 -1118
- sempy_labs/_model_bpa_bulk.py +363 -0
- sempy_labs/_model_bpa_rules.py +831 -0
- sempy_labs/_model_dependencies.py +20 -16
- sempy_labs/_one_lake_integration.py +18 -12
- sempy_labs/_query_scale_out.py +116 -129
- sempy_labs/_refresh_semantic_model.py +23 -10
- sempy_labs/_translations.py +367 -288
- sempy_labs/_vertipaq.py +152 -123
- sempy_labs/directlake/__init__.py +7 -1
- sempy_labs/directlake/_directlake_schema_compare.py +33 -30
- sempy_labs/directlake/_directlake_schema_sync.py +60 -77
- sempy_labs/directlake/_dl_helper.py +233 -0
- sempy_labs/directlake/_get_directlake_lakehouse.py +7 -8
- sempy_labs/directlake/_get_shared_expression.py +5 -3
- sempy_labs/directlake/_guardrails.py +20 -16
- sempy_labs/directlake/_list_directlake_model_calc_tables.py +17 -10
- sempy_labs/directlake/_show_unsupported_directlake_objects.py +3 -2
- sempy_labs/directlake/_update_directlake_model_lakehouse_connection.py +10 -5
- sempy_labs/directlake/_update_directlake_partition_entity.py +169 -22
- sempy_labs/directlake/_warm_cache.py +7 -4
- sempy_labs/lakehouse/_get_lakehouse_columns.py +1 -1
- sempy_labs/lakehouse/_get_lakehouse_tables.py +65 -71
- sempy_labs/lakehouse/_lakehouse.py +5 -3
- sempy_labs/lakehouse/_shortcuts.py +20 -13
- sempy_labs/migration/__init__.py +1 -1
- sempy_labs/migration/_create_pqt_file.py +184 -186
- sempy_labs/migration/_migrate_calctables_to_lakehouse.py +240 -269
- sempy_labs/migration/_migrate_calctables_to_semantic_model.py +78 -77
- sempy_labs/migration/_migrate_model_objects_to_semantic_model.py +444 -425
- sempy_labs/migration/_migrate_tables_columns_to_semantic_model.py +96 -102
- sempy_labs/migration/_migration_validation.py +2 -2
- sempy_labs/migration/_refresh_calc_tables.py +94 -100
- sempy_labs/report/_BPAReportTemplate.json +232 -0
- sempy_labs/report/__init__.py +6 -2
- sempy_labs/report/_bpareporttemplate/.pbi/localSettings.json +9 -0
- sempy_labs/report/_bpareporttemplate/.platform +11 -0
- sempy_labs/report/_bpareporttemplate/StaticResources/SharedResources/BaseThemes/CY24SU06.json +710 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/01d72098bda5055bd500/page.json +11 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/01d72098bda5055bd500/visuals/1b08bce3bebabb0a27a8/visual.json +191 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/01d72098bda5055bd500/visuals/2f22ddb70c301693c165/visual.json +438 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/01d72098bda5055bd500/visuals/3b1182230aa6c600b43a/visual.json +127 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/01d72098bda5055bd500/visuals/58577ba6380c69891500/visual.json +576 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/01d72098bda5055bd500/visuals/a2a8fa5028b3b776c96c/visual.json +207 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/01d72098bda5055bd500/visuals/adfd47ef30652707b987/visual.json +506 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/01d72098bda5055bd500/visuals/b6a80ee459e716e170b1/visual.json +127 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/01d72098bda5055bd500/visuals/ce3130a721c020cc3d81/visual.json +513 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/92735ae19b31712208ad/page.json +8 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/92735ae19b31712208ad/visuals/66e60dfb526437cd78d1/visual.json +112 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/c597da16dc7e63222a82/page.json +11 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/c597da16dc7e63222a82/visuals/07deb8bce824e1be37d7/visual.json +513 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/c597da16dc7e63222a82/visuals/0b1c68838818b32ad03b/visual.json +352 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/c597da16dc7e63222a82/visuals/0c171de9d2683d10b930/visual.json +37 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/c597da16dc7e63222a82/visuals/0efa01be0510e40a645e/visual.json +542 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/c597da16dc7e63222a82/visuals/6bf2f0eb830ab53cc668/visual.json +221 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/c597da16dc7e63222a82/visuals/88d8141cb8500b60030c/visual.json +127 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/c597da16dc7e63222a82/visuals/a753273590beed656a03/visual.json +576 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/c597da16dc7e63222a82/visuals/b8fdc82cddd61ac447bc/visual.json +127 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/d37dce724a0ccc30044b/page.json +9 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/d37dce724a0ccc30044b/visuals/ce8532a7e25020271077/visual.json +38 -0
- sempy_labs/report/_bpareporttemplate/definition/pages/pages.json +10 -0
- sempy_labs/report/_bpareporttemplate/definition/report.json +176 -0
- sempy_labs/report/_bpareporttemplate/definition/version.json +4 -0
- sempy_labs/report/_bpareporttemplate/definition.pbir +14 -0
- sempy_labs/report/_generate_report.py +260 -139
- sempy_labs/report/_report_functions.py +90 -59
- sempy_labs/report/_report_rebind.py +40 -34
- sempy_labs/tom/__init__.py +1 -4
- sempy_labs/tom/_model.py +601 -181
- semantic_link_labs-0.5.0.dist-info/METADATA +0 -22
- semantic_link_labs-0.5.0.dist-info/RECORD +0 -53
- sempy_labs/directlake/_fallback.py +0 -58
- {semantic_link_labs-0.5.0.dist-info → semantic_link_labs-0.7.0.dist-info}/LICENSE +0 -0
- {semantic_link_labs-0.5.0.dist-info → semantic_link_labs-0.7.0.dist-info}/top_level.txt +0 -0
|
@@ -1,24 +1,19 @@
|
|
|
1
1
|
import sempy
|
|
2
2
|
import sempy.fabric as fabric
|
|
3
|
-
|
|
4
|
-
from sempy_labs.
|
|
3
|
+
from sempy_labs.lakehouse import get_lakehouse_columns
|
|
4
|
+
from sempy_labs.directlake._dl_helper import get_direct_lake_source
|
|
5
5
|
from sempy_labs.tom import connect_semantic_model
|
|
6
|
-
from sempy_labs._helper_functions import (
|
|
7
|
-
format_dax_object_name,
|
|
8
|
-
resolve_lakehouse_name,
|
|
9
|
-
get_direct_lake_sql_endpoint,
|
|
10
|
-
)
|
|
11
6
|
from typing import Optional
|
|
12
7
|
from sempy._utils._log import log
|
|
13
8
|
import sempy_labs._icons as icons
|
|
14
9
|
|
|
10
|
+
|
|
15
11
|
@log
|
|
16
12
|
def direct_lake_schema_sync(
|
|
17
13
|
dataset: str,
|
|
18
14
|
workspace: Optional[str] = None,
|
|
19
15
|
add_to_model: Optional[bool] = False,
|
|
20
|
-
|
|
21
|
-
lakehouse_workspace: Optional[str] = None,
|
|
16
|
+
**kwargs,
|
|
22
17
|
):
|
|
23
18
|
"""
|
|
24
19
|
Shows/adds columns which exist in the lakehouse but do not exist in the semantic model (only for tables in the semantic model).
|
|
@@ -33,91 +28,79 @@ def direct_lake_schema_sync(
|
|
|
33
28
|
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
34
29
|
add_to_model : bool, default=False
|
|
35
30
|
If set to True, columns which exist in the lakehouse but do not exist in the semantic model are added to the semantic model. No new tables are added.
|
|
36
|
-
lakehouse : str, default=None
|
|
37
|
-
The Fabric lakehouse used by the Direct Lake semantic model.
|
|
38
|
-
Defaults to None which resolves to the lakehouse attached to the notebook.
|
|
39
|
-
lakehouse_workspace : str, default=None
|
|
40
|
-
The Fabric workspace used by the lakehouse.
|
|
41
|
-
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
42
|
-
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
43
31
|
"""
|
|
44
32
|
|
|
45
33
|
sempy.fabric._client._utils._init_analysis_services()
|
|
46
34
|
import Microsoft.AnalysisServices.Tabular as TOM
|
|
47
35
|
import System
|
|
48
36
|
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
if
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
37
|
+
if "lakehouse" in kwargs:
|
|
38
|
+
print(
|
|
39
|
+
"The 'lakehouse' parameter has been deprecated as it is no longer necessary. Please remove this parameter from the function going forward."
|
|
40
|
+
)
|
|
41
|
+
del kwargs["lakehouse"]
|
|
42
|
+
if "lakehouse_workspace" in kwargs:
|
|
43
|
+
print(
|
|
44
|
+
"The 'lakehouse_workspace' parameter has been deprecated as it is no longer necessary. Please remove this parameter from the function going forward."
|
|
45
|
+
)
|
|
46
|
+
del kwargs["lakehouse_workspace"]
|
|
59
47
|
|
|
60
|
-
|
|
61
|
-
dfI_filt = dfI[(dfI["Id"] == sqlEndpointId)]
|
|
62
|
-
|
|
63
|
-
if len(dfI_filt) == 0:
|
|
64
|
-
raise ValueError(f"{icons.red_dot} The SQL Endpoint in the '{dataset}' semantic model in the '{workspace} workspace does not point to the '{lakehouse}' lakehouse in the '{lakehouse_workspace}' workspace as specified.")
|
|
48
|
+
workspace = fabric.resolve_workspace_name(workspace)
|
|
65
49
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
dfC = fabric.list_columns(dataset=dataset, workspace=workspace)
|
|
69
|
-
dfC_filt = dfC[dfC["Table Name"].isin(dfP_filt["Table Name"].values)]
|
|
70
|
-
dfC_filt = pd.merge(
|
|
71
|
-
dfC_filt, dfP_filt[["Table Name", "Query"]], on="Table Name", how="left"
|
|
72
|
-
)
|
|
73
|
-
dfC_filt["Column Object"] = format_dax_object_name(
|
|
74
|
-
dfC_filt["Query"], dfC_filt["Source"]
|
|
50
|
+
artifact_type, lakehouse_name, lakehouse_id, lakehouse_workspace_id = (
|
|
51
|
+
get_direct_lake_source(dataset=dataset, workspace=workspace)
|
|
75
52
|
)
|
|
76
53
|
|
|
77
|
-
|
|
78
|
-
|
|
54
|
+
if artifact_type == "Warehouse":
|
|
55
|
+
raise ValueError(
|
|
56
|
+
f"{icons.red_dot} This function is only valid for Direct Lake semantic models which source from Fabric lakehouses (not warehouses)."
|
|
57
|
+
)
|
|
58
|
+
lakehouse_workspace = fabric.resolve_workspace_name(lakehouse_workspace_id)
|
|
59
|
+
|
|
60
|
+
if artifact_type == "Warehouse":
|
|
61
|
+
raise ValueError(
|
|
62
|
+
f"{icons.red_dot} This function is only valid for Direct Lake semantic models which source from Fabric lakehouses (not warehouses)."
|
|
63
|
+
)
|
|
79
64
|
|
|
80
|
-
|
|
81
|
-
"string": "String",
|
|
82
|
-
"bigint": "Int64",
|
|
83
|
-
"int": "Int64",
|
|
84
|
-
"smallint": "Int64",
|
|
85
|
-
"boolean": "Boolean",
|
|
86
|
-
"timestamp": "DateTime",
|
|
87
|
-
"date": "DateTime",
|
|
88
|
-
"decimal(38,18)": "Decimal",
|
|
89
|
-
"double": "Double",
|
|
90
|
-
}
|
|
65
|
+
lc = get_lakehouse_columns(lakehouse_name, lakehouse_workspace)
|
|
91
66
|
|
|
92
67
|
with connect_semantic_model(
|
|
93
|
-
|
|
94
|
-
|
|
68
|
+
dataset=dataset, readonly=False, workspace=workspace
|
|
69
|
+
) as tom:
|
|
95
70
|
|
|
96
|
-
for i, r in
|
|
71
|
+
for i, r in lc.iterrows():
|
|
97
72
|
lakeTName = r["Table Name"]
|
|
98
73
|
lakeCName = r["Column Name"]
|
|
99
|
-
fullColName = r["Full Column Name"]
|
|
100
74
|
dType = r["Data Type"]
|
|
101
75
|
|
|
102
|
-
if
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
if
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
else:
|
|
76
|
+
if any(
|
|
77
|
+
p.Source.EntityName == lakeTName
|
|
78
|
+
for p in tom.all_partitions()
|
|
79
|
+
if p.SourceType == TOM.PartitionSourceType.Entity
|
|
80
|
+
):
|
|
81
|
+
table_name = next(
|
|
82
|
+
t.Name
|
|
83
|
+
for t in tom.model.Tables
|
|
84
|
+
for p in t.Partitions
|
|
85
|
+
if p.SourceType == TOM.PartitionSourceType.Entity
|
|
86
|
+
and p.Source.EntityName == lakeTName
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
if not any(
|
|
90
|
+
c.SourceColumn == lakeCName and c.Parent.Name == table_name
|
|
91
|
+
for c in tom.all_columns()
|
|
92
|
+
):
|
|
120
93
|
print(
|
|
121
|
-
f"{icons.yellow_dot} The {
|
|
94
|
+
f"{icons.yellow_dot} The '{lakeCName}' column exists in the '{lakeTName}' lakehouse table but not in the '{dataset}' semantic model within the '{workspace}' workspace."
|
|
122
95
|
)
|
|
123
|
-
|
|
96
|
+
if add_to_model:
|
|
97
|
+
dt = icons.data_type_mapping.get(dType)
|
|
98
|
+
tom.add_data_column(
|
|
99
|
+
table_name=table_name,
|
|
100
|
+
column_name=lakeCName,
|
|
101
|
+
source_column=lakeCName,
|
|
102
|
+
data_type=System.Enum.Parse(TOM.DataType, dt),
|
|
103
|
+
)
|
|
104
|
+
print(
|
|
105
|
+
f"{icons.green_dot} The '{lakeCName}' column in the '{lakeTName}' lakehouse table was added to the '{dataset}' semantic model within the '{workspace}' workspace."
|
|
106
|
+
)
|
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
import sempy.fabric as fabric
|
|
2
|
+
import numpy as np
|
|
3
|
+
import pandas as pd
|
|
4
|
+
from typing import Optional, List, Union, Tuple
|
|
5
|
+
from uuid import UUID
|
|
6
|
+
import sempy_labs._icons as icons
|
|
7
|
+
from sempy._utils._log import log
|
|
8
|
+
from sempy_labs._helper_functions import retry, resolve_dataset_id
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def check_fallback_reason(
|
|
12
|
+
dataset: str, workspace: Optional[str] = None
|
|
13
|
+
) -> pd.DataFrame:
|
|
14
|
+
"""
|
|
15
|
+
Shows the reason a table in a Direct Lake semantic model would fallback to DirectQuery.
|
|
16
|
+
|
|
17
|
+
Parameters
|
|
18
|
+
----------
|
|
19
|
+
dataset : str
|
|
20
|
+
Name of the semantic model.
|
|
21
|
+
workspace : str, default=None
|
|
22
|
+
The Fabric workspace name.
|
|
23
|
+
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
24
|
+
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
25
|
+
|
|
26
|
+
Returns
|
|
27
|
+
-------
|
|
28
|
+
pandas.DataFrame
|
|
29
|
+
The tables in the semantic model and their fallback reason.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
workspace = fabric.resolve_workspace_name(workspace)
|
|
33
|
+
|
|
34
|
+
dfP = fabric.list_partitions(dataset=dataset, workspace=workspace)
|
|
35
|
+
dfP_filt = dfP[dfP["Mode"] == "DirectLake"]
|
|
36
|
+
|
|
37
|
+
if len(dfP_filt) == 0:
|
|
38
|
+
raise ValueError(
|
|
39
|
+
f"{icons.red_dot} The '{dataset}' semantic model is not in Direct Lake. This function is only applicable to Direct Lake semantic models."
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
df = fabric.evaluate_dax(
|
|
43
|
+
dataset=dataset,
|
|
44
|
+
workspace=workspace,
|
|
45
|
+
dax_string="""
|
|
46
|
+
SELECT [TableName] AS [Table Name],[FallbackReason] AS [FallbackReasonID]
|
|
47
|
+
FROM $SYSTEM.TMSCHEMA_DELTA_TABLE_METADATA_STORAGES
|
|
48
|
+
""",
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
value_mapping = {
|
|
52
|
+
0: "No reason for fallback",
|
|
53
|
+
1: "This table is not framed",
|
|
54
|
+
2: "This object is a view in the lakehouse",
|
|
55
|
+
3: "The table does not exist in the lakehouse",
|
|
56
|
+
4: "Transient error",
|
|
57
|
+
5: "Using OLS will result in fallback to DQ",
|
|
58
|
+
6: "Using RLS will result in fallback to DQ",
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
# Create a new column based on the mapping
|
|
62
|
+
df["Fallback Reason Detail"] = np.vectorize(value_mapping.get)(
|
|
63
|
+
df["FallbackReasonID"]
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
return df
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@log
|
|
70
|
+
def generate_direct_lake_semantic_model(
|
|
71
|
+
dataset: str,
|
|
72
|
+
lakehouse_tables: Union[str, List[str]],
|
|
73
|
+
workspace: Optional[str] = None,
|
|
74
|
+
lakehouse: Optional[str] = None,
|
|
75
|
+
lakehouse_workspace: Optional[str] = None,
|
|
76
|
+
overwrite: Optional[bool] = False,
|
|
77
|
+
refresh: Optional[bool] = True,
|
|
78
|
+
):
|
|
79
|
+
"""
|
|
80
|
+
Dynamically generates a Direct Lake semantic model based on tables in a Fabric lakehouse.
|
|
81
|
+
|
|
82
|
+
Parameters
|
|
83
|
+
----------
|
|
84
|
+
dataset : str
|
|
85
|
+
Name of the semantic model to be created.
|
|
86
|
+
lakehouse_tables : str | List[str]
|
|
87
|
+
The table(s) within the Fabric lakehouse to add to the semantic model. All columns from these tables will be added to the semantic model.
|
|
88
|
+
workspace : str, default=None
|
|
89
|
+
The Fabric workspace name in which the semantic model will reside.
|
|
90
|
+
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
91
|
+
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
92
|
+
lakehouse : str, default=None
|
|
93
|
+
The lakehouse which stores the delta tables which will feed the Direct Lake semantic model.
|
|
94
|
+
Defaults to None which resolves to the attached lakehouse.
|
|
95
|
+
lakehouse_workspace : str, default=None
|
|
96
|
+
The Fabric workspace in which the lakehouse resides.
|
|
97
|
+
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
98
|
+
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
99
|
+
overwrite : bool, default=False
|
|
100
|
+
If set to True, overwrites the existing semantic model if it already exists.
|
|
101
|
+
refresh: bool, default=True
|
|
102
|
+
If True, refreshes the newly created semantic model after it is created.
|
|
103
|
+
|
|
104
|
+
Returns
|
|
105
|
+
-------
|
|
106
|
+
"""
|
|
107
|
+
|
|
108
|
+
from sempy_labs.lakehouse import get_lakehouse_tables, get_lakehouse_columns
|
|
109
|
+
from sempy_labs import create_blank_semantic_model, refresh_semantic_model
|
|
110
|
+
from sempy_labs.tom import connect_semantic_model
|
|
111
|
+
from sempy_labs.directlake import get_shared_expression
|
|
112
|
+
|
|
113
|
+
if isinstance(lakehouse_tables, str):
|
|
114
|
+
lakehouse_tables = [lakehouse_tables]
|
|
115
|
+
|
|
116
|
+
dfLT = get_lakehouse_tables(lakehouse=lakehouse, workspace=lakehouse_workspace)
|
|
117
|
+
|
|
118
|
+
# Validate lakehouse tables
|
|
119
|
+
for t in lakehouse_tables:
|
|
120
|
+
if t not in dfLT["Table Name"].values:
|
|
121
|
+
raise ValueError(
|
|
122
|
+
f"{icons.red_dot} The '{t}' table does not exist as a delta table in the '{lakehouse}' within the '{workspace}' workspace."
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
dfLC = get_lakehouse_columns(lakehouse=lakehouse, workspace=lakehouse_workspace)
|
|
126
|
+
expr = get_shared_expression(lakehouse=lakehouse, workspace=lakehouse_workspace)
|
|
127
|
+
dfD = fabric.list_datasets(workspace=workspace)
|
|
128
|
+
dfD_filt = dfD[dfD["Dataset Name"] == dataset]
|
|
129
|
+
dfD_filt_len = len(dfD_filt)
|
|
130
|
+
|
|
131
|
+
if dfD_filt_len > 0 and overwrite is False:
|
|
132
|
+
raise ValueError(
|
|
133
|
+
f"{icons.red_dot} The '{dataset}' semantic model within the '{workspace}' workspace already exists. Overwrite is set to False so the new semantic model has not been created."
|
|
134
|
+
)
|
|
135
|
+
if dfD_filt_len > 0 and overwrite:
|
|
136
|
+
print(
|
|
137
|
+
f"{icons.warning} Overwriting the existing '{dataset}' semantic model within the '{workspace}' workspace."
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
create_blank_semantic_model(dataset=dataset, workspace=workspace)
|
|
141
|
+
|
|
142
|
+
@retry(
|
|
143
|
+
sleep_time=1,
|
|
144
|
+
timeout_error_message=f"{icons.red_dot} Function timed out after 1 minute",
|
|
145
|
+
)
|
|
146
|
+
def dyn_connect():
|
|
147
|
+
with connect_semantic_model(
|
|
148
|
+
dataset=dataset, readonly=True, workspace=workspace
|
|
149
|
+
) as tom:
|
|
150
|
+
|
|
151
|
+
tom.model
|
|
152
|
+
|
|
153
|
+
dyn_connect()
|
|
154
|
+
|
|
155
|
+
expression_name = "DatabaseQuery"
|
|
156
|
+
with connect_semantic_model(
|
|
157
|
+
dataset=dataset, workspace=workspace, readonly=False
|
|
158
|
+
) as tom:
|
|
159
|
+
if not any(e.Name == expression_name for e in tom.model.Expressions):
|
|
160
|
+
tom.add_expression(name=expression_name, expression=expr)
|
|
161
|
+
|
|
162
|
+
for t in lakehouse_tables:
|
|
163
|
+
tom.add_table(name=t)
|
|
164
|
+
tom.add_entity_partition(table_name=t, entity_name=t)
|
|
165
|
+
dfLC_filt = dfLC[dfLC["Table Name"] == t]
|
|
166
|
+
for i, r in dfLC_filt.iterrows():
|
|
167
|
+
lakeCName = r["Column Name"]
|
|
168
|
+
dType = r["Data Type"]
|
|
169
|
+
dt = icons.data_type_mapping.get(dType)
|
|
170
|
+
tom.add_data_column(
|
|
171
|
+
table_name=t,
|
|
172
|
+
column_name=lakeCName,
|
|
173
|
+
source_column=lakeCName,
|
|
174
|
+
data_type=dt,
|
|
175
|
+
)
|
|
176
|
+
|
|
177
|
+
if refresh:
|
|
178
|
+
refresh_semantic_model(dataset=dataset, workspace=workspace)
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def get_direct_lake_source(
|
|
182
|
+
dataset: str, workspace: Optional[str] = None
|
|
183
|
+
) -> Tuple[str, str, UUID, UUID]:
|
|
184
|
+
"""
|
|
185
|
+
Obtains the source information for a direct lake semantic model.
|
|
186
|
+
|
|
187
|
+
Parameters
|
|
188
|
+
----------
|
|
189
|
+
dataset : str
|
|
190
|
+
The name of the semantic model.
|
|
191
|
+
workspace : str, default=None
|
|
192
|
+
The Fabric workspace name.
|
|
193
|
+
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
194
|
+
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
195
|
+
|
|
196
|
+
Returns
|
|
197
|
+
-------
|
|
198
|
+
Tuple[str, str, UUID, UUID]
|
|
199
|
+
If the source of the direct lake semantic model is a lakehouse this will return: 'Lakehouse', Lakehouse Name, SQL Endpoint Id, Workspace Id
|
|
200
|
+
If the source of the direct lake semantic model is a warehouse this will return: 'Warehouse', Warehouse Name, Warehouse Id, Workspace Id
|
|
201
|
+
If the semantic model is not a Direct Lake semantic model, it will return None, None, None.
|
|
202
|
+
"""
|
|
203
|
+
|
|
204
|
+
workspace = fabric.resolve_workspace_name(workspace)
|
|
205
|
+
dataset_id = resolve_dataset_id(dataset, workspace)
|
|
206
|
+
client = fabric.PowerBIRestClient()
|
|
207
|
+
request_body = {
|
|
208
|
+
"artifacts": [
|
|
209
|
+
{
|
|
210
|
+
"objectId": dataset_id,
|
|
211
|
+
"type": "dataset",
|
|
212
|
+
}
|
|
213
|
+
]
|
|
214
|
+
}
|
|
215
|
+
response = client.post(
|
|
216
|
+
"metadata/relations/upstream?apiVersion=3", json=request_body
|
|
217
|
+
)
|
|
218
|
+
artifacts = response.json().get("artifacts", [])
|
|
219
|
+
sql_id, sql_object_name, sql_workspace_id, artifact_type = None, None, None, None
|
|
220
|
+
|
|
221
|
+
for artifact in artifacts:
|
|
222
|
+
object_type = artifact.get("typeName")
|
|
223
|
+
display_name = artifact.get("displayName")
|
|
224
|
+
if object_type in ["Datawarehouse", "Lakewarehouse"]:
|
|
225
|
+
artifact_type = (
|
|
226
|
+
"Warehouse" if object_type == "Datawarehouse" else "Lakehouse"
|
|
227
|
+
)
|
|
228
|
+
sql_id = artifact.get("objectId")
|
|
229
|
+
sql_workspace_id = artifact.get("workspace", {}).get("objectId")
|
|
230
|
+
sql_object_name = display_name
|
|
231
|
+
break
|
|
232
|
+
|
|
233
|
+
return artifact_type, sql_object_name, sql_id, sql_workspace_id
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import sempy
|
|
2
1
|
import sempy.fabric as fabric
|
|
3
2
|
from sempy_labs._helper_functions import (
|
|
4
3
|
resolve_lakehouse_id,
|
|
@@ -7,7 +6,7 @@ from sempy_labs._helper_functions import (
|
|
|
7
6
|
)
|
|
8
7
|
from typing import Optional, Tuple
|
|
9
8
|
from uuid import UUID
|
|
10
|
-
|
|
9
|
+
|
|
11
10
|
|
|
12
11
|
def get_direct_lake_lakehouse(
|
|
13
12
|
dataset: str,
|
|
@@ -49,13 +48,13 @@ def get_direct_lake_lakehouse(
|
|
|
49
48
|
lakehouse_id = fabric.get_lakehouse_id()
|
|
50
49
|
lakehouse = resolve_lakehouse_name(lakehouse_id, lakehouse_workspace)
|
|
51
50
|
|
|
52
|
-
dfP = fabric.list_partitions(dataset=dataset, workspace=workspace)
|
|
53
|
-
dfP_filt = dfP[dfP["Mode"] == "DirectLake"]
|
|
51
|
+
# dfP = fabric.list_partitions(dataset=dataset, workspace=workspace)
|
|
52
|
+
# dfP_filt = dfP[dfP["Mode"] == "DirectLake"]
|
|
54
53
|
|
|
55
|
-
if len(dfP_filt) == 0:
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
54
|
+
# if len(dfP_filt) == 0:
|
|
55
|
+
# raise ValueError(
|
|
56
|
+
# f"{icons.red_dot} The '{dataset}' semantic model within the '{workspace}' workspace is not in Direct Lake mode."
|
|
57
|
+
# )
|
|
59
58
|
|
|
60
59
|
sqlEndpointId = get_direct_lake_sql_endpoint(dataset, workspace)
|
|
61
60
|
|
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
import sempy
|
|
2
1
|
import sempy.fabric as fabric
|
|
3
2
|
from sempy_labs._helper_functions import resolve_lakehouse_name
|
|
4
3
|
from sempy_labs._list_functions import list_lakehouses
|
|
5
4
|
from typing import Optional
|
|
6
5
|
import sempy_labs._icons as icons
|
|
7
6
|
|
|
7
|
+
|
|
8
8
|
def get_shared_expression(
|
|
9
9
|
lakehouse: Optional[str] = None, workspace: Optional[str] = None
|
|
10
|
-
):
|
|
10
|
+
) -> str:
|
|
11
11
|
"""
|
|
12
12
|
Dynamically generates the M expression used by a Direct Lake model for a given lakehouse.
|
|
13
13
|
|
|
@@ -40,7 +40,9 @@ def get_shared_expression(
|
|
|
40
40
|
provStatus = lakeDetail["SQL Endpoint Provisioning Status"].iloc[0]
|
|
41
41
|
|
|
42
42
|
if provStatus == "InProgress":
|
|
43
|
-
raise ValueError(
|
|
43
|
+
raise ValueError(
|
|
44
|
+
f"{icons.red_dot} The SQL Endpoint for the '{lakehouse}' lakehouse within the '{workspace}' workspace has not yet been provisioned. Please wait until it has been provisioned."
|
|
45
|
+
)
|
|
44
46
|
|
|
45
47
|
sh = (
|
|
46
48
|
'let\n\tdatabase = Sql.Database("'
|
|
@@ -1,12 +1,13 @@
|
|
|
1
|
-
import sempy
|
|
2
1
|
import sempy.fabric as fabric
|
|
3
2
|
import pandas as pd
|
|
4
|
-
from typing import
|
|
3
|
+
from typing import Optional
|
|
4
|
+
import sempy_labs._icons as icons
|
|
5
5
|
|
|
6
6
|
|
|
7
7
|
def get_direct_lake_guardrails() -> pd.DataFrame:
|
|
8
8
|
"""
|
|
9
|
-
Shows the guardrails for when Direct Lake semantic models will fallback to Direct Query
|
|
9
|
+
Shows the guardrails for when Direct Lake semantic models will fallback to Direct Query
|
|
10
|
+
based on Microsoft's `online documentation <https://learn.microsoft.com/power-bi/enterprise/directlake-overview>`_.
|
|
10
11
|
|
|
11
12
|
Parameters
|
|
12
13
|
----------
|
|
@@ -27,14 +28,14 @@ def get_direct_lake_guardrails() -> pd.DataFrame:
|
|
|
27
28
|
return df
|
|
28
29
|
|
|
29
30
|
|
|
30
|
-
def get_sku_size(workspace: Optional[str] = None):
|
|
31
|
+
def get_sku_size(workspace: Optional[str] = None) -> str:
|
|
31
32
|
"""
|
|
32
33
|
Shows the SKU size for a workspace.
|
|
33
34
|
|
|
34
35
|
Parameters
|
|
35
36
|
----------
|
|
36
37
|
workspace : str, default=None
|
|
37
|
-
The Fabric workspace.
|
|
38
|
+
The Fabric workspace name.
|
|
38
39
|
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
39
40
|
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
40
41
|
|
|
@@ -46,18 +47,21 @@ def get_sku_size(workspace: Optional[str] = None):
|
|
|
46
47
|
|
|
47
48
|
workspace = fabric.resolve_workspace_name(workspace)
|
|
48
49
|
|
|
50
|
+
dfW = fabric.list_workspaces(filter=f"name eq '{workspace}'")
|
|
51
|
+
|
|
52
|
+
if len(dfW) == 0:
|
|
53
|
+
raise ValueError(f"{icons.red_dot} The '{workspace}' is not a valid workspace.")
|
|
54
|
+
|
|
55
|
+
capacity_id = dfW["Capacity Id"].iloc[0]
|
|
49
56
|
dfC = fabric.list_capacities()
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
sku_value = dfCW.loc[dfCW["Name"] == workspace, "Sku"].iloc[0]
|
|
59
|
-
|
|
60
|
-
return sku_value
|
|
57
|
+
dfC_filt = dfC[dfC["Id"] == capacity_id]
|
|
58
|
+
|
|
59
|
+
if len(dfC_filt) == 0:
|
|
60
|
+
raise ValueError(
|
|
61
|
+
f"{icons.red_dot} The '{capacity_id}' Id is not a valid capacity Id."
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
return dfC_filt["Sku"].iloc[0]
|
|
61
65
|
|
|
62
66
|
|
|
63
67
|
def get_directlake_guardrails_for_sku(sku_size: str) -> pd.DataFrame:
|
|
@@ -1,14 +1,16 @@
|
|
|
1
|
-
import sempy
|
|
2
1
|
import sempy.fabric as fabric
|
|
3
2
|
import pandas as pd
|
|
4
|
-
from sempy_labs._list_functions import list_tables
|
|
3
|
+
from sempy_labs._list_functions import list_tables
|
|
5
4
|
from sempy_labs.tom import connect_semantic_model
|
|
6
5
|
from typing import Optional
|
|
7
6
|
from sempy._utils._log import log
|
|
8
7
|
import sempy_labs._icons as icons
|
|
9
8
|
|
|
9
|
+
|
|
10
10
|
@log
|
|
11
|
-
def list_direct_lake_model_calc_tables(
|
|
11
|
+
def list_direct_lake_model_calc_tables(
|
|
12
|
+
dataset: str, workspace: Optional[str] = None
|
|
13
|
+
) -> pd.DataFrame:
|
|
12
14
|
"""
|
|
13
15
|
Shows the calculated tables and their respective DAX expression for a Direct Lake model (which has been migrated from import/DirectQuery).
|
|
14
16
|
|
|
@@ -32,18 +34,21 @@ def list_direct_lake_model_calc_tables(dataset: str, workspace: Optional[str] =
|
|
|
32
34
|
df = pd.DataFrame(columns=["Table Name", "Source Expression"])
|
|
33
35
|
|
|
34
36
|
with connect_semantic_model(
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
37
|
+
dataset=dataset, readonly=True, workspace=workspace
|
|
38
|
+
) as tom:
|
|
39
|
+
|
|
38
40
|
is_direct_lake = tom.is_direct_lake()
|
|
39
41
|
|
|
40
42
|
if not is_direct_lake:
|
|
41
|
-
raise ValueError(
|
|
43
|
+
raise ValueError(
|
|
44
|
+
f"{icons.red_dot} The '{dataset}' semantic model is not in Direct Lake mode."
|
|
45
|
+
)
|
|
42
46
|
else:
|
|
43
|
-
dfA = list_annotations(dataset, workspace)
|
|
47
|
+
dfA = fabric.list_annotations(dataset=dataset, workspace=workspace)
|
|
44
48
|
dfT = list_tables(dataset, workspace)
|
|
45
49
|
dfA_filt = dfA[
|
|
46
|
-
(dfA["Object Type"] == "Model")
|
|
50
|
+
(dfA["Object Type"] == "Model")
|
|
51
|
+
& (dfA["Annotation Name"].isin(dfT["Name"]))
|
|
47
52
|
]
|
|
48
53
|
|
|
49
54
|
for i, r in dfA_filt.iterrows():
|
|
@@ -51,6 +56,8 @@ def list_direct_lake_model_calc_tables(dataset: str, workspace: Optional[str] =
|
|
|
51
56
|
se = r["Annotation Value"]
|
|
52
57
|
|
|
53
58
|
new_data = {"Table Name": tName, "Source Expression": se}
|
|
54
|
-
df = pd.concat(
|
|
59
|
+
df = pd.concat(
|
|
60
|
+
[df, pd.DataFrame(new_data, index=[0])], ignore_index=True
|
|
61
|
+
)
|
|
55
62
|
|
|
56
63
|
return df
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import sempy
|
|
2
1
|
import sempy.fabric as fabric
|
|
3
2
|
import pandas as pd
|
|
4
3
|
from sempy_labs._list_functions import list_tables
|
|
@@ -6,12 +5,14 @@ from sempy_labs._helper_functions import format_dax_object_name
|
|
|
6
5
|
from typing import Optional, Tuple
|
|
7
6
|
from sempy._utils._log import log
|
|
8
7
|
|
|
8
|
+
|
|
9
9
|
@log
|
|
10
10
|
def show_unsupported_direct_lake_objects(
|
|
11
11
|
dataset: str, workspace: Optional[str] = None
|
|
12
12
|
) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
|
|
13
13
|
"""
|
|
14
|
-
Returns a list of a semantic model's objects which are not supported by Direct Lake based on
|
|
14
|
+
Returns a list of a semantic model's objects which are not supported by Direct Lake based on
|
|
15
|
+
`official documentation <https://learn.microsoft.com/power-bi/enterprise/directlake-overview#known-issues-and-limitations>`_.
|
|
15
16
|
|
|
16
17
|
Parameters
|
|
17
18
|
----------
|
|
@@ -1,9 +1,7 @@
|
|
|
1
|
-
import sempy
|
|
2
1
|
import sempy.fabric as fabric
|
|
3
2
|
from sempy_labs.directlake._get_shared_expression import get_shared_expression
|
|
4
3
|
from sempy_labs._helper_functions import (
|
|
5
4
|
resolve_lakehouse_name,
|
|
6
|
-
resolve_workspace_name_and_id,
|
|
7
5
|
)
|
|
8
6
|
from sempy_labs.tom import connect_semantic_model
|
|
9
7
|
from typing import Optional
|
|
@@ -54,13 +52,18 @@ def update_direct_lake_model_lakehouse_connection(
|
|
|
54
52
|
dfI_filt = dfI[(dfI["Display Name"] == lakehouse)]
|
|
55
53
|
|
|
56
54
|
if len(dfI_filt) == 0:
|
|
57
|
-
raise ValueError(
|
|
55
|
+
raise ValueError(
|
|
56
|
+
f"{icons.red_dot} The '{lakehouse}' lakehouse does not exist within the '{lakehouse_workspace}' workspace. "
|
|
57
|
+
f"Therefore it cannot be used to support the '{dataset}' semantic model within the '{workspace}' workspace."
|
|
58
|
+
)
|
|
58
59
|
|
|
59
60
|
dfP = fabric.list_partitions(dataset=dataset, workspace=workspace)
|
|
60
61
|
dfP_filt = dfP[dfP["Mode"] == "DirectLake"]
|
|
61
62
|
|
|
62
63
|
if len(dfP_filt) == 0:
|
|
63
|
-
raise ValueError(
|
|
64
|
+
raise ValueError(
|
|
65
|
+
f"{icons.red_dot} The '{dataset}' semantic model is not in Direct Lake. This function is only applicable to Direct Lake semantic models."
|
|
66
|
+
)
|
|
64
67
|
else:
|
|
65
68
|
with connect_semantic_model(
|
|
66
69
|
dataset=dataset, readonly=False, workspace=workspace
|
|
@@ -73,4 +76,6 @@ def update_direct_lake_model_lakehouse_connection(
|
|
|
73
76
|
f"{icons.green_dot} The expression in the '{dataset}' semantic model has been updated to point to the '{lakehouse}' lakehouse in the '{lakehouse_workspace}' workspace."
|
|
74
77
|
)
|
|
75
78
|
except Exception as e:
|
|
76
|
-
raise ValueError(
|
|
79
|
+
raise ValueError(
|
|
80
|
+
f"{icons.red_dot} The expression in the '{dataset}' semantic model was not updated."
|
|
81
|
+
) from e
|