semantic-link-labs 0.4.1__py3-none-any.whl → 0.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of semantic-link-labs might be problematic. Click here for more details.
- {semantic_link_labs-0.4.1.dist-info → semantic_link_labs-0.5.0.dist-info}/METADATA +1 -1
- semantic_link_labs-0.5.0.dist-info/RECORD +53 -0
- {semantic_link_labs-0.4.1.dist-info → semantic_link_labs-0.5.0.dist-info}/WHEEL +1 -1
- sempy_labs/__init__.py +51 -27
- sempy_labs/_ai.py +32 -51
- sempy_labs/_clear_cache.py +2 -3
- sempy_labs/_connections.py +39 -38
- sempy_labs/_dax.py +5 -9
- sempy_labs/_generate_semantic_model.py +15 -21
- sempy_labs/_helper_functions.py +20 -25
- sempy_labs/_icons.py +6 -0
- sempy_labs/_list_functions.py +1172 -392
- sempy_labs/_model_auto_build.py +3 -5
- sempy_labs/_model_bpa.py +20 -24
- sempy_labs/_model_dependencies.py +7 -14
- sempy_labs/_one_lake_integration.py +14 -24
- sempy_labs/_query_scale_out.py +13 -31
- sempy_labs/_refresh_semantic_model.py +8 -18
- sempy_labs/_translations.py +5 -5
- sempy_labs/_vertipaq.py +11 -18
- sempy_labs/directlake/_directlake_schema_compare.py +11 -15
- sempy_labs/directlake/_directlake_schema_sync.py +35 -40
- sempy_labs/directlake/_fallback.py +3 -7
- sempy_labs/directlake/_get_directlake_lakehouse.py +3 -4
- sempy_labs/directlake/_get_shared_expression.py +5 -11
- sempy_labs/directlake/_guardrails.py +5 -7
- sempy_labs/directlake/_list_directlake_model_calc_tables.py +28 -26
- sempy_labs/directlake/_show_unsupported_directlake_objects.py +3 -4
- sempy_labs/directlake/_update_directlake_model_lakehouse_connection.py +11 -16
- sempy_labs/directlake/_update_directlake_partition_entity.py +25 -15
- sempy_labs/directlake/_warm_cache.py +10 -15
- sempy_labs/lakehouse/__init__.py +0 -2
- sempy_labs/lakehouse/_get_lakehouse_columns.py +4 -3
- sempy_labs/lakehouse/_get_lakehouse_tables.py +12 -11
- sempy_labs/lakehouse/_lakehouse.py +6 -7
- sempy_labs/lakehouse/_shortcuts.py +10 -111
- sempy_labs/migration/__init__.py +4 -2
- sempy_labs/migration/_create_pqt_file.py +5 -14
- sempy_labs/migration/_migrate_calctables_to_lakehouse.py +7 -7
- sempy_labs/migration/_migrate_calctables_to_semantic_model.py +4 -4
- sempy_labs/migration/_migrate_model_objects_to_semantic_model.py +3 -8
- sempy_labs/migration/_migrate_tables_columns_to_semantic_model.py +6 -6
- sempy_labs/migration/_migration_validation.py +5 -164
- sempy_labs/migration/_refresh_calc_tables.py +5 -5
- sempy_labs/report/__init__.py +2 -2
- sempy_labs/report/_generate_report.py +14 -19
- sempy_labs/report/_report_functions.py +41 -83
- sempy_labs/report/_report_rebind.py +43 -44
- sempy_labs/tom/__init__.py +6 -0
- sempy_labs/{_tom.py → tom/_model.py} +274 -337
- semantic_link_labs-0.4.1.dist-info/RECORD +0 -52
- {semantic_link_labs-0.4.1.dist-info → semantic_link_labs-0.5.0.dist-info}/LICENSE +0 -0
- {semantic_link_labs-0.4.1.dist-info → semantic_link_labs-0.5.0.dist-info}/top_level.txt +0 -0
|
@@ -10,8 +10,10 @@ from IPython.display import display
|
|
|
10
10
|
from sempy_labs.lakehouse._get_lakehouse_columns import get_lakehouse_columns
|
|
11
11
|
from sempy_labs._list_functions import list_tables
|
|
12
12
|
from typing import Optional
|
|
13
|
+
import sempy_labs._icons as icons
|
|
14
|
+
from sempy._utils._log import log
|
|
13
15
|
|
|
14
|
-
|
|
16
|
+
@log
|
|
15
17
|
def direct_lake_schema_compare(
|
|
16
18
|
dataset: str,
|
|
17
19
|
workspace: Optional[str] = None,
|
|
@@ -38,14 +40,12 @@ def direct_lake_schema_compare(
|
|
|
38
40
|
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
39
41
|
"""
|
|
40
42
|
|
|
41
|
-
|
|
42
|
-
workspace_id = fabric.get_workspace_id()
|
|
43
|
-
workspace = fabric.resolve_workspace_name(workspace_id)
|
|
43
|
+
workspace = fabric.resolve_workspace_name(workspace)
|
|
44
44
|
|
|
45
45
|
if lakehouse_workspace is None:
|
|
46
46
|
lakehouse_workspace = workspace
|
|
47
47
|
|
|
48
|
-
if lakehouse
|
|
48
|
+
if lakehouse is None:
|
|
49
49
|
lakehouse_id = fabric.get_lakehouse_id()
|
|
50
50
|
lakehouse = resolve_lakehouse_name(lakehouse_id, lakehouse_workspace)
|
|
51
51
|
|
|
@@ -55,14 +55,10 @@ def direct_lake_schema_compare(
|
|
|
55
55
|
dfI_filt = dfI[(dfI["Id"] == sqlEndpointId)]
|
|
56
56
|
|
|
57
57
|
if len(dfI_filt) == 0:
|
|
58
|
-
|
|
59
|
-
f"The SQL Endpoint in the '{dataset}' semantic model in the '{workspace} workspace does not point to the '{lakehouse}' lakehouse in the '{lakehouse_workspace}' workspace as specified."
|
|
60
|
-
)
|
|
61
|
-
return
|
|
58
|
+
raise ValueError(f"{icons.red_dot} The SQL Endpoint in the '{dataset}' semantic model in the '{workspace} workspace does not point to the '{lakehouse}' lakehouse in the '{lakehouse_workspace}' workspace as specified.")
|
|
62
59
|
|
|
63
60
|
if not any(r["Mode"] == "DirectLake" for i, r in dfP.iterrows()):
|
|
64
|
-
|
|
65
|
-
return
|
|
61
|
+
raise ValueError(f"{icons.red_dot} The '{dataset}' semantic model is not in Direct Lake mode.")
|
|
66
62
|
|
|
67
63
|
dfT = list_tables(dataset, workspace)
|
|
68
64
|
dfC = fabric.list_columns(dataset=dataset, workspace=workspace)
|
|
@@ -92,19 +88,19 @@ def direct_lake_schema_compare(
|
|
|
92
88
|
|
|
93
89
|
if len(missingtbls) == 0:
|
|
94
90
|
print(
|
|
95
|
-
f"All tables exist in the '{lakehouse}' lakehouse within the '{lakehouse_workspace}' workspace."
|
|
91
|
+
f"{icons.green_dot} All tables exist in the '{lakehouse}' lakehouse within the '{lakehouse_workspace}' workspace."
|
|
96
92
|
)
|
|
97
93
|
else:
|
|
98
94
|
print(
|
|
99
|
-
f"The following tables exist in the '{dataset}' semantic model within the '{workspace}' workspace but do not exist in the '{lakehouse}' lakehouse within the '{lakehouse_workspace}' workspace."
|
|
95
|
+
f"{icons.yellow_dot} The following tables exist in the '{dataset}' semantic model within the '{workspace}' workspace but do not exist in the '{lakehouse}' lakehouse within the '{lakehouse_workspace}' workspace."
|
|
100
96
|
)
|
|
101
97
|
display(missingtbls)
|
|
102
98
|
if len(missingcols) == 0:
|
|
103
99
|
print(
|
|
104
|
-
f"All columns exist in the '{lakehouse}' lakehouse within the '{lakehouse_workspace}' workspace."
|
|
100
|
+
f"{icons.green_dot} All columns exist in the '{lakehouse}' lakehouse within the '{lakehouse_workspace}' workspace."
|
|
105
101
|
)
|
|
106
102
|
else:
|
|
107
103
|
print(
|
|
108
|
-
f"The following columns exist in the '{dataset}' semantic model within the '{workspace}' workspace but do not exist in the '{lakehouse}' lakehouse within the '{lakehouse_workspace}' workspace."
|
|
104
|
+
f"{icons.yellow_dot} The following columns exist in the '{dataset}' semantic model within the '{workspace}' workspace but do not exist in the '{lakehouse}' lakehouse within the '{lakehouse_workspace}' workspace."
|
|
109
105
|
)
|
|
110
106
|
display(missingcols)
|
|
@@ -2,6 +2,7 @@ import sempy
|
|
|
2
2
|
import sempy.fabric as fabric
|
|
3
3
|
import pandas as pd
|
|
4
4
|
from sempy_labs.lakehouse._get_lakehouse_columns import get_lakehouse_columns
|
|
5
|
+
from sempy_labs.tom import connect_semantic_model
|
|
5
6
|
from sempy_labs._helper_functions import (
|
|
6
7
|
format_dax_object_name,
|
|
7
8
|
resolve_lakehouse_name,
|
|
@@ -9,7 +10,7 @@ from sempy_labs._helper_functions import (
|
|
|
9
10
|
)
|
|
10
11
|
from typing import Optional
|
|
11
12
|
from sempy._utils._log import log
|
|
12
|
-
|
|
13
|
+
import sempy_labs._icons as icons
|
|
13
14
|
|
|
14
15
|
@log
|
|
15
16
|
def direct_lake_schema_sync(
|
|
@@ -45,14 +46,12 @@ def direct_lake_schema_sync(
|
|
|
45
46
|
import Microsoft.AnalysisServices.Tabular as TOM
|
|
46
47
|
import System
|
|
47
48
|
|
|
48
|
-
|
|
49
|
-
workspace_id = fabric.get_workspace_id()
|
|
50
|
-
workspace = fabric.resolve_workspace_name(workspace_id)
|
|
49
|
+
workspace = fabric.resolve_workspace_name(workspace)
|
|
51
50
|
|
|
52
51
|
if lakehouse_workspace is None:
|
|
53
52
|
lakehouse_workspace = workspace
|
|
54
53
|
|
|
55
|
-
if lakehouse
|
|
54
|
+
if lakehouse is None:
|
|
56
55
|
lakehouse_id = fabric.get_lakehouse_id()
|
|
57
56
|
lakehouse = resolve_lakehouse_name(lakehouse_id, lakehouse_workspace)
|
|
58
57
|
|
|
@@ -62,10 +61,7 @@ def direct_lake_schema_sync(
|
|
|
62
61
|
dfI_filt = dfI[(dfI["Id"] == sqlEndpointId)]
|
|
63
62
|
|
|
64
63
|
if len(dfI_filt) == 0:
|
|
65
|
-
|
|
66
|
-
f"The SQL Endpoint in the '{dataset}' semantic model in the '{workspace} workspace does not point to the '{lakehouse}' lakehouse in the '{lakehouse_workspace}' workspace as specified."
|
|
67
|
-
)
|
|
68
|
-
return
|
|
64
|
+
raise ValueError(f"{icons.red_dot} The SQL Endpoint in the '{dataset}' semantic model in the '{workspace} workspace does not point to the '{lakehouse}' lakehouse in the '{lakehouse_workspace}' workspace as specified.")
|
|
69
65
|
|
|
70
66
|
dfP = fabric.list_partitions(dataset=dataset, workspace=workspace)
|
|
71
67
|
dfP_filt = dfP[dfP["Source Type"] == "Entity"]
|
|
@@ -93,36 +89,35 @@ def direct_lake_schema_sync(
|
|
|
93
89
|
"double": "Double",
|
|
94
90
|
}
|
|
95
91
|
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
92
|
+
with connect_semantic_model(
|
|
93
|
+
dataset=dataset, readonly=False, workspace=workspace
|
|
94
|
+
) as tom:
|
|
95
|
+
|
|
96
|
+
for i, r in lc_filt.iterrows():
|
|
97
|
+
lakeTName = r["Table Name"]
|
|
98
|
+
lakeCName = r["Column Name"]
|
|
99
|
+
fullColName = r["Full Column Name"]
|
|
100
|
+
dType = r["Data Type"]
|
|
101
|
+
|
|
102
|
+
if fullColName not in dfC_filt["Column Object"].values:
|
|
103
|
+
dfL = dfP_filt[dfP_filt["Query"] == lakeTName]
|
|
104
|
+
tName = dfL["Table Name"].iloc[0]
|
|
105
|
+
if add_to_model:
|
|
106
|
+
col = TOM.DataColumn()
|
|
107
|
+
col.Name = lakeCName
|
|
108
|
+
col.SourceColumn = lakeCName
|
|
109
|
+
dt = mapping.get(dType)
|
|
110
|
+
try:
|
|
111
|
+
col.DataType = System.Enum.Parse(TOM.DataType, dt)
|
|
112
|
+
except Exception as e:
|
|
113
|
+
raise ValueError(f"{icons.red_dot} Failed to map '{dType}' data type to the semantic model data types.") from e
|
|
114
|
+
|
|
115
|
+
tom.model.Tables[tName].Columns.Add(col)
|
|
116
|
+
print(
|
|
117
|
+
f"{icons.green_dot} The '{lakeCName}' column has been added to the '{tName}' table as a '{dt}' data type within the '{dataset}' semantic model within the '{workspace}' workspace."
|
|
118
|
+
)
|
|
119
|
+
else:
|
|
115
120
|
print(
|
|
116
|
-
f"
|
|
121
|
+
f"{icons.yellow_dot} The {fullColName} column exists in the lakehouse but not in the '{tName}' table in the '{dataset}' semantic model within the '{workspace}' workspace."
|
|
117
122
|
)
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
m.Tables[tName].Columns.Add(col)
|
|
121
|
-
print(
|
|
122
|
-
f"The '{lakeCName}' column has been added to the '{tName}' table as a '{dt}' data type within the '{dataset}' semantic model within the '{workspace}' workspace."
|
|
123
|
-
)
|
|
124
|
-
else:
|
|
125
|
-
print(
|
|
126
|
-
f"The {fullColName} column exists in the lakehouse but not in the '{tName}' table in the '{dataset}' semantic model within the '{workspace}' workspace."
|
|
127
|
-
)
|
|
128
|
-
m.SaveChanges()
|
|
123
|
+
|
|
@@ -2,7 +2,7 @@ import sempy
|
|
|
2
2
|
import sempy.fabric as fabric
|
|
3
3
|
import numpy as np
|
|
4
4
|
from typing import List, Optional, Union
|
|
5
|
-
|
|
5
|
+
import sempy_labs._icons as icons
|
|
6
6
|
|
|
7
7
|
def check_fallback_reason(dataset: str, workspace: Optional[str] = None):
|
|
8
8
|
"""
|
|
@@ -23,17 +23,13 @@ def check_fallback_reason(dataset: str, workspace: Optional[str] = None):
|
|
|
23
23
|
The tables in the semantic model and their fallback reason.
|
|
24
24
|
"""
|
|
25
25
|
|
|
26
|
-
|
|
27
|
-
workspace_id = fabric.get_workspace_id()
|
|
28
|
-
workspace = fabric.resolve_workspace_name(workspace_id)
|
|
26
|
+
workspace = fabric.resolve_workspace_name(workspace)
|
|
29
27
|
|
|
30
28
|
dfP = fabric.list_partitions(dataset=dataset, workspace=workspace)
|
|
31
29
|
dfP_filt = dfP[dfP["Mode"] == "DirectLake"]
|
|
32
30
|
|
|
33
31
|
if len(dfP_filt) == 0:
|
|
34
|
-
|
|
35
|
-
f"The '{dataset}' semantic model is not in Direct Lake. This function is only applicable to Direct Lake semantic models."
|
|
36
|
-
)
|
|
32
|
+
raise ValueError(f"{icons.red_dot} The '{dataset}' semantic model is not in Direct Lake. This function is only applicable to Direct Lake semantic models.")
|
|
37
33
|
else:
|
|
38
34
|
df = fabric.evaluate_dax(
|
|
39
35
|
dataset=dataset,
|
|
@@ -7,8 +7,7 @@ from sempy_labs._helper_functions import (
|
|
|
7
7
|
)
|
|
8
8
|
from typing import Optional, Tuple
|
|
9
9
|
from uuid import UUID
|
|
10
|
-
|
|
11
|
-
|
|
10
|
+
import sempy_labs._icons as icons
|
|
12
11
|
|
|
13
12
|
def get_direct_lake_lakehouse(
|
|
14
13
|
dataset: str,
|
|
@@ -46,7 +45,7 @@ def get_direct_lake_lakehouse(
|
|
|
46
45
|
if lakehouse_workspace is None:
|
|
47
46
|
lakehouse_workspace = workspace
|
|
48
47
|
|
|
49
|
-
if lakehouse
|
|
48
|
+
if lakehouse is None:
|
|
50
49
|
lakehouse_id = fabric.get_lakehouse_id()
|
|
51
50
|
lakehouse = resolve_lakehouse_name(lakehouse_id, lakehouse_workspace)
|
|
52
51
|
|
|
@@ -55,7 +54,7 @@ def get_direct_lake_lakehouse(
|
|
|
55
54
|
|
|
56
55
|
if len(dfP_filt) == 0:
|
|
57
56
|
raise ValueError(
|
|
58
|
-
f"
|
|
57
|
+
f"{icons.red_dot} The '{dataset}' semantic model within the '{workspace}' workspace is not in Direct Lake mode."
|
|
59
58
|
)
|
|
60
59
|
|
|
61
60
|
sqlEndpointId = get_direct_lake_sql_endpoint(dataset, workspace)
|
|
@@ -1,12 +1,9 @@
|
|
|
1
1
|
import sempy
|
|
2
2
|
import sempy.fabric as fabric
|
|
3
|
-
from sempy_labs._helper_functions import
|
|
4
|
-
resolve_lakehouse_name,
|
|
5
|
-
resolve_workspace_name_and_id,
|
|
6
|
-
)
|
|
3
|
+
from sempy_labs._helper_functions import resolve_lakehouse_name
|
|
7
4
|
from sempy_labs._list_functions import list_lakehouses
|
|
8
5
|
from typing import Optional
|
|
9
|
-
|
|
6
|
+
import sempy_labs._icons as icons
|
|
10
7
|
|
|
11
8
|
def get_shared_expression(
|
|
12
9
|
lakehouse: Optional[str] = None, workspace: Optional[str] = None
|
|
@@ -30,8 +27,8 @@ def get_shared_expression(
|
|
|
30
27
|
Shows the expression which can be used to connect a Direct Lake semantic model to its SQL Endpoint.
|
|
31
28
|
"""
|
|
32
29
|
|
|
33
|
-
|
|
34
|
-
if lakehouse
|
|
30
|
+
workspace = fabric.resolve_workspace_name(workspace)
|
|
31
|
+
if lakehouse is None:
|
|
35
32
|
lakehouse_id = fabric.get_lakehouse_id()
|
|
36
33
|
lakehouse = resolve_lakehouse_name(lakehouse_id)
|
|
37
34
|
|
|
@@ -43,10 +40,7 @@ def get_shared_expression(
|
|
|
43
40
|
provStatus = lakeDetail["SQL Endpoint Provisioning Status"].iloc[0]
|
|
44
41
|
|
|
45
42
|
if provStatus == "InProgress":
|
|
46
|
-
|
|
47
|
-
f"The SQL Endpoint for the '{lakehouse}' lakehouse within the '{workspace}' workspace has not yet been provisioned. Please wait until it has been provisioned."
|
|
48
|
-
)
|
|
49
|
-
return
|
|
43
|
+
raise ValueError(f"{icons.red_dot} The SQL Endpoint for the '{lakehouse}' lakehouse within the '{workspace}' workspace has not yet been provisioned. Please wait until it has been provisioned.")
|
|
50
44
|
|
|
51
45
|
sh = (
|
|
52
46
|
'let\n\tdatabase = Sql.Database("'
|
|
@@ -4,9 +4,9 @@ import pandas as pd
|
|
|
4
4
|
from typing import List, Optional, Union
|
|
5
5
|
|
|
6
6
|
|
|
7
|
-
def get_direct_lake_guardrails():
|
|
7
|
+
def get_direct_lake_guardrails() -> pd.DataFrame:
|
|
8
8
|
"""
|
|
9
|
-
Shows the guardrails for when Direct Lake semantic models will fallback to Direct Query based on Microsoft's online documentation.
|
|
9
|
+
Shows the guardrails for when Direct Lake semantic models will fallback to Direct Query based on Microsoft's `online documentation <https://learn.microsoft.com/power-bi/enterprise/directlake-overview>`_.
|
|
10
10
|
|
|
11
11
|
Parameters
|
|
12
12
|
----------
|
|
@@ -44,9 +44,7 @@ def get_sku_size(workspace: Optional[str] = None):
|
|
|
44
44
|
The SKU size for a workspace.
|
|
45
45
|
"""
|
|
46
46
|
|
|
47
|
-
|
|
48
|
-
workspace_id = fabric.get_workspace_id()
|
|
49
|
-
workspace = fabric.resolve_workspace_name(workspace_id)
|
|
47
|
+
workspace = fabric.resolve_workspace_name(workspace)
|
|
50
48
|
|
|
51
49
|
dfC = fabric.list_capacities()
|
|
52
50
|
dfW = fabric.list_workspaces().sort_values(by="Name", ascending=True)
|
|
@@ -62,10 +60,10 @@ def get_sku_size(workspace: Optional[str] = None):
|
|
|
62
60
|
return sku_value
|
|
63
61
|
|
|
64
62
|
|
|
65
|
-
def get_directlake_guardrails_for_sku(sku_size: str):
|
|
63
|
+
def get_directlake_guardrails_for_sku(sku_size: str) -> pd.DataFrame:
|
|
66
64
|
"""
|
|
67
65
|
Shows the guardrails for Direct Lake based on the SKU used by your workspace's capacity.
|
|
68
|
-
*Use the result of the 'get_sku_size' function as an input for this function's
|
|
66
|
+
* Use the result of the 'get_sku_size' function as an input for this function's sku_size parameter.*
|
|
69
67
|
|
|
70
68
|
Parameters
|
|
71
69
|
----------
|
|
@@ -2,14 +2,15 @@ import sempy
|
|
|
2
2
|
import sempy.fabric as fabric
|
|
3
3
|
import pandas as pd
|
|
4
4
|
from sempy_labs._list_functions import list_tables, list_annotations
|
|
5
|
+
from sempy_labs.tom import connect_semantic_model
|
|
5
6
|
from typing import Optional
|
|
6
7
|
from sempy._utils._log import log
|
|
7
|
-
|
|
8
|
+
import sempy_labs._icons as icons
|
|
8
9
|
|
|
9
10
|
@log
|
|
10
|
-
def list_direct_lake_model_calc_tables(dataset: str, workspace: Optional[str] = None):
|
|
11
|
+
def list_direct_lake_model_calc_tables(dataset: str, workspace: Optional[str] = None) -> pd.DataFrame:
|
|
11
12
|
"""
|
|
12
|
-
Shows the calculated tables and their respective DAX expression for a Direct Lake model (which has been migrated from import/DirectQuery.
|
|
13
|
+
Shows the calculated tables and their respective DAX expression for a Direct Lake model (which has been migrated from import/DirectQuery).
|
|
13
14
|
|
|
14
15
|
Parameters
|
|
15
16
|
----------
|
|
@@ -26,29 +27,30 @@ def list_direct_lake_model_calc_tables(dataset: str, workspace: Optional[str] =
|
|
|
26
27
|
A pandas dataframe showing the calculated tables which were migrated to Direct Lake and whose DAX expressions are stored as model annotations.
|
|
27
28
|
"""
|
|
28
29
|
|
|
29
|
-
|
|
30
|
-
workspace_id = fabric.get_workspace_id()
|
|
31
|
-
workspace = fabric.resolve_workspace_name(workspace_id)
|
|
30
|
+
workspace = fabric.resolve_workspace_name(workspace)
|
|
32
31
|
|
|
33
32
|
df = pd.DataFrame(columns=["Table Name", "Source Expression"])
|
|
34
33
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
34
|
+
with connect_semantic_model(
|
|
35
|
+
dataset=dataset, readonly=True, workspace=workspace
|
|
36
|
+
) as tom:
|
|
37
|
+
|
|
38
|
+
is_direct_lake = tom.is_direct_lake()
|
|
39
|
+
|
|
40
|
+
if not is_direct_lake:
|
|
41
|
+
raise ValueError(f"{icons.red_dot} The '{dataset}' semantic model is not in Direct Lake mode.")
|
|
42
|
+
else:
|
|
43
|
+
dfA = list_annotations(dataset, workspace)
|
|
44
|
+
dfT = list_tables(dataset, workspace)
|
|
45
|
+
dfA_filt = dfA[
|
|
46
|
+
(dfA["Object Type"] == "Model") & (dfA["Annotation Name"].isin(dfT["Name"]))
|
|
47
|
+
]
|
|
48
|
+
|
|
49
|
+
for i, r in dfA_filt.iterrows():
|
|
50
|
+
tName = r["Annotation Name"]
|
|
51
|
+
se = r["Annotation Value"]
|
|
52
|
+
|
|
53
|
+
new_data = {"Table Name": tName, "Source Expression": se}
|
|
54
|
+
df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
|
|
55
|
+
|
|
56
|
+
return df
|
|
@@ -4,8 +4,9 @@ import pandas as pd
|
|
|
4
4
|
from sempy_labs._list_functions import list_tables
|
|
5
5
|
from sempy_labs._helper_functions import format_dax_object_name
|
|
6
6
|
from typing import Optional, Tuple
|
|
7
|
+
from sempy._utils._log import log
|
|
7
8
|
|
|
8
|
-
|
|
9
|
+
@log
|
|
9
10
|
def show_unsupported_direct_lake_objects(
|
|
10
11
|
dataset: str, workspace: Optional[str] = None
|
|
11
12
|
) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
|
|
@@ -29,9 +30,7 @@ def show_unsupported_direct_lake_objects(
|
|
|
29
30
|
|
|
30
31
|
pd.options.mode.chained_assignment = None
|
|
31
32
|
|
|
32
|
-
|
|
33
|
-
workspace_id = fabric.get_workspace_id()
|
|
34
|
-
workspace = fabric.resolve_workspace_name(workspace_id)
|
|
33
|
+
workspace = fabric.resolve_workspace_name(workspace)
|
|
35
34
|
|
|
36
35
|
dfT = list_tables(dataset, workspace)
|
|
37
36
|
dfC = fabric.list_columns(dataset=dataset, workspace=workspace)
|
|
@@ -5,8 +5,9 @@ from sempy_labs._helper_functions import (
|
|
|
5
5
|
resolve_lakehouse_name,
|
|
6
6
|
resolve_workspace_name_and_id,
|
|
7
7
|
)
|
|
8
|
-
from sempy_labs.
|
|
9
|
-
from typing import
|
|
8
|
+
from sempy_labs.tom import connect_semantic_model
|
|
9
|
+
from typing import Optional
|
|
10
|
+
import sempy_labs._icons as icons
|
|
10
11
|
|
|
11
12
|
|
|
12
13
|
def update_direct_lake_model_lakehouse_connection(
|
|
@@ -39,12 +40,12 @@ def update_direct_lake_model_lakehouse_connection(
|
|
|
39
40
|
|
|
40
41
|
"""
|
|
41
42
|
|
|
42
|
-
|
|
43
|
+
workspace = fabric.resolve_workspace_name(workspace)
|
|
43
44
|
|
|
44
|
-
if lakehouse_workspace
|
|
45
|
+
if lakehouse_workspace is None:
|
|
45
46
|
lakehouse_workspace = workspace
|
|
46
47
|
|
|
47
|
-
if lakehouse
|
|
48
|
+
if lakehouse is None:
|
|
48
49
|
lakehouse_id = fabric.get_lakehouse_id()
|
|
49
50
|
lakehouse = resolve_lakehouse_name(lakehouse_id, lakehouse_workspace)
|
|
50
51
|
|
|
@@ -53,17 +54,13 @@ def update_direct_lake_model_lakehouse_connection(
|
|
|
53
54
|
dfI_filt = dfI[(dfI["Display Name"] == lakehouse)]
|
|
54
55
|
|
|
55
56
|
if len(dfI_filt) == 0:
|
|
56
|
-
|
|
57
|
-
f"The '{lakehouse}' lakehouse does not exist within the '{lakehouse_workspace}' workspace. Therefore it cannot be used to support the '{dataset}' semantic model within the '{workspace}' workspace."
|
|
58
|
-
)
|
|
57
|
+
raise ValueError(f"{icons.red_dot} The '{lakehouse}' lakehouse does not exist within the '{lakehouse_workspace}' workspace. Therefore it cannot be used to support the '{dataset}' semantic model within the '{workspace}' workspace.")
|
|
59
58
|
|
|
60
59
|
dfP = fabric.list_partitions(dataset=dataset, workspace=workspace)
|
|
61
60
|
dfP_filt = dfP[dfP["Mode"] == "DirectLake"]
|
|
62
61
|
|
|
63
62
|
if len(dfP_filt) == 0:
|
|
64
|
-
|
|
65
|
-
f"The '{dataset}' semantic model is not in Direct Lake. This function is only applicable to Direct Lake semantic models."
|
|
66
|
-
)
|
|
63
|
+
raise ValueError(f"{icons.red_dot} The '{dataset}' semantic model is not in Direct Lake. This function is only applicable to Direct Lake semantic models.")
|
|
67
64
|
else:
|
|
68
65
|
with connect_semantic_model(
|
|
69
66
|
dataset=dataset, readonly=False, workspace=workspace
|
|
@@ -73,9 +70,7 @@ def update_direct_lake_model_lakehouse_connection(
|
|
|
73
70
|
try:
|
|
74
71
|
tom.model.Expressions["DatabaseQuery"].Expression = shEx
|
|
75
72
|
print(
|
|
76
|
-
f"The expression in the '{dataset}' semantic model has been updated to point to the '{lakehouse}' lakehouse in the '{lakehouse_workspace}' workspace."
|
|
77
|
-
)
|
|
78
|
-
except:
|
|
79
|
-
print(
|
|
80
|
-
f"ERROR: The expression in the '{dataset}' semantic model was not updated."
|
|
73
|
+
f"{icons.green_dot} The expression in the '{dataset}' semantic model has been updated to point to the '{lakehouse}' lakehouse in the '{lakehouse_workspace}' workspace."
|
|
81
74
|
)
|
|
75
|
+
except Exception as e:
|
|
76
|
+
raise ValueError(f"{icons.red_dot} The expression in the '{dataset}' semantic model was not updated.") from e
|
|
@@ -1,13 +1,17 @@
|
|
|
1
|
+
import sempy
|
|
1
2
|
import sempy.fabric as fabric
|
|
2
|
-
from sempy_labs.
|
|
3
|
+
from sempy_labs.tom import connect_semantic_model
|
|
4
|
+
from sempy_labs._helper_functions import resolve_lakehouse_name
|
|
3
5
|
from typing import List, Optional, Union
|
|
4
|
-
|
|
6
|
+
import sempy_labs._icons as icons
|
|
5
7
|
|
|
6
8
|
def update_direct_lake_partition_entity(
|
|
7
9
|
dataset: str,
|
|
8
10
|
table_name: Union[str, List[str]],
|
|
9
11
|
entity_name: Union[str, List[str]],
|
|
10
12
|
workspace: Optional[str] = None,
|
|
13
|
+
lakehouse: Optional[str] = None,
|
|
14
|
+
lakehouse_workspace: Optional[str] = None
|
|
11
15
|
):
|
|
12
16
|
"""
|
|
13
17
|
Remaps a table (or tables) in a Direct Lake semantic model to a table in a lakehouse.
|
|
@@ -24,10 +28,24 @@ def update_direct_lake_partition_entity(
|
|
|
24
28
|
The Fabric workspace name in which the semantic model exists.
|
|
25
29
|
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
26
30
|
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
31
|
+
lakehouse : str, default=None
|
|
32
|
+
The Fabric lakehouse used by the Direct Lake semantic model.
|
|
33
|
+
Defaults to None which resolves to the lakehouse attached to the notebook.
|
|
34
|
+
lakehouse_workspace : str, default=None
|
|
35
|
+
The Fabric workspace used by the lakehouse.
|
|
36
|
+
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
37
|
+
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
27
38
|
"""
|
|
28
39
|
|
|
29
40
|
workspace = fabric.resolve_workspace_name(workspace)
|
|
30
41
|
|
|
42
|
+
if lakehouse_workspace is None:
|
|
43
|
+
lakehouse_workspace = workspace
|
|
44
|
+
|
|
45
|
+
if lakehouse is None:
|
|
46
|
+
lakehouse_id = fabric.get_lakehouse_id()
|
|
47
|
+
lakehouse = resolve_lakehouse_name(lakehouse_id, lakehouse_workspace)
|
|
48
|
+
|
|
31
49
|
# Support both str & list types
|
|
32
50
|
if isinstance(table_name, str):
|
|
33
51
|
table_name = [table_name]
|
|
@@ -35,20 +53,14 @@ def update_direct_lake_partition_entity(
|
|
|
35
53
|
entity_name = [entity_name]
|
|
36
54
|
|
|
37
55
|
if len(table_name) != len(entity_name):
|
|
38
|
-
|
|
39
|
-
f"ERROR: The 'table_name' and 'entity_name' arrays must be of equal length."
|
|
40
|
-
)
|
|
41
|
-
return
|
|
56
|
+
raise ValueError(f"{icons.red_dot} The 'table_name' and 'entity_name' arrays must be of equal length.")
|
|
42
57
|
|
|
43
58
|
with connect_semantic_model(
|
|
44
59
|
dataset=dataset, readonly=False, workspace=workspace
|
|
45
60
|
) as tom:
|
|
46
61
|
|
|
47
62
|
if not tom.is_direct_lake():
|
|
48
|
-
|
|
49
|
-
f"The '{dataset}' semantic model within the '{workspace}' workspace is not in Direct Lake mode."
|
|
50
|
-
)
|
|
51
|
-
return
|
|
63
|
+
raise ValueError(f"{icons.red_dot} The '{dataset}' semantic model within the '{workspace}' workspace is not in Direct Lake mode.")
|
|
52
64
|
|
|
53
65
|
for tName in table_name:
|
|
54
66
|
i = table_name.index(tName)
|
|
@@ -56,9 +68,7 @@ def update_direct_lake_partition_entity(
|
|
|
56
68
|
try:
|
|
57
69
|
tom.model.Tables[tName].Partitions[0].EntityName = eName
|
|
58
70
|
print(
|
|
59
|
-
f"The '{tName}' table in the '{dataset}' semantic model has been updated to point to the '{eName}' table in the '{lakehouse}' lakehouse within the '{lakehouse_workspace}' workspace."
|
|
60
|
-
)
|
|
61
|
-
except:
|
|
62
|
-
print(
|
|
63
|
-
f"ERROR: The '{tName}' table in the '{dataset}' semantic model has not been updated."
|
|
71
|
+
f"{icons.green_dot} The '{tName}' table in the '{dataset}' semantic model has been updated to point to the '{eName}' table in the '{lakehouse}' lakehouse within the '{lakehouse_workspace}' workspace."
|
|
64
72
|
)
|
|
73
|
+
except Exception as e:
|
|
74
|
+
raise ValueError(f"{icons.red_dot} The '{tName}' table in the '{dataset}' semantic model has not been updated.") from e
|
|
@@ -45,10 +45,7 @@ def warm_direct_lake_cache_perspective(
|
|
|
45
45
|
|
|
46
46
|
dfP = fabric.list_partitions(dataset=dataset, workspace=workspace)
|
|
47
47
|
if not any(r["Mode"] == "DirectLake" for i, r in dfP.iterrows()):
|
|
48
|
-
|
|
49
|
-
f"{icons.red_dot} The '{dataset}' semantic model in the '{workspace}' workspace is not in Direct Lake mode. This function is specifically for semantic models in Direct Lake mode."
|
|
50
|
-
)
|
|
51
|
-
return
|
|
48
|
+
raise ValueError(f"{icons.red_dot} The '{dataset}' semantic model in the '{workspace}' workspace is not in Direct Lake mode. This function is specifically for semantic models in Direct Lake mode.")
|
|
52
49
|
|
|
53
50
|
dfPersp = fabric.list_perspectives(dataset=dataset, workspace=workspace)
|
|
54
51
|
dfPersp["DAX Object Name"] = format_dax_object_name(
|
|
@@ -57,10 +54,8 @@ def warm_direct_lake_cache_perspective(
|
|
|
57
54
|
dfPersp_filt = dfPersp[dfPersp["Perspective Name"] == perspective]
|
|
58
55
|
|
|
59
56
|
if len(dfPersp_filt) == 0:
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
)
|
|
63
|
-
return
|
|
57
|
+
raise ValueError(f"{icons.red_dot} The '{perspective} perspective does not exist or contains no objects within the '{dataset}' semantic model in the '{workspace}' workspace.")
|
|
58
|
+
|
|
64
59
|
dfPersp_c = dfPersp_filt[dfPersp_filt["Object Type"] == "Column"]
|
|
65
60
|
|
|
66
61
|
column_values = dfPersp_c["DAX Object Name"].tolist()
|
|
@@ -131,7 +126,7 @@ def warm_direct_lake_cache_perspective(
|
|
|
131
126
|
bar.set_description(f"Warming the '{tableName}' table...")
|
|
132
127
|
css = ",".join(map(str, filtered_list))
|
|
133
128
|
dax = """EVALUATE TOPN(1,SUMMARIZECOLUMNS(""" + css + "))" ""
|
|
134
|
-
|
|
129
|
+
fabric.evaluate_dax(dataset=dataset, dax_string=dax, workspace=workspace)
|
|
135
130
|
|
|
136
131
|
print(f"{icons.green_dot} The following columns have been put into memory:")
|
|
137
132
|
|
|
@@ -166,12 +161,13 @@ def warm_direct_lake_cache_isresident(
|
|
|
166
161
|
Returns a pandas dataframe showing the columns that have been put into memory.
|
|
167
162
|
"""
|
|
168
163
|
|
|
164
|
+
workspace = fabric.resolve_workspace_name(workspace)
|
|
165
|
+
|
|
169
166
|
dfP = fabric.list_partitions(dataset=dataset, workspace=workspace)
|
|
170
167
|
if not any(r["Mode"] == "DirectLake" for i, r in dfP.iterrows()):
|
|
171
|
-
|
|
172
|
-
f"The '{dataset}' semantic model in the '{workspace}' workspace is not in Direct Lake mode. This function is specifically for semantic models in Direct Lake mode."
|
|
168
|
+
raise ValueError(
|
|
169
|
+
f"{icons.red_dot} The '{dataset}' semantic model in the '{workspace}' workspace is not in Direct Lake mode. This function is specifically for semantic models in Direct Lake mode."
|
|
173
170
|
)
|
|
174
|
-
return
|
|
175
171
|
|
|
176
172
|
# Identify columns which are currently in memory (Is Resident = True)
|
|
177
173
|
dfC = fabric.list_columns(dataset=dataset, workspace=workspace, extended=True)
|
|
@@ -181,10 +177,9 @@ def warm_direct_lake_cache_isresident(
|
|
|
181
177
|
dfC_filtered = dfC[dfC["Is Resident"]]
|
|
182
178
|
|
|
183
179
|
if len(dfC_filtered) == 0:
|
|
184
|
-
|
|
180
|
+
raise ValueError(
|
|
185
181
|
f"{icons.yellow_dot} At present, no columns are in memory in the '{dataset}' semantic model in the '{workspace}' workspace."
|
|
186
182
|
)
|
|
187
|
-
return
|
|
188
183
|
|
|
189
184
|
# Refresh/frame dataset
|
|
190
185
|
refresh_semantic_model(dataset=dataset, refresh_type="full", workspace=workspace)
|
|
@@ -199,7 +194,7 @@ def warm_direct_lake_cache_isresident(
|
|
|
199
194
|
bar.set_description(f"Warming the '{tableName}' table...")
|
|
200
195
|
css = ",".join(map(str, column_values))
|
|
201
196
|
dax = """EVALUATE TOPN(1,SUMMARIZECOLUMNS(""" + css + "))" ""
|
|
202
|
-
|
|
197
|
+
fabric.evaluate_dax(dataset=dataset, dax_string=dax, workspace=workspace)
|
|
203
198
|
|
|
204
199
|
print(
|
|
205
200
|
f"{icons.green_dot} The following columns have been put into memory. Temperature indicates the column temperature prior to the semantic model refresh."
|
sempy_labs/lakehouse/__init__.py
CHANGED
|
@@ -6,7 +6,6 @@ from sempy_labs.lakehouse._lakehouse import (
|
|
|
6
6
|
)
|
|
7
7
|
|
|
8
8
|
from sempy_labs.lakehouse._shortcuts import (
|
|
9
|
-
list_shortcuts,
|
|
10
9
|
# create_shortcut,
|
|
11
10
|
create_shortcut_onelake,
|
|
12
11
|
delete_shortcut,
|
|
@@ -17,7 +16,6 @@ __all__ = [
|
|
|
17
16
|
"get_lakehouse_tables",
|
|
18
17
|
"lakehouse_attached",
|
|
19
18
|
"optimize_lakehouse_tables",
|
|
20
|
-
"list_shortcuts",
|
|
21
19
|
# create_shortcut,
|
|
22
20
|
"create_shortcut_onelake",
|
|
23
21
|
"delete_shortcut",
|