semantic-link-labs 0.9.5__py3-none-any.whl → 0.9.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of semantic-link-labs might be problematic. Click here for more details.
- {semantic_link_labs-0.9.5.dist-info → semantic_link_labs-0.9.7.dist-info}/METADATA +8 -5
- {semantic_link_labs-0.9.5.dist-info → semantic_link_labs-0.9.7.dist-info}/RECORD +65 -61
- {semantic_link_labs-0.9.5.dist-info → semantic_link_labs-0.9.7.dist-info}/WHEEL +1 -1
- sempy_labs/__init__.py +19 -1
- sempy_labs/_ai.py +3 -1
- sempy_labs/_capacities.py +37 -2
- sempy_labs/_capacity_migration.py +11 -14
- sempy_labs/_connections.py +2 -4
- sempy_labs/_dataflows.py +2 -2
- sempy_labs/_dax_query_view.py +57 -0
- sempy_labs/_delta_analyzer.py +16 -14
- sempy_labs/_delta_analyzer_history.py +298 -0
- sempy_labs/_environments.py +8 -1
- sempy_labs/_eventhouses.py +5 -1
- sempy_labs/_external_data_shares.py +4 -10
- sempy_labs/_generate_semantic_model.py +2 -1
- sempy_labs/_graphQL.py +5 -1
- sempy_labs/_helper_functions.py +440 -63
- sempy_labs/_icons.py +6 -6
- sempy_labs/_kql_databases.py +5 -1
- sempy_labs/_list_functions.py +8 -38
- sempy_labs/_managed_private_endpoints.py +9 -2
- sempy_labs/_mirrored_databases.py +3 -1
- sempy_labs/_ml_experiments.py +1 -1
- sempy_labs/_model_bpa.py +2 -11
- sempy_labs/_model_bpa_bulk.py +33 -38
- sempy_labs/_model_bpa_rules.py +1 -1
- sempy_labs/_one_lake_integration.py +2 -1
- sempy_labs/_semantic_models.py +20 -0
- sempy_labs/_sql.py +6 -2
- sempy_labs/_sqldatabase.py +61 -100
- sempy_labs/_vertipaq.py +8 -11
- sempy_labs/_warehouses.py +14 -3
- sempy_labs/_workspace_identity.py +6 -0
- sempy_labs/_workspaces.py +42 -2
- sempy_labs/admin/_basic_functions.py +29 -2
- sempy_labs/admin/_reports.py +1 -1
- sempy_labs/admin/_scanner.py +2 -4
- sempy_labs/admin/_tenant.py +8 -3
- sempy_labs/directlake/_directlake_schema_compare.py +2 -1
- sempy_labs/directlake/_directlake_schema_sync.py +65 -19
- sempy_labs/directlake/_dl_helper.py +0 -6
- sempy_labs/directlake/_generate_shared_expression.py +19 -12
- sempy_labs/directlake/_guardrails.py +2 -1
- sempy_labs/directlake/_update_directlake_model_lakehouse_connection.py +90 -57
- sempy_labs/directlake/_update_directlake_partition_entity.py +5 -2
- sempy_labs/graph/_groups.py +6 -0
- sempy_labs/graph/_teams.py +2 -0
- sempy_labs/graph/_users.py +4 -0
- sempy_labs/lakehouse/__init__.py +12 -3
- sempy_labs/lakehouse/_blobs.py +231 -0
- sempy_labs/lakehouse/_shortcuts.py +29 -8
- sempy_labs/migration/_direct_lake_to_import.py +47 -10
- sempy_labs/migration/_migration_validation.py +0 -4
- sempy_labs/report/__init__.py +4 -0
- sempy_labs/report/_download_report.py +4 -6
- sempy_labs/report/_generate_report.py +6 -6
- sempy_labs/report/_report_functions.py +5 -4
- sempy_labs/report/_report_helper.py +17 -5
- sempy_labs/report/_report_rebind.py +8 -6
- sempy_labs/report/_reportwrapper.py +17 -8
- sempy_labs/report/_save_report.py +147 -0
- sempy_labs/tom/_model.py +154 -23
- {semantic_link_labs-0.9.5.dist-info → semantic_link_labs-0.9.7.dist-info/licenses}/LICENSE +0 -0
- {semantic_link_labs-0.9.5.dist-info → semantic_link_labs-0.9.7.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
from sempy_labs._helper_functions import (
|
|
2
|
+
resolve_workspace_id,
|
|
3
|
+
resolve_lakehouse_id,
|
|
4
|
+
_xml_to_dict,
|
|
5
|
+
_create_dataframe,
|
|
6
|
+
_update_dataframe_datatypes,
|
|
7
|
+
)
|
|
8
|
+
from sempy._utils._log import log
|
|
9
|
+
from uuid import UUID
|
|
10
|
+
from typing import Optional, List
|
|
11
|
+
import sempy_labs._icons as icons
|
|
12
|
+
import xml.etree.ElementTree as ET
|
|
13
|
+
import pandas as pd
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _request_blob_api(
|
|
17
|
+
request: str,
|
|
18
|
+
method: str = "get",
|
|
19
|
+
payload: Optional[dict] = None,
|
|
20
|
+
status_codes: int | List[int] = 200,
|
|
21
|
+
):
|
|
22
|
+
|
|
23
|
+
import requests
|
|
24
|
+
import notebookutils
|
|
25
|
+
from sempy.fabric.exceptions import FabricHTTPException
|
|
26
|
+
|
|
27
|
+
if isinstance(status_codes, int):
|
|
28
|
+
status_codes = [status_codes]
|
|
29
|
+
|
|
30
|
+
token = notebookutils.credentials.getToken("storage")
|
|
31
|
+
|
|
32
|
+
headers = {
|
|
33
|
+
"Authorization": f"Bearer {token}",
|
|
34
|
+
"Content-Type": "application/json",
|
|
35
|
+
"x-ms-version": "2025-05-05",
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
response = requests.request(
|
|
39
|
+
method.upper(),
|
|
40
|
+
f"https://onelake.blob.fabric.microsoft.com/{request}",
|
|
41
|
+
headers=headers,
|
|
42
|
+
json=payload,
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
if response.status_code not in status_codes:
|
|
46
|
+
raise FabricHTTPException(response)
|
|
47
|
+
|
|
48
|
+
return response
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@log
|
|
52
|
+
def list_blobs(
|
|
53
|
+
lakehouse: Optional[str | UUID] = None,
|
|
54
|
+
workspace: Optional[str | UUID] = None,
|
|
55
|
+
container: Optional[str] = None,
|
|
56
|
+
) -> pd.DataFrame:
|
|
57
|
+
"""
|
|
58
|
+
Returns a list of blobs for a given lakehouse.
|
|
59
|
+
|
|
60
|
+
This function leverages the following API: `List Blobs <https://learn.microsoft.com/rest/api/storageservices/list-blobs?tabs=microsoft-entra-id>`_.
|
|
61
|
+
|
|
62
|
+
Parameters
|
|
63
|
+
----------
|
|
64
|
+
lakehouse : str | uuid.UUID, default=None
|
|
65
|
+
The Fabric lakehouse name or ID.
|
|
66
|
+
Defaults to None which resolves to the lakehouse attached to the notebook.
|
|
67
|
+
workspace : str | uuid.UUID, default=None
|
|
68
|
+
The Fabric workspace name or ID used by the lakehouse.
|
|
69
|
+
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
70
|
+
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
71
|
+
container : str, default=None
|
|
72
|
+
The container name to list blobs from. If None, lists all blobs in the lakehouse.
|
|
73
|
+
Valid values are "Tables" or "Files". If not specified, the function will list all blobs in the lakehouse.
|
|
74
|
+
|
|
75
|
+
Returns
|
|
76
|
+
-------
|
|
77
|
+
pandas.DataFrame
|
|
78
|
+
A pandas dataframe showing a list of blobs in the lakehouse.
|
|
79
|
+
"""
|
|
80
|
+
|
|
81
|
+
workspace_id = resolve_workspace_id(workspace)
|
|
82
|
+
lakehouse_id = resolve_lakehouse_id(lakehouse, workspace_id)
|
|
83
|
+
|
|
84
|
+
if container is None:
|
|
85
|
+
path_prefix = f"{workspace_id}/{lakehouse_id}"
|
|
86
|
+
else:
|
|
87
|
+
if container not in ["Tables", "Files"]:
|
|
88
|
+
raise ValueError(
|
|
89
|
+
f"{icons.red_dot} Invalid container '{container}' within the file_path parameter. Expected 'Tables' or 'Files'."
|
|
90
|
+
)
|
|
91
|
+
path_prefix = f"{workspace_id}/{lakehouse_id}/{container}"
|
|
92
|
+
|
|
93
|
+
response = _request_blob_api(
|
|
94
|
+
request=f"{path_prefix}?restype=container&comp=list&include=deleted"
|
|
95
|
+
)
|
|
96
|
+
root = ET.fromstring(response.content)
|
|
97
|
+
response_json = _xml_to_dict(root)
|
|
98
|
+
|
|
99
|
+
columns = {
|
|
100
|
+
"Blob Name": "str",
|
|
101
|
+
"Is Deleted": "bool",
|
|
102
|
+
"Deletion Id": "str",
|
|
103
|
+
"Creation Time": "datetime",
|
|
104
|
+
"Expiry Time": "datetime",
|
|
105
|
+
"Etag": "str",
|
|
106
|
+
"Resource Type": "str",
|
|
107
|
+
"Content Length": "int",
|
|
108
|
+
"Content Type": "str",
|
|
109
|
+
"Content Encoding": "str",
|
|
110
|
+
"Content Language": "str",
|
|
111
|
+
"Content CRC64": "str",
|
|
112
|
+
"Content MD5": "str",
|
|
113
|
+
"Cache Control": "str",
|
|
114
|
+
"Content Disposition": "str",
|
|
115
|
+
"Blob Type": "str",
|
|
116
|
+
"Access Tier": "str",
|
|
117
|
+
"Access Tier Inferred": "str",
|
|
118
|
+
"Server Encrypted": "bool",
|
|
119
|
+
"Deleted Time": "str",
|
|
120
|
+
"Remaining Retention Days": "str",
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
df = _create_dataframe(columns=columns)
|
|
124
|
+
|
|
125
|
+
for blob in (
|
|
126
|
+
response_json.get("EnumerationResults", {}).get("Blobs", {}).get("Blob", {})
|
|
127
|
+
):
|
|
128
|
+
p = blob.get("Properties", {})
|
|
129
|
+
new_data = {
|
|
130
|
+
"Blob Name": blob.get("Name"),
|
|
131
|
+
"Is Deleted": blob.get("Deleted", False),
|
|
132
|
+
"Deletion Id": blob.get("DeletionId"),
|
|
133
|
+
"Creation Time": p.get("Creation-Time"),
|
|
134
|
+
"Expiry Time": p.get("Expiry-Time"),
|
|
135
|
+
"Etag": p.get("Etag"),
|
|
136
|
+
"Resource Type": p.get("ResourceType"),
|
|
137
|
+
"Content Length": p.get("Content-Length"),
|
|
138
|
+
"Content Type": p.get("Content-Type"),
|
|
139
|
+
"Content Encoding": p.get("Content-Encoding"),
|
|
140
|
+
"Content Language": p.get("Content-Language"),
|
|
141
|
+
"Content CRC64": p.get("Content-CRC64"),
|
|
142
|
+
"Content MD5": p.get("Content-MD5"),
|
|
143
|
+
"Cache Control": p.get("Cache-Control"),
|
|
144
|
+
"Content Disposition": p.get("Content-Disposition"),
|
|
145
|
+
"Blob Type": p.get("BlobType"),
|
|
146
|
+
"Access Tier": p.get("AccessTier"),
|
|
147
|
+
"Access Tier Inferred": p.get("AccessTierInferred"),
|
|
148
|
+
"Server Encrypted": p.get("ServerEncrypted"),
|
|
149
|
+
"Deleted Time": p.get("DeletedTime"),
|
|
150
|
+
"Remaining Retention Days": p.get("RemainingRetentionDays"),
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True)
|
|
154
|
+
|
|
155
|
+
_update_dataframe_datatypes(dataframe=df, column_map=columns)
|
|
156
|
+
|
|
157
|
+
return df
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
@log
|
|
161
|
+
def recover_lakehouse_object(
|
|
162
|
+
file_path: str,
|
|
163
|
+
lakehouse: Optional[str | UUID] = None,
|
|
164
|
+
workspace: Optional[str | UUID] = None,
|
|
165
|
+
):
|
|
166
|
+
"""
|
|
167
|
+
Recovers an object (i.e. table, file, folder) in a lakehouse from a deleted state. Only `soft-deleted objects <https://learn.microsoft.com/fabric/onelake/onelake-disaster-recovery#soft-delete-for-onelake-files>`_ can be recovered (deleted for less than 7 days).
|
|
168
|
+
|
|
169
|
+
Parameters
|
|
170
|
+
----------
|
|
171
|
+
file_path : str
|
|
172
|
+
The file path of the object to restore. For example: "Tables/my_delta_table".
|
|
173
|
+
lakehouse : str | uuid.UUID, default=None
|
|
174
|
+
The Fabric lakehouse name or ID.
|
|
175
|
+
Defaults to None which resolves to the lakehouse attached to the notebook.
|
|
176
|
+
workspace : str | uuid.UUID, default=None
|
|
177
|
+
The Fabric workspace name or ID used by the lakehouse.
|
|
178
|
+
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
179
|
+
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
180
|
+
"""
|
|
181
|
+
|
|
182
|
+
workspace_id = resolve_workspace_id(workspace)
|
|
183
|
+
lakehouse_id = resolve_lakehouse_id(lakehouse, workspace_id)
|
|
184
|
+
|
|
185
|
+
blob_path_prefix = f"{lakehouse_id}/{file_path}"
|
|
186
|
+
|
|
187
|
+
container = file_path.split("/")[0]
|
|
188
|
+
if container not in ["Tables", "Files"]:
|
|
189
|
+
raise ValueError(
|
|
190
|
+
f"{icons.red_dot} Invalid container '{container}' within the file_path parameter. Expected 'Tables' or 'Files'."
|
|
191
|
+
)
|
|
192
|
+
|
|
193
|
+
df = list_blobs(lakehouse=lakehouse, workspace=workspace, container=container)
|
|
194
|
+
|
|
195
|
+
for _, r in df.iterrows():
|
|
196
|
+
blob_name = r.get("Blob Name")
|
|
197
|
+
is_deleted = r.get("Is Deleted")
|
|
198
|
+
if blob_name.startswith(blob_path_prefix) and is_deleted:
|
|
199
|
+
print(f"{icons.in_progress} Restoring the '{blob_name}' blob...")
|
|
200
|
+
_request_blob_api(
|
|
201
|
+
request=f"{workspace_id}/{lakehouse_id}/{file_path}?comp=undelete",
|
|
202
|
+
method="put",
|
|
203
|
+
)
|
|
204
|
+
print(f"{icons.green_dot} The '{blob_name}' blob has been restored.")
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def _get_user_delegation_key():
|
|
208
|
+
|
|
209
|
+
# https://learn.microsoft.com/rest/api/storageservices/get-user-delegation-key
|
|
210
|
+
|
|
211
|
+
from datetime import datetime, timedelta, timezone
|
|
212
|
+
|
|
213
|
+
utc_now = datetime.now(timezone.utc)
|
|
214
|
+
start_time = utc_now + timedelta(minutes=2)
|
|
215
|
+
expiry_time = start_time + timedelta(minutes=45)
|
|
216
|
+
start_str = start_time.strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
217
|
+
expiry_str = expiry_time.strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
218
|
+
|
|
219
|
+
payload = f"""<?xml version="1.0" encoding="utf-8"?>
|
|
220
|
+
<KeyInfo>
|
|
221
|
+
<Start>{start_str}</Start>
|
|
222
|
+
<Expiry>{expiry_str}</Expiry>
|
|
223
|
+
</KeyInfo>"""
|
|
224
|
+
|
|
225
|
+
response = _request_blob_api(
|
|
226
|
+
request="restype=service&comp=userdelegationkey",
|
|
227
|
+
method="post",
|
|
228
|
+
payload=payload,
|
|
229
|
+
)
|
|
230
|
+
|
|
231
|
+
return response.content
|
|
@@ -5,6 +5,7 @@ from sempy_labs._helper_functions import (
|
|
|
5
5
|
resolve_workspace_name_and_id,
|
|
6
6
|
_base_api,
|
|
7
7
|
_create_dataframe,
|
|
8
|
+
resolve_workspace_name,
|
|
8
9
|
)
|
|
9
10
|
from sempy._utils._log import log
|
|
10
11
|
from typing import Optional
|
|
@@ -16,29 +17,33 @@ from sempy.fabric.exceptions import FabricHTTPException
|
|
|
16
17
|
@log
|
|
17
18
|
def create_shortcut_onelake(
|
|
18
19
|
table_name: str,
|
|
19
|
-
source_lakehouse: str,
|
|
20
|
+
source_lakehouse: str | UUID,
|
|
20
21
|
source_workspace: str | UUID,
|
|
21
|
-
destination_lakehouse: str,
|
|
22
|
+
destination_lakehouse: Optional[str | UUID] = None,
|
|
22
23
|
destination_workspace: Optional[str | UUID] = None,
|
|
23
24
|
shortcut_name: Optional[str] = None,
|
|
24
25
|
source_path: str = "Tables",
|
|
25
26
|
destination_path: str = "Tables",
|
|
27
|
+
shortcut_conflict_policy: Optional[str] = None,
|
|
26
28
|
):
|
|
27
29
|
"""
|
|
28
30
|
Creates a `shortcut <https://learn.microsoft.com/fabric/onelake/onelake-shortcuts>`_ to a delta table in OneLake.
|
|
29
31
|
|
|
30
32
|
This is a wrapper function for the following API: `OneLake Shortcuts - Create Shortcut <https://learn.microsoft.com/rest/api/fabric/core/onelake-shortcuts/create-shortcut>`_.
|
|
31
33
|
|
|
34
|
+
Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
|
|
35
|
+
|
|
32
36
|
Parameters
|
|
33
37
|
----------
|
|
34
38
|
table_name : str
|
|
35
39
|
The table name for which a shortcut will be created.
|
|
36
|
-
source_lakehouse : str
|
|
40
|
+
source_lakehouse : str | uuid.UUID
|
|
37
41
|
The Fabric lakehouse in which the table resides.
|
|
38
42
|
source_workspace : str | uuid.UUID
|
|
39
43
|
The name or ID of the Fabric workspace in which the source lakehouse exists.
|
|
40
|
-
destination_lakehouse : str
|
|
44
|
+
destination_lakehouse : str | uuid.UUID, default=None
|
|
41
45
|
The Fabric lakehouse in which the shortcut will be created.
|
|
46
|
+
Defaults to None which resolves to the lakehouse attached to the notebook.
|
|
42
47
|
destination_workspace : str | uuid.UUID, default=None
|
|
43
48
|
The name or ID of the Fabric workspace in which the shortcut will be created.
|
|
44
49
|
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
@@ -49,6 +54,8 @@ def create_shortcut_onelake(
|
|
|
49
54
|
A string representing the full path to the table/file in the source lakehouse, including either "Files" or "Tables". Examples: Tables/FolderName/SubFolderName; Files/FolderName/SubFolderName.
|
|
50
55
|
destination_path: str, default="Tables"
|
|
51
56
|
A string representing the full path where the shortcut is created, including either "Files" or "Tables". Examples: Tables/FolderName/SubFolderName; Files/FolderName/SubFolderName.
|
|
57
|
+
shortcut_conflict_policy : str, default=None
|
|
58
|
+
When provided, it defines the action to take when a shortcut with the same name and path already exists. The default action is 'Abort'. Additional ShortcutConflictPolicy types may be added over time.
|
|
52
59
|
"""
|
|
53
60
|
|
|
54
61
|
if not (source_path.startswith("Files") or source_path.startswith("Tables")):
|
|
@@ -101,7 +108,8 @@ def create_shortcut_onelake(
|
|
|
101
108
|
# Check if the shortcut already exists
|
|
102
109
|
try:
|
|
103
110
|
response = _base_api(
|
|
104
|
-
request=f"/v1/workspaces/{destination_workspace_id}/items/{destination_lakehouse_id}/shortcuts/{destination_path}/{actual_shortcut_name}"
|
|
111
|
+
request=f"/v1/workspaces/{destination_workspace_id}/items/{destination_lakehouse_id}/shortcuts/{destination_path}/{actual_shortcut_name}",
|
|
112
|
+
client="fabric_sp",
|
|
105
113
|
)
|
|
106
114
|
response_json = response.json()
|
|
107
115
|
del response_json["target"]["type"]
|
|
@@ -117,11 +125,21 @@ def create_shortcut_onelake(
|
|
|
117
125
|
except FabricHTTPException:
|
|
118
126
|
pass
|
|
119
127
|
|
|
128
|
+
url = f"/v1/workspaces/{destination_workspace_id}/items/{destination_lakehouse_id}/shortcuts"
|
|
129
|
+
|
|
130
|
+
if shortcut_conflict_policy:
|
|
131
|
+
if shortcut_conflict_policy not in ["Abort", "GenerateUniqueName"]:
|
|
132
|
+
raise ValueError(
|
|
133
|
+
f"{icons.red_dot} The 'shortcut_conflict_policy' parameter must be either 'Abort' or 'GenerateUniqueName'."
|
|
134
|
+
)
|
|
135
|
+
url += f"?shortcutConflictPolicy={shortcut_conflict_policy}"
|
|
136
|
+
|
|
120
137
|
_base_api(
|
|
121
|
-
request=
|
|
138
|
+
request=url,
|
|
122
139
|
payload=payload,
|
|
123
140
|
status_codes=201,
|
|
124
141
|
method="post",
|
|
142
|
+
client="fabric_sp",
|
|
125
143
|
)
|
|
126
144
|
|
|
127
145
|
print(
|
|
@@ -209,6 +227,8 @@ def delete_shortcut(
|
|
|
209
227
|
|
|
210
228
|
This is a wrapper function for the following API: `OneLake Shortcuts - Delete Shortcut <https://learn.microsoft.com/rest/api/fabric/core/onelake-shortcuts/delete-shortcut>`_.
|
|
211
229
|
|
|
230
|
+
Service Principal Authentication is supported (see `here <https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Service%20Principal.ipynb>`_ for examples).
|
|
231
|
+
|
|
212
232
|
Parameters
|
|
213
233
|
----------
|
|
214
234
|
shortcut_name : str
|
|
@@ -232,6 +252,7 @@ def delete_shortcut(
|
|
|
232
252
|
_base_api(
|
|
233
253
|
request=f"/v1/workspaces/{workspace_id}/items/{lakehouse_id}/shortcuts/{shortcut_path}/{shortcut_name}",
|
|
234
254
|
method="delete",
|
|
255
|
+
client="fabric_sp",
|
|
235
256
|
)
|
|
236
257
|
|
|
237
258
|
print(
|
|
@@ -286,7 +307,7 @@ def list_shortcuts(
|
|
|
286
307
|
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
287
308
|
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
288
309
|
path: str, default=None
|
|
289
|
-
The path within lakehouse where to look for shortcuts. If
|
|
310
|
+
The path within lakehouse where to look for shortcuts. If provided, must start with either "Files" or "Tables". Examples: Tables/FolderName/SubFolderName; Files/FolderName/SubFolderName.
|
|
290
311
|
Defaults to None which will retun all shortcuts on the given lakehouse
|
|
291
312
|
|
|
292
313
|
Returns
|
|
@@ -359,7 +380,7 @@ def list_shortcuts(
|
|
|
359
380
|
source_item_id = tgt.get(sources.get(tgt_type), {}).get("itemId")
|
|
360
381
|
bucket = tgt.get(sources.get(tgt_type), {}).get("bucket")
|
|
361
382
|
source_workspace_name = (
|
|
362
|
-
|
|
383
|
+
resolve_workspace_name(workspace_id=source_workspace_id)
|
|
363
384
|
if source_workspace_id is not None
|
|
364
385
|
else None
|
|
365
386
|
)
|
|
@@ -1,11 +1,16 @@
|
|
|
1
1
|
import sempy
|
|
2
2
|
from uuid import UUID
|
|
3
3
|
import sempy_labs._icons as icons
|
|
4
|
+
from typing import Optional
|
|
4
5
|
|
|
5
6
|
|
|
6
|
-
def migrate_direct_lake_to_import(
|
|
7
|
+
def migrate_direct_lake_to_import(
|
|
8
|
+
dataset: str | UUID,
|
|
9
|
+
workspace: Optional[str | UUID] = None,
|
|
10
|
+
mode: str = "import",
|
|
11
|
+
):
|
|
7
12
|
"""
|
|
8
|
-
Migrates a semantic model from Direct Lake mode to import mode. After running this function, you must go to the semantic model settings and update the cloud connection. Not doing so will result in an inablity to refresh/use the semantic model.
|
|
13
|
+
Migrates a semantic model or specific table(s) from a Direct Lake mode to import or DirectQuery mode. After running this function, you must go to the semantic model settings and update the cloud connection. Not doing so will result in an inablity to refresh/use the semantic model.
|
|
9
14
|
|
|
10
15
|
Parameters
|
|
11
16
|
----------
|
|
@@ -15,12 +20,29 @@ def migrate_direct_lake_to_import(dataset: str | UUID, workspace: str | UUID):
|
|
|
15
20
|
The Fabric workspace name or ID.
|
|
16
21
|
Defaults to None which resolves to the workspace of the attached lakehouse
|
|
17
22
|
or if no lakehouse attached, resolves to the workspace of the notebook.
|
|
23
|
+
mode : str, default="import"
|
|
24
|
+
The mode to migrate to. Can be either "import" or "directquery".
|
|
18
25
|
"""
|
|
19
26
|
|
|
20
27
|
sempy.fabric._client._utils._init_analysis_services()
|
|
21
28
|
import Microsoft.AnalysisServices.Tabular as TOM
|
|
22
29
|
from sempy_labs.tom import connect_semantic_model
|
|
23
30
|
|
|
31
|
+
modes = {
|
|
32
|
+
"import": "Import",
|
|
33
|
+
"directquery": "DirectQuery",
|
|
34
|
+
"dq": "DirectQuery",
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
# Resolve mode
|
|
38
|
+
mode = mode.lower()
|
|
39
|
+
actual_mode = modes.get(mode)
|
|
40
|
+
if actual_mode is None:
|
|
41
|
+
raise ValueError(f"Invalid mode '{mode}'. Must be one of {list(modes.keys())}.")
|
|
42
|
+
|
|
43
|
+
# if isinstance(tables, str):
|
|
44
|
+
# tables = [tables]
|
|
45
|
+
|
|
24
46
|
with connect_semantic_model(
|
|
25
47
|
dataset=dataset, workspace=workspace, readonly=False
|
|
26
48
|
) as tom:
|
|
@@ -31,7 +53,14 @@ def migrate_direct_lake_to_import(dataset: str | UUID, workspace: str | UUID):
|
|
|
31
53
|
)
|
|
32
54
|
return
|
|
33
55
|
|
|
34
|
-
|
|
56
|
+
# if tables is None:
|
|
57
|
+
table_list = [t for t in tom.model.Tables]
|
|
58
|
+
# else:
|
|
59
|
+
# table_list = [t for t in tom.model.Tables if t.Name in tables]
|
|
60
|
+
# if not table_list:
|
|
61
|
+
# raise ValueError(f"{icons.red_dot} No tables found to migrate.")
|
|
62
|
+
|
|
63
|
+
for t in table_list:
|
|
35
64
|
table_name = t.Name
|
|
36
65
|
if t.Partitions.Count == 1 and all(
|
|
37
66
|
p.Mode == TOM.ModeType.DirectLake for p in t.Partitions
|
|
@@ -51,16 +80,24 @@ def migrate_direct_lake_to_import(dataset: str | UUID, workspace: str | UUID):
|
|
|
51
80
|
table_name=table_name,
|
|
52
81
|
partition_name=partition_name,
|
|
53
82
|
expression=expression,
|
|
54
|
-
mode=
|
|
83
|
+
mode=actual_mode,
|
|
55
84
|
)
|
|
56
85
|
# Remove Direct Lake partition
|
|
57
86
|
tom.remove_object(object=p)
|
|
87
|
+
# if tables is not None:
|
|
88
|
+
# print(
|
|
89
|
+
# f"{icons.green_dot} The '{table_name}' table has been migrated to '{actual_mode}' mode."
|
|
90
|
+
# )
|
|
58
91
|
|
|
59
92
|
tom.model.Model.DefaultMode = TOM.ModeType.Import
|
|
93
|
+
# if tables is None:
|
|
94
|
+
print(
|
|
95
|
+
f"{icons.green_dot} All tables which were in Direct Lake mode have been migrated to '{actual_mode}' mode."
|
|
96
|
+
)
|
|
60
97
|
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
98
|
+
# Check
|
|
99
|
+
# for t in tom.model.Tables:
|
|
100
|
+
# if t.Partitions.Count == 1 and all(p.Mode == TOM.ModeType.Import for p in t.Partitions) and t.CalculationGroup is None:
|
|
101
|
+
# p = next(p for p in t.Partitions)
|
|
102
|
+
# print(p.Name)
|
|
103
|
+
# print(p.Source.Expression)
|
|
@@ -42,10 +42,6 @@ def migration_validation(
|
|
|
42
42
|
f"{icons.red_dot} The 'dataset' and 'new_dataset' parameters are both set to '{dataset}'. These parameters must be set to different values."
|
|
43
43
|
)
|
|
44
44
|
|
|
45
|
-
workspace = fabric.resolve_workspace_name(workspace)
|
|
46
|
-
if new_dataset_workspace is None:
|
|
47
|
-
new_dataset_workspace = workspace
|
|
48
|
-
|
|
49
45
|
icons.sll_tags.append("DirectLakeMigration")
|
|
50
46
|
|
|
51
47
|
dfA = list_semantic_model_objects(dataset=dataset, workspace=workspace)
|
sempy_labs/report/__init__.py
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
from sempy_labs.report._save_report import (
|
|
2
|
+
save_report_as_pbip,
|
|
3
|
+
)
|
|
1
4
|
from sempy_labs.report._reportwrapper import (
|
|
2
5
|
ReportWrapper,
|
|
3
6
|
)
|
|
@@ -46,4 +49,5 @@ __all__ = [
|
|
|
46
49
|
"run_report_bpa",
|
|
47
50
|
"get_report_datasources",
|
|
48
51
|
"download_report",
|
|
52
|
+
"save_report_as_pbip",
|
|
49
53
|
]
|
|
@@ -3,10 +3,11 @@ import sempy_labs._icons as icons
|
|
|
3
3
|
from typing import Optional
|
|
4
4
|
from sempy_labs._helper_functions import (
|
|
5
5
|
resolve_workspace_name_and_id,
|
|
6
|
-
|
|
6
|
+
resolve_lakehouse_name_and_id,
|
|
7
7
|
_base_api,
|
|
8
8
|
resolve_item_id,
|
|
9
9
|
_mount,
|
|
10
|
+
resolve_workspace_name,
|
|
10
11
|
)
|
|
11
12
|
from sempy_labs.lakehouse._lakehouse import lakehouse_attached
|
|
12
13
|
from uuid import UUID
|
|
@@ -44,11 +45,8 @@ def download_report(
|
|
|
44
45
|
)
|
|
45
46
|
|
|
46
47
|
(workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace)
|
|
47
|
-
lakehouse_id =
|
|
48
|
-
lakehouse_workspace =
|
|
49
|
-
lakehouse_name = resolve_lakehouse_name(
|
|
50
|
-
lakehouse_id=lakehouse_id, workspace=lakehouse_workspace
|
|
51
|
-
)
|
|
48
|
+
(lakehouse_name, lakehouse_id) = resolve_lakehouse_name_and_id()
|
|
49
|
+
lakehouse_workspace = resolve_workspace_name()
|
|
52
50
|
|
|
53
51
|
download_types = ["LiveConnect", "IncludeModel"]
|
|
54
52
|
if download_type not in download_types:
|
|
@@ -319,9 +319,9 @@ def _create_report(
|
|
|
319
319
|
|
|
320
320
|
from sempy_labs.report import report_rebind
|
|
321
321
|
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
322
|
+
(report_workspace_name, report_workspace_id) = resolve_workspace_name_and_id(
|
|
323
|
+
workspace=report_workspace
|
|
324
|
+
)
|
|
325
325
|
|
|
326
326
|
dfR = fabric.list_reports(workspace=report_workspace)
|
|
327
327
|
dfR_filt = dfR[dfR["Name"] == report]
|
|
@@ -338,7 +338,7 @@ def _create_report(
|
|
|
338
338
|
)
|
|
339
339
|
|
|
340
340
|
print(
|
|
341
|
-
f"{icons.green_dot} The '{report}' report has been created within the '{
|
|
341
|
+
f"{icons.green_dot} The '{report}' report has been created within the '{report_workspace_name}'"
|
|
342
342
|
)
|
|
343
343
|
updated_report = True
|
|
344
344
|
# Update the report if it exists
|
|
@@ -352,12 +352,12 @@ def _create_report(
|
|
|
352
352
|
status_codes=None,
|
|
353
353
|
)
|
|
354
354
|
print(
|
|
355
|
-
f"{icons.green_dot} The '{report}' report has been updated within the '{
|
|
355
|
+
f"{icons.green_dot} The '{report}' report has been updated within the '{report_workspace_name}'"
|
|
356
356
|
)
|
|
357
357
|
updated_report = True
|
|
358
358
|
else:
|
|
359
359
|
raise ValueError(
|
|
360
|
-
f"{icons.red_dot} The '{report}' report within the '{
|
|
360
|
+
f"{icons.red_dot} The '{report}' report within the '{report_workspace_name}' workspace already exists and the 'overwrite' parameter was set to False."
|
|
361
361
|
)
|
|
362
362
|
|
|
363
363
|
# Rebind the report to the semantic model to make sure it is pointed at the correct semantic model
|
|
@@ -18,6 +18,7 @@ from sempy_labs._helper_functions import (
|
|
|
18
18
|
_base_api,
|
|
19
19
|
_create_spark_session,
|
|
20
20
|
_mount,
|
|
21
|
+
resolve_workspace_id,
|
|
21
22
|
)
|
|
22
23
|
from typing import List, Optional, Union
|
|
23
24
|
from sempy._utils._log import log
|
|
@@ -115,9 +116,9 @@ def report_dependency_tree(workspace: Optional[str | UUID] = None):
|
|
|
115
116
|
dfR.rename(columns={"Name": "Report Name"}, inplace=True)
|
|
116
117
|
dfR = dfR[["Report Name", "Dataset Name"]]
|
|
117
118
|
|
|
118
|
-
report_icon = "\
|
|
119
|
-
dataset_icon = "\
|
|
120
|
-
workspace_icon = "\
|
|
119
|
+
report_icon = "\U0001f4f6"
|
|
120
|
+
dataset_icon = "\U0001f9ca"
|
|
121
|
+
workspace_icon = "\U0001f465"
|
|
121
122
|
|
|
122
123
|
node_dict = {}
|
|
123
124
|
rootNode = Node(workspace_name)
|
|
@@ -192,7 +193,7 @@ def clone_report(
|
|
|
192
193
|
target_workspace = workspace_name
|
|
193
194
|
target_workspace_id = workspace_id
|
|
194
195
|
else:
|
|
195
|
-
target_workspace_id =
|
|
196
|
+
target_workspace_id = resolve_workspace_id(workspace=target_workspace)
|
|
196
197
|
|
|
197
198
|
if target_dataset is not None:
|
|
198
199
|
if target_dataset_workspace is None:
|
|
@@ -236,15 +236,27 @@ def find_entity_property_pairs(data, result=None, keys_path=None):
|
|
|
236
236
|
keys_path = []
|
|
237
237
|
|
|
238
238
|
if isinstance(data, dict):
|
|
239
|
+
expression = data.get("Expression", {})
|
|
240
|
+
source_ref = (
|
|
241
|
+
expression.get("SourceRef", {}) if isinstance(expression, dict) else {}
|
|
242
|
+
)
|
|
243
|
+
|
|
239
244
|
if (
|
|
240
|
-
|
|
245
|
+
isinstance(source_ref, dict)
|
|
246
|
+
and "Entity" in source_ref
|
|
241
247
|
and "Property" in data
|
|
242
248
|
):
|
|
243
|
-
entity =
|
|
244
|
-
property_value = data.get("Property")
|
|
245
|
-
|
|
249
|
+
entity = source_ref.get("Entity", "")
|
|
250
|
+
property_value = data.get("Property", "")
|
|
251
|
+
|
|
252
|
+
object_type = (
|
|
253
|
+
keys_path[-1].replace("HierarchyLevel", "Hierarchy")
|
|
254
|
+
if keys_path
|
|
255
|
+
else "Unknown"
|
|
256
|
+
)
|
|
246
257
|
result[property_value] = (entity, object_type)
|
|
247
|
-
keys_path
|
|
258
|
+
if keys_path:
|
|
259
|
+
keys_path.pop()
|
|
248
260
|
|
|
249
261
|
# Recursively search the rest of the dictionary
|
|
250
262
|
for key, value in data.items():
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import sempy.fabric as fabric
|
|
2
1
|
from sempy_labs._helper_functions import (
|
|
3
2
|
resolve_dataset_id,
|
|
4
3
|
resolve_workspace_name_and_id,
|
|
5
4
|
resolve_report_id,
|
|
6
5
|
_base_api,
|
|
6
|
+
resolve_dataset_name_and_id,
|
|
7
7
|
)
|
|
8
8
|
from typing import Optional, List
|
|
9
9
|
from sempy._utils._log import log
|
|
@@ -104,10 +104,12 @@ def report_rebind_all(
|
|
|
104
104
|
f"{icons.red_dot} The 'dataset' and 'new_dataset' parameters are both set to '{dataset}'. These parameters must be set to different values."
|
|
105
105
|
)
|
|
106
106
|
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
107
|
+
(dataset_name, dataset_id) = resolve_dataset_name_and_id(
|
|
108
|
+
dataset=dataset, workspace=dataset_workspace
|
|
109
|
+
)
|
|
110
|
+
(dataset_workspace_name, dataset_workspace_id) = resolve_workspace_name_and_id(
|
|
111
|
+
workspace=dataset_workspace
|
|
112
|
+
)
|
|
111
113
|
|
|
112
114
|
if isinstance(report_workspace, str):
|
|
113
115
|
report_workspace = [report_workspace]
|
|
@@ -118,7 +120,7 @@ def report_rebind_all(
|
|
|
118
120
|
|
|
119
121
|
if len(dfR) == 0:
|
|
120
122
|
print(
|
|
121
|
-
f"{icons.info} The '{
|
|
123
|
+
f"{icons.info} The '{dataset_name}' semantic model within the '{dataset_workspace_name}' workspace has no dependent reports."
|
|
122
124
|
)
|
|
123
125
|
return
|
|
124
126
|
|