semantic-link-labs 0.11.1__py3-none-any.whl → 0.11.3__py3-none-any.whl

sempy_labs/_a_lib_info.py

@@ -1,2 +1,2 @@
 lib_name = "semanticlinklabs"
-lib_version = "0.11.1"
+lib_version = "0.11.3"

sempy_labs/_ai.py

@@ -4,7 +4,7 @@ import pandas as pd
 from typing import List, Optional, Union
 from IPython.display import display
 import sempy_labs._icons as icons
-from sempy_labs._helper_functions import (
+from .._helper_functions import (
     _read_delta_table,
     _run_spark_sql_query,
 )

sempy_labs/_capacities.py

@@ -4,12 +4,12 @@ import sempy_labs._icons as icons
 from sempy.fabric.exceptions import FabricHTTPException
 import requests
 import pandas as pd
-from sempy_labs._authentication import (
+from ._authentication import (
     _get_headers,
     ServicePrincipalTokenProvider,
 )
 from uuid import UUID
-from sempy_labs._helper_functions import (
+from ._helper_functions import (
     _is_valid_uuid,
     _update_dataframe_datatypes,
     _base_api,

sempy_labs/_capacity_migration.py

@@ -2,20 +2,20 @@ import sempy.fabric as fabric
 from typing import Optional, List
 from sempy._utils._log import log
 import sempy_labs._icons as icons
-from sempy_labs._workspaces import assign_workspace_to_capacity
-from sempy_labs.admin import (
+from ._workspaces import assign_workspace_to_capacity
+from .admin import (
     assign_workspaces_to_capacity,
 )
-from sempy_labs.admin._capacities import (
+from .admin._capacities import (
     _list_capacities_meta,
     list_capacities,
 )
-from sempy_labs._helper_functions import (
+from ._helper_functions import (
     resolve_capacity_id,
     convert_to_alphanumeric_lowercase,
     _base_api,
 )
-from sempy_labs._capacities import create_fabric_capacity
+from ._capacities import create_fabric_capacity
 from uuid import UUID
 
 

sempy_labs/_clear_cache.py

@@ -1,5 +1,5 @@
 import sempy.fabric as fabric
-from sempy_labs._helper_functions import (
+from ._helper_functions import (
     is_default_semantic_model,
     _get_adls_client,
     resolve_workspace_name_and_id,

sempy_labs/_connections.py

@@ -1,6 +1,6 @@
 import pandas as pd
 from typing import Optional
-from sempy_labs._helper_functions import (
+from ._helper_functions import (
     _is_valid_uuid,
     resolve_workspace_name_and_id,
     _update_dataframe_datatypes,
@@ -10,7 +10,7 @@ from sempy_labs._helper_functions import (
 )
 from uuid import UUID
 import sempy_labs._icons as icons
-from sempy_labs._gateways import _resolve_gateway_id
+from ._gateways import _resolve_gateway_id
 from sempy._utils._log import log
 
 

sempy_labs/_dashboards.py

@@ -1,7 +1,7 @@
 from typing import Optional
 from uuid import UUID
 import pandas as pd
-from sempy_labs._helper_functions import (
+from ._helper_functions import (
     _create_dataframe,
     _base_api,
     resolve_workspace_id,
@@ -44,22 +44,22 @@ def list_dashboards(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
 
     response = _base_api(request=f"/v1.0/myorg/groups/{workspace_id}/dashboards")
 
-    dfs = []
+    rows = []
     for v in response.json().get("value", []):
-        new_data = {
-            "Dashboard ID": v.get("id"),
-            "Dashboard Name": v.get("displayName"),
-            "Read Only": v.get("isReadOnly"),
-            "Web URL": v.get("webUrl"),
-            "Embed URL": v.get("embedUrl"),
-            "Data Classification": v.get("dataClassification"),
-            "Users": v.get("users"),
-            "Subscriptions": v.get("subscriptions"),
-        }
-        dfs.append(pd.DataFrame(new_data, index=[0]))
-
-    if dfs:
-        df = pd.concat(dfs, ignore_index=True)
+        rows.append(
+            {
+                "Dashboard ID": v.get("id"),
+                "Dashboard Name": v.get("displayName"),
+                "Read Only": v.get("isReadOnly"),
+                "Web URL": v.get("webUrl"),
+                "Embed URL": v.get("embedUrl"),
+                "Data Classification": v.get("dataClassification"),
+                "Users": v.get("users"),
+                "Subscriptions": v.get("subscriptions"),
+            }
+        )
+    if rows:
+        df = pd.DataFrame(rows, columns=list(columns.keys()))
         _update_dataframe_datatypes(dataframe=df, column_map=columns)
 
     return df

sempy_labs/_data_pipelines.py

@@ -1,6 +1,6 @@
 import pandas as pd
 from typing import Optional
-from sempy_labs._helper_functions import (
+from ._helper_functions import (
     resolve_workspace_name_and_id,
     _decode_b64,
     _base_api,

sempy_labs/_dataflows.py

@@ -1,5 +1,5 @@
 import pandas as pd
-from sempy_labs._helper_functions import (
+from ._helper_functions import (
     resolve_workspace_name_and_id,
     _is_valid_uuid,
     _update_dataframe_datatypes,
@@ -51,18 +51,19 @@ def list_dataflows(workspace: Optional[str | UUID] = None):
         request=f"/v1.0/myorg/groups/{workspace_id}/dataflows", client="fabric_sp"
     )
 
-    dfs = []
+    rows = []
     for v in response.json().get("value", []):
         gen = v.get("generation")
-        new_data = {
-            "Dataflow Id": v.get("objectId"),
-            "Dataflow Name": v.get("name"),
-            "Description": "",
-            "Configured By": v.get("configuredBy"),
-            "Users": ", ".join(v.get("users", [])),
-            "Generation": "Gen2" if gen == 2 else "Gen1",
-        }
-        dfs.append(pd.DataFrame(new_data, index=[0]))
+        rows.append(
+            {
+                "Dataflow Id": v.get("objectId"),
+                "Dataflow Name": v.get("name"),
+                "Description": "",
+                "Configured By": v.get("configuredBy"),
+                "Users": ", ".join(v.get("users", [])),
+                "Generation": "Gen2" if gen == 2 else "Gen1",
+            }
+        )
 
     responses = _base_api(
         request=f"/v1/workspaces/{workspace_id}/dataflows",
@@ -72,18 +73,19 @@ def list_dataflows(workspace: Optional[str | UUID] = None):
     for r in responses:
         for v in r.get("value", []):
             gen = v.get("generation")
-            new_data = {
-                "Dataflow Id": v.get("id"),
-                "Dataflow Name": v.get("displayName"),
-                "Description": v.get("description"),
-                "Configured By": "",
-                "Users": "",
-                "Generation": "Gen2 CI/CD",
-            }
-            dfs.append(pd.DataFrame(new_data, index=[0]))
+            rows.append(
+                {
+                    "Dataflow Id": v.get("id"),
+                    "Dataflow Name": v.get("displayName"),
+                    "Description": v.get("description"),
+                    "Configured By": "",
+                    "Users": "",
+                    "Generation": "Gen2 CI/CD",
+                }
+            )
 
-    if dfs:
-        df = pd.concat(dfs, ignore_index=True)
+    if rows:
+        df = pd.DataFrame(rows, columns=list(columns.keys()))
         _update_dataframe_datatypes(dataframe=df, column_map=columns)
 
     return df
@@ -414,6 +416,32 @@ def upgrade_dataflow(
     )
     query_groups_value = json.loads(matches[0].value) if matches else []
 
+    queries_metadata = get_jsonpath_value(
+        data=definition, path="$['pbi:mashup'].queriesMetadata"
+    )
+
+    default_staging = True if "DefaultStaging" in queries_metadata else False
+
+    # Collect keys to delete
+    keys_to_delete = [
+        key
+        for key in queries_metadata
+        if key.endswith("_DataDestination")
+        or key.endswith("_WriteToDataDestination")
+        or key.endswith("_TransformForWriteToDataDestination")
+        or key == "FastCopyStaging"
+    ]
+
+    # Delete them
+    for key in keys_to_delete:
+        del queries_metadata[key]
+
+    # Set load enabled and isHidden
+    for key, items in queries_metadata.items():
+        items["loadEnabled"] = False
+        if key in ["DefaultDestination", "DefaultStaging"]:
+            items["isHidden"] = True
+
     # Prepare the dataflow definition
     query_metadata = {
         "formatVersion": "202502",
@@ -421,9 +449,7 @@ def upgrade_dataflow(
         "name": new_dataflow_name,
         "queryGroups": query_groups_value,
         "documentLocale": get_jsonpath_value(data=definition, path="$.culture"),
-        "queriesMetadata": get_jsonpath_value(
-            data=definition, path="$['pbi:mashup'].queriesMetadata"
-        ),
+        "queriesMetadata": queries_metadata,
         "fastCombine": get_jsonpath_value(
             data=definition, path="$['pbi:mashup'].fastCombine", default=False
         ),
@@ -433,8 +459,57 @@ def upgrade_dataflow(
         # "connections": [],
     }
 
+    fast_copy = get_jsonpath_value(
+        data=definition, path="$['ppdf:fastCopy']", default=False
+    )
+    max_concurrency = get_jsonpath_value(
+        data=definition, path="$['ppdf:maxConcurrency']"
+    )
+    if fast_copy:
+        query_metadata["computeEngineSettings"] = {}
+
+        if max_concurrency:
+            query_metadata["computeEngineSettings"]["maxConcurrency"] = max_concurrency
+
     mashup_doc = get_jsonpath_value(data=definition, path="$['pbi:mashup'].document")
 
+    # Remove the FastCopyStaging section if it exists
+    new_mashup_doc = ""
+    if default_staging and fast_copy:
+        new_mashup_doc = '[DefaultOutputDestinationSettings = [DestinationDefinition = [Kind = "Reference", QueryName = "DefaultDestination", IsNewTarget = true], UpdateMethod = [Kind = "Replace"]], StagingDefinition = [Kind = "FastCopy"]]\r\nsection Section1'
+    elif default_staging and not fast_copy:
+        new_mashup_doc = '[DefaultOutputDestinationSettings = [DestinationDefinition = [Kind = "Reference", QueryName = "DefaultDestination", IsNewTarget = true], UpdateMethod = [Kind = "Replace"]]\r\nsection Section1'
+    elif not default_staging and fast_copy:
+        new_mashup_doc = '[StagingDefinition = [Kind = "FastCopy"]]\r\nsection Section1'
+    else:
+        new_mashup_doc = "section Section1"
+    for i in mashup_doc.split(";\r\nshared "):
+        # if 'IsParameterQuery=true' in i:
+        # Add to queries_metadata
+        if not (
+            "FastCopyStaging = let" in i
+            or '_WriteToDataDestination" = let' in i
+            or "_WriteToDataDestination = let" in i
+            or '_DataDestination" = let' in i
+            or "_DataDestination = let" in i
+            or '_TransformForWriteToDataDestination" = let' in i
+            or "_TransformForWriteToDataDestination = let" in i
+        ):
+            if i != "section Section1":
+                if default_staging and (
+                    "IsParameterQuery=true" not in i
+                    and not i.startswith("DefaultStaging")
+                    and not i.startswith("DefaultDestination")
+                ):
+                    new_mashup_doc += (
+                        ";\r\n[BindToDefaultDestination = true]\r\nshared " + i
+                    )
+                else:
+                    new_mashup_doc += ";\r\nshared " + i
+    new_mashup_doc = f"{new_mashup_doc};"
+
+    return new_mashup_doc, query_metadata
+
     # Add the dataflow definition to the payload
     new_definition = {
         "parts": [
@@ -445,7 +520,7 @@ def upgrade_dataflow(
             },
             {
                 "path": "mashup.pq",
-                "payload": _conv_b64(mashup_doc, json_dumps=False),
+                "payload": _conv_b64(new_mashup_doc, json_dumps=False),
                 "payloadType": "InlineBase64",
             },
         ]

sempy_labs/_dax.py

@@ -1,17 +1,17 @@
 import sempy.fabric as fabric
 import pandas as pd
-from sempy_labs._helper_functions import (
+from ._helper_functions import (
     resolve_workspace_name_and_id,
     format_dax_object_name,
     resolve_dataset_name_and_id,
     _base_api,
     generate_guid,
 )
-from sempy_labs._model_dependencies import get_model_calc_dependencies
+from ._model_dependencies import get_model_calc_dependencies
 from typing import Optional, List, Tuple
 from sempy._utils._log import log
 from uuid import UUID
-from sempy_labs.directlake._warm_cache import _put_columns_into_memory
+from .directlake._warm_cache import _put_columns_into_memory
 import sempy_labs._icons as icons
 import time
 

sempy_labs/_delta_analyzer.py

@@ -5,7 +5,7 @@ import os
 from uuid import UUID
 from typing import Dict, Optional
 import pyarrow.parquet as pq
-from sempy_labs._helper_functions import (
+from ._helper_functions import (
     create_abfss_path,
     save_as_delta_table,
     _get_column_aggregate,
@@ -21,11 +21,11 @@ from sempy_labs._helper_functions import (
     _get_delta_table,
 )
 from sempy._utils._log import log
-from sempy_labs.lakehouse._get_lakehouse_tables import get_lakehouse_tables
-from sempy_labs.lakehouse._lakehouse import (
+from .lakehouse._get_lakehouse_tables import get_lakehouse_tables
+from .lakehouse._lakehouse import (
     lakehouse_attached,
 )
-from sempy_labs.lakehouse._helper import (
+from .lakehouse._helper import (
     is_v_ordered,
 )
 import sempy_labs._icons as icons

sempy_labs/_delta_analyzer_history.py

@@ -1,7 +1,7 @@
 import pandas as pd
 from typing import Optional
 import pyarrow.parquet as pq
-from sempy_labs._helper_functions import (
+from ._helper_functions import (
     create_abfss_path,
     resolve_workspace_id,
     resolve_lakehouse_id,

sempy_labs/_deployment_pipelines.py

@@ -1,5 +1,5 @@
 import pandas as pd
-from sempy_labs._helper_functions import (
+from ._helper_functions import (
     _is_valid_uuid,
     _base_api,
     _update_dataframe_datatypes,

sempy_labs/_environments.py

@@ -1,7 +1,7 @@
 import pandas as pd
 import sempy_labs._icons as icons
 from typing import Optional
-from sempy_labs._helper_functions import (
+from ._helper_functions import (
     resolve_workspace_name_and_id,
     resolve_workspace_id,
     _base_api,
@@ -88,29 +88,30 @@ def list_environments(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
         client="fabric_sp",
     )
 
-    dfs = []
+    rows = []
     for r in responses:
         for v in r.get("value", []):
             pub = v.get("properties", {}).get("publishDetails", {})
-            new_data = {
-                "Environment Name": v.get("displayName"),
-                "Environment Id": v.get("id"),
-                "Description": v.get("description"),
-                "Publish State": pub.get("state"),
-                "Publish Target Version": pub.get("targetVersion"),
-                "Publish Start Time": pub.get("startTime"),
-                "Publish End Time": pub.get("endTime"),
-                "Spark Libraries State": pub.get("componentPublishInfo", {})
-                .get("sparkLibraries", {})
-                .get("state"),
-                "Spark Settings State": pub.get("componentPublishInfo", {})
-                .get("sparkSettings", {})
-                .get("state"),
-            }
-            dfs.append(pd.DataFrame(new_data, index=[0]))
-
-    if dfs:
-        df = pd.concat(dfs, ignore_index=True)
+            rows.append(
+                {
+                    "Environment Name": v.get("displayName"),
+                    "Environment Id": v.get("id"),
+                    "Description": v.get("description"),
+                    "Publish State": pub.get("state"),
+                    "Publish Target Version": pub.get("targetVersion"),
+                    "Publish Start Time": pub.get("startTime"),
+                    "Publish End Time": pub.get("endTime"),
+                    "Spark Libraries State": pub.get("componentPublishInfo", {})
+                    .get("sparkLibraries", {})
+                    .get("state"),
+                    "Spark Settings State": pub.get("componentPublishInfo", {})
+                    .get("sparkSettings", {})
+                    .get("state"),
+                }
+            )
+
+    if rows:
+        df = pd.DataFrame(rows, columns=list(columns.keys()))
 
     return df
 

sempy_labs/_eventhouses.py

@@ -1,6 +1,6 @@
 import pandas as pd
 from typing import Optional
-from sempy_labs._helper_functions import (
+from ._helper_functions import (
     _base_api,
     _create_dataframe,
     _conv_b64,
@@ -103,18 +103,19 @@ def list_eventhouses(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
         client="fabric_sp",
     )
 
-    dfs = []
+    rows = []
     for r in responses:
         for v in r.get("value", []):
-            new_data = {
-                "Eventhouse Name": v.get("displayName"),
-                "Eventhouse Id": v.get("id"),
-                "Description": v.get("description"),
-            }
-            dfs.append(pd.DataFrame(new_data, index=[0]))
-
-    if dfs:
-        df = pd.concat(dfs, ignore_index=True)
+            rows.append(
+                {
+                    "Eventhouse Name": v.get("displayName"),
+                    "Eventhouse Id": v.get("id"),
+                    "Description": v.get("description"),
+                }
+            )
+
+    if rows:
+        df = pd.DataFrame(rows, columns=list(columns.keys()))
 
     return df
 

sempy_labs/_eventstreams.py

@@ -1,6 +1,6 @@
 import pandas as pd
 from typing import Optional
-from sempy_labs._helper_functions import (
+from ._helper_functions import (
     _base_api,
     delete_item,
     _create_dataframe,
@@ -44,18 +44,19 @@ def list_eventstreams(workspace: Optional[str | UUID] = None) -> pd.DataFrame:
         request=f"/v1/workspaces/{workspace_id}/eventstreams", uses_pagination=True
     )
 
-    dfs = []
+    rows = []
     for r in responses:
         for v in r.get("value", []):
-            new_data = {
-                "Eventstream Name": v.get("displayName"),
-                "Eventstream Id": v.get("id"),
-                "Description": v.get("description"),
-            }
-            dfs.append(pd.DataFrame(new_data, index=[0]))
-
-    if dfs:
-        df = pd.concat(dfs, ignore_index=True)
+            rows.append(
+                {
+                    "Eventstream Name": v.get("displayName"),
+                    "Eventstream Id": v.get("id"),
+                    "Description": v.get("description"),
+                }
+            )
+
+    if rows:
+        df = pd.DataFrame(rows, columns=list(columns.keys()))
 
     return df