semantic-link-labs 0.11.1__py3-none-any.whl → 0.11.2__py3-none-any.whl

sempy_labs/lakehouse/_get_lakehouse_tables.py

@@ -2,7 +2,7 @@ import os
 import pandas as pd
 import pyarrow.parquet as pq
 from datetime import datetime
-from sempy_labs._helper_functions import (
+from .._helper_functions import (
     _get_column_aggregate,
     resolve_workspace_name_and_id,
     resolve_lakehouse_name_and_id,
@@ -15,11 +15,11 @@ from sempy_labs._helper_functions import (
     create_abfss_path,
     _pure_python_notebook,
 )
-from sempy_labs.directlake._guardrails import (
+from ..directlake._guardrails import (
     get_sku_size,
     get_directlake_guardrails_for_sku,
 )
-from sempy_labs.lakehouse._lakehouse import lakehouse_attached
+from ._lakehouse import lakehouse_attached
 from typing import Optional
 import sempy_labs._icons as icons
 from sempy._utils._log import log
@@ -97,7 +97,7 @@ def get_lakehouse_tables(
     except Exception as e:
         API_called = False
 
-    dfs = []
+    rows = []
     local_path = None
     if API_called:
         if not responses[0].get("data"):
@@ -105,16 +105,17 @@ def get_lakehouse_tables(
 
         for r in responses:
             for i in r.get("data", []):
-                new_data = {
-                    "Workspace Name": workspace_name,
-                    "Lakehouse Name": lakehouse_name,
-                    "Schema Name": "",
-                    "Table Name": i.get("name"),
-                    "Format": i.get("format"),
-                    "Type": i.get("type"),
-                    "Location": i.get("location"),
-                }
-                dfs.append(pd.DataFrame(new_data, index=[0]))
+                rows.append(
+                    {
+                        "Workspace Name": workspace_name,
+                        "Lakehouse Name": lakehouse_name,
+                        "Schema Name": "",
+                        "Table Name": i.get("name"),
+                        "Format": i.get("format"),
+                        "Type": i.get("type"),
+                        "Location": i.get("location"),
+                    }
+                )
     else:
         local_path = _mount(lakehouse=lakehouse_id, workspace=workspace_id)
         tables_path = os.path.join(local_path, "Tables")
@@ -127,19 +128,20 @@ def get_lakehouse_tables(
                 location_path = create_abfss_path(
                     lakehouse_id, workspace_id, table_name, schema_name
                 )
-                new_data = {
-                    "Workspace Name": workspace_name,
-                    "Lakehouse Name": lakehouse_name,
-                    "Schema Name": schema_name,
-                    "Table Name": table_name,
-                    "Format": "delta",
-                    "Type": "Managed",
-                    "Location": location_path,
-                }
-                dfs.append(pd.DataFrame(new_data, index=[0]))
-
-    if dfs:
-        df = pd.concat(dfs, ignore_index=True)
+                rows.append(
+                    {
+                        "Workspace Name": workspace_name,
+                        "Lakehouse Name": lakehouse_name,
+                        "Schema Name": schema_name,
+                        "Table Name": table_name,
+                        "Format": "delta",
+                        "Type": "Managed",
+                        "Location": location_path,
+                    }
+                )
+
+    if rows:
+        df = pd.DataFrame(rows, columns=list(columns.keys()))
 
     if extended:
         sku_value = get_sku_size(workspace_id)

sempy_labs/lakehouse/_helper.py

@@ -1,7 +1,7 @@
 from uuid import UUID
 from typing import Optional, Literal
 import pyarrow.dataset as ds
-from sempy_labs._helper_functions import (
+from .._helper_functions import (
     _mount,
     delete_item,
     _base_api,
@@ -11,6 +11,7 @@ from sempy_labs._helper_functions import (
 from sempy._utils._log import log
 import sempy_labs._icons as icons
 import os
+import json
 
 
 @log
@@ -51,7 +52,34 @@ def is_v_ordered(
     )
     ds_schema = ds.dataset(table_path).schema.metadata
 
-    return any(b"vorder" in key for key in ds_schema.keys())
+    if ds_schema:
+        return any(b"vorder" in key for key in ds_schema.keys())
+
+    delta_log_path = os.path.join(table_path, "_delta_log")
+
+    def read_vorder_tag(delta_log_path):
+        json_files = sorted(
+            [f for f in os.listdir(delta_log_path) if f.endswith(".json")], reverse=True
+        )
+
+        if not json_files:
+            return False
+
+        latest_file = os.path.join(delta_log_path, json_files[0])
+
+        with open(latest_file, "r") as f:
+            for line in f:
+                try:
+                    data = json.loads(line)
+                    if "commitInfo" in data:
+                        tags = data["commitInfo"].get("tags", {})
+                        return tags.get("VORDER", "false").lower() == "true"
+                except json.JSONDecodeError:
+                    continue  # Skip malformed lines
+
+        return False  # Default if not found
+
+    return read_vorder_tag(delta_log_path)
 
 
 @log

sempy_labs/lakehouse/_lakehouse.py

@@ -2,7 +2,7 @@ from tqdm.auto import tqdm
 from typing import List, Optional, Union
 from sempy._utils._log import log
 from uuid import UUID
-from sempy_labs._helper_functions import (
+from .._helper_functions import (
     _base_api,
     resolve_lakehouse_name_and_id,
     resolve_workspace_name_and_id,
@@ -13,7 +13,7 @@ import sempy_labs._icons as icons
 import re
 import time
 import pandas as pd
-from sempy_labs._job_scheduler import (
+from .._job_scheduler import (
     _get_item_job_instance,
 )
 

sempy_labs/lakehouse/_livy_sessions.py

@@ -1,4 +1,4 @@
-from sempy_labs._helper_functions import (
+from .._helper_functions import (
     resolve_workspace_id,
     resolve_lakehouse_id,
     _base_api,
@@ -86,53 +86,58 @@ def list_livy_sessions(
         client="fabric_sp",
     )
 
-    dfs = []
+    rows = []
     for r in responses:
         for v in r.get("value", []):
             queued_duration = v.get("queuedDuration", {})
             running_duration = v.get("runningDuration", {})
             total_duration = v.get("totalDuration", {})
-            new_data = {
-                "Spark Application Id": v.get("sparkApplicationId"),
-                "State:": v.get("state"),
-                "Livy Id": v.get("livyId"),
-                "Origin": v.get("origin"),
-                "Attempt Number": v.get("attemptNumber"),
-                "Max Number Of Attempts": v.get("maxNumberOfAttempts"),
-                "Livy Name": v.get("livyName"),
-                "Submitter Id": v["submitter"].get("id"),
-                "Submitter Type": v["submitter"].get("type"),
-                "Item Workspace Id": v["item"].get("workspaceId"),
-                "Item Id": v["item"].get("itemId"),
-                "Item Reference Type": v["item"].get("referenceType"),
-                "Item Name": v.get("itemName"),
-                "Item Type": v.get("itemType"),
-                "Job Type": v.get("jobType"),
-                "Submitted Date Time": v.get("submittedDateTime"),
-                "Start Date Time": v.get("startDateTime"),
-                "End Date Time": v.get("endDateTime"),
-                "Queued Duration Value": queued_duration.get("value"),
-                "Queued Duration Time Unit": queued_duration.get("timeUnit"),
-                "Running Duration Value": running_duration.get("value"),
-                "Running Duration Time Unit": running_duration.get("timeUnit"),
-                "Total Duration Value": total_duration.get("value"),
-                "Total Duration Time Unit": total_duration.get("timeUnit"),
-                "Job Instance Id": v.get("jobInstanceId"),
-                "Creator Item Workspace Id": v["creatorItem"].get("workspaceId"),
-                "Creator Item Id": v["creatorItem"].get("itemId"),
-                "Creator Item Reference Type": v["creatorItem"].get("referenceType"),
-                "Creator Item Name": v.get("creatorItemName"),
-                "Creator Item Type": v.get("creatorItemType"),
-                "Cancellation Reason": v.get("cancellationReason"),
-                "Capacity Id": v.get("capacityId"),
-                "Operation Name": v.get("operationName"),
-                "Runtime Version": v.get("runtimeVersion"),
-                "Livy Session Item Resource Uri": v.get("livySessionItemResourceUri"),
-            }
-            dfs.append(pd.DataFrame(new_data, index=[0]))
+            rows.append(
+                {
+                    "Spark Application Id": v.get("sparkApplicationId"),
+                    "State:": v.get("state"),
+                    "Livy Id": v.get("livyId"),
+                    "Origin": v.get("origin"),
+                    "Attempt Number": v.get("attemptNumber"),
+                    "Max Number Of Attempts": v.get("maxNumberOfAttempts"),
+                    "Livy Name": v.get("livyName"),
+                    "Submitter Id": v["submitter"].get("id"),
+                    "Submitter Type": v["submitter"].get("type"),
+                    "Item Workspace Id": v["item"].get("workspaceId"),
+                    "Item Id": v["item"].get("itemId"),
+                    "Item Reference Type": v["item"].get("referenceType"),
+                    "Item Name": v.get("itemName"),
+                    "Item Type": v.get("itemType"),
+                    "Job Type": v.get("jobType"),
+                    "Submitted Date Time": v.get("submittedDateTime"),
+                    "Start Date Time": v.get("startDateTime"),
+                    "End Date Time": v.get("endDateTime"),
+                    "Queued Duration Value": queued_duration.get("value"),
+                    "Queued Duration Time Unit": queued_duration.get("timeUnit"),
+                    "Running Duration Value": running_duration.get("value"),
+                    "Running Duration Time Unit": running_duration.get("timeUnit"),
+                    "Total Duration Value": total_duration.get("value"),
+                    "Total Duration Time Unit": total_duration.get("timeUnit"),
+                    "Job Instance Id": v.get("jobInstanceId"),
+                    "Creator Item Workspace Id": v["creatorItem"].get("workspaceId"),
+                    "Creator Item Id": v["creatorItem"].get("itemId"),
+                    "Creator Item Reference Type": v["creatorItem"].get(
+                        "referenceType"
+                    ),
+                    "Creator Item Name": v.get("creatorItemName"),
+                    "Creator Item Type": v.get("creatorItemType"),
+                    "Cancellation Reason": v.get("cancellationReason"),
+                    "Capacity Id": v.get("capacityId"),
+                    "Operation Name": v.get("operationName"),
+                    "Runtime Version": v.get("runtimeVersion"),
+                    "Livy Session Item Resource Uri": v.get(
+                        "livySessionItemResourceUri"
+                    ),
+                }
+            )
 
-    if dfs:
-        df = pd.concat(dfs, ignore_index=True)
+    if rows:
+        df = pd.DataFrame(rows, columns=list(columns.keys()))
         _update_dataframe_datatypes(dataframe=df, column_map=columns)
 
     return df

sempy_labs/lakehouse/_shortcuts.py

@@ -1,6 +1,6 @@
 import sempy.fabric as fabric
 import pandas as pd
-from sempy_labs._helper_functions import (
+from .._helper_functions import (
     resolve_lakehouse_name_and_id,
     resolve_workspace_name_and_id,
     _base_api,
@@ -371,7 +371,7 @@ def list_shortcuts(
         "S3Compatible": "s3Compatible",
     }
 
-    dfs = []
+    rows = []
     for r in responses:
         for i in r.get("value", []):
             tgt = i.get("target", {})
@@ -402,25 +402,26 @@ def list_shortcuts(
                 source_item_type = dfI_filt["Type"].iloc[0]
                 source_item_name = dfI_filt["Display Name"].iloc[0]
 
-            new_data = {
-                "Shortcut Name": i.get("name"),
-                "Shortcut Path": i.get("path"),
-                "Source Type": tgt_type,
-                "Source Workspace Id": source_workspace_id,
-                "Source Workspace Name": source_workspace_name,
-                "Source Item Id": source_item_id,
-                "Source Item Name": source_item_name,
-                "Source Item Type": source_item_type,
-                "OneLake Path": tgt.get(sources.get("oneLake"), {}).get("path"),
-                "Connection Id": connection_id,
-                "Location": location,
-                "Bucket": bucket,
-                "SubPath": sub_path,
-                "Source Properties Raw": str(tgt),
-            }
-            dfs.append(pd.DataFrame(new_data, index=[0]))
+            rows.append(
+                {
+                    "Shortcut Name": i.get("name"),
+                    "Shortcut Path": i.get("path"),
+                    "Source Type": tgt_type,
+                    "Source Workspace Id": source_workspace_id,
+                    "Source Workspace Name": source_workspace_name,
+                    "Source Item Id": source_item_id,
+                    "Source Item Name": source_item_name,
+                    "Source Item Type": source_item_type,
+                    "OneLake Path": tgt.get(sources.get("oneLake"), {}).get("path"),
+                    "Connection Id": connection_id,
+                    "Location": location,
+                    "Bucket": bucket,
+                    "SubPath": sub_path,
+                    "Source Properties Raw": str(tgt),
+                }
+            )
 
-    if dfs:
-        df = pd.concat(dfs, ignore_index=True)
+    if rows:
+        df = pd.DataFrame(rows, columns=list(columns.keys()))
 
     return df

sempy_labs/migration/__init__.py

@@ -1,24 +1,24 @@
-from sempy_labs.migration._create_pqt_file import create_pqt_file
-from sempy_labs.migration._migrate_calctables_to_lakehouse import (
+from ._create_pqt_file import create_pqt_file
+from ._migrate_calctables_to_lakehouse import (
     migrate_calc_tables_to_lakehouse,
     migrate_field_parameters,
 )
-from sempy_labs.migration._migrate_calctables_to_semantic_model import (
+from ._migrate_calctables_to_semantic_model import (
     migrate_calc_tables_to_semantic_model,
 )
-from sempy_labs.migration._migrate_model_objects_to_semantic_model import (
+from ._migrate_model_objects_to_semantic_model import (
     migrate_model_objects_to_semantic_model,
 )
-from sempy_labs.migration._migrate_tables_columns_to_semantic_model import (
+from ._migrate_tables_columns_to_semantic_model import (
     migrate_tables_columns_to_semantic_model,
 )
-from sempy_labs.migration._migration_validation import (
+from ._migration_validation import (
     migration_validation,
 )
-from sempy_labs.migration._refresh_calc_tables import (
+from ._refresh_calc_tables import (
     refresh_calc_tables,
 )
-from sempy_labs.migration._direct_lake_to_import import (
+from ._direct_lake_to_import import (
     migrate_direct_lake_to_import,
 )
 

sempy_labs/migration/_create_pqt_file.py

@@ -2,12 +2,12 @@ import sempy
 import json
 import os
 import shutil
-from sempy_labs.lakehouse._lakehouse import lakehouse_attached
+from ..lakehouse._lakehouse import lakehouse_attached
 from sempy._utils._log import log
 from typing import Optional
 import sempy_labs._icons as icons
 from uuid import UUID
-from sempy_labs._helper_functions import (
+from .._helper_functions import (
     resolve_dataset_name_and_id,
     resolve_workspace_name_and_id,
 )

sempy_labs/migration/_migrate_calctables_to_lakehouse.py

@@ -2,15 +2,15 @@ import sempy
 import sempy.fabric as fabric
 import pandas as pd
 import re
-from sempy_labs.lakehouse._get_lakehouse_tables import get_lakehouse_tables
-from sempy_labs._helper_functions import (
+from ..lakehouse._get_lakehouse_tables import get_lakehouse_tables
+from .._helper_functions import (
     resolve_lakehouse_name,
     resolve_lakehouse_id,
     retry,
     generate_guid,
     save_as_delta_table,
 )
-from sempy_labs.tom import connect_semantic_model
+from ..tom import connect_semantic_model
 from typing import Optional
 from sempy._utils._log import log
 import sempy_labs._icons as icons

sempy_labs/migration/_migrate_calctables_to_semantic_model.py

@@ -1,13 +1,12 @@
 import sempy.fabric as fabric
 import re
-import datetime
-from sempy_labs.lakehouse._get_lakehouse_tables import get_lakehouse_tables
-from sempy_labs._helper_functions import (
+from ..lakehouse._get_lakehouse_tables import get_lakehouse_tables
+from .._helper_functions import (
     resolve_lakehouse_name,
     format_dax_object_name,
     retry,
 )
-from sempy_labs.tom import connect_semantic_model
+from ..tom import connect_semantic_model
 from typing import Optional
 from sempy._utils._log import log
 import sempy_labs._icons as icons

sempy_labs/migration/_migrate_model_objects_to_semantic_model.py

@@ -1,12 +1,12 @@
 import sempy
 import sempy.fabric as fabric
 import re
-from sempy_labs._helper_functions import (
+from .._helper_functions import (
     create_relationship_name,
     retry,
     format_dax_object_name,
 )
-from sempy_labs.tom import connect_semantic_model
+from ..tom import connect_semantic_model
 from typing import Optional
 from sempy._utils._log import log
 import sempy_labs._icons as icons

sempy_labs/migration/_migrate_tables_columns_to_semantic_model.py

@@ -1,9 +1,9 @@
 import sempy.fabric as fabric
 import pandas as pd
-from sempy_labs.directlake._generate_shared_expression import generate_shared_expression
-from sempy_labs._helper_functions import resolve_lakehouse_name, retry
-from sempy_labs.lakehouse._lakehouse import lakehouse_attached
-from sempy_labs.tom import connect_semantic_model
+from ..directlake._generate_shared_expression import generate_shared_expression
+from .._helper_functions import resolve_lakehouse_name, retry
+from ..lakehouse._lakehouse import lakehouse_attached
+from ..tom import connect_semantic_model
 from typing import Optional
 from sempy._utils._log import log
 import sempy_labs._icons as icons

sempy_labs/migration/_migration_validation.py

@@ -1,7 +1,6 @@
-import sempy.fabric as fabric
 import pandas as pd
 from typing import Optional
-from sempy_labs._list_functions import list_semantic_model_objects
+from .._list_functions import list_semantic_model_objects
 from sempy._utils._log import log
 import sempy_labs._icons as icons
 

sempy_labs/migration/_refresh_calc_tables.py

@@ -1,12 +1,12 @@
 import sempy.fabric as fabric
 import pandas as pd
 import re
-from sempy_labs.tom import connect_semantic_model
+from ..tom import connect_semantic_model
 from typing import Optional
 from sempy._utils._log import log
 import sempy_labs._icons as icons
 from uuid import UUID
-from sempy_labs._helper_functions import (
+from .._helper_functions import (
     resolve_workspace_name_and_id,
     resolve_dataset_name_and_id,
     save_as_delta_table,

sempy_labs/mirrored_azure_databricks_catalog/__init__.py

@@ -1,7 +1,7 @@
-from sempy_labs.mirrored_azure_databricks_catalog._refresh_catalog_metadata import (
+from ._refresh_catalog_metadata import (
     refresh_catalog_metadata,
 )
-from sempy_labs.mirrored_azure_databricks_catalog._discover import (
+from ._discover import (
     discover_catalogs,
     discover_schemas,
     discover_tables,

sempy_labs/mirrored_azure_databricks_catalog/_discover.py

@@ -1,6 +1,6 @@
 from uuid import UUID
 from typing import Optional
-from sempy_labs._helper_functions import (
+from .._helper_functions import (
     resolve_workspace_id,
     _base_api,
     _create_dataframe,
@@ -54,20 +54,20 @@ def discover_catalogs(
 
     df = _create_dataframe(columns=columns)
 
-    dfs = []
+    rows = []
     for r in responses:
         for i in r.get("value", []):
-            new_data = {
-                "Catalog Name": i.get("name"),
-                "Catalog Full Name": i.get("fullName"),
-                "Catalog Type": i.get("catalogType"),
-                "Storage Location": i.get("storageLocation"),
-            }
-
-            dfs.append(pd.DataFrame(new_data, index=[0]))
-
-    if dfs:
-        df = pd.concat(dfs, ignore_index=True)
+            rows.append(
+                {
+                    "Catalog Name": i.get("name"),
+                    "Catalog Full Name": i.get("fullName"),
+                    "Catalog Type": i.get("catalogType"),
+                    "Storage Location": i.get("storageLocation"),
+                }
+            )
+
+    if rows:
+        df = pd.DataFrame(rows, columns=list(columns.keys()))
 
     return df
 
@@ -120,20 +120,20 @@ def discover_schemas(
 
     df = _create_dataframe(columns=columns)
 
-    dfs = []
+    rows = []
     for r in responses:
         for i in r.get("value", []):
-            new_data = {
-                "Catalog Name": catalog,
-                "Schema Name": i.get("name"),
-                "Schema Full Name": i.get("fullName"),
-                "Storage Location": i.get("storageLocation"),
-            }
-
-            dfs.append(pd.DataFrame(new_data, index=[0]))
-
-    if dfs:
-        df = pd.concat(dfs, ignore_index=True)
+            rows.append(
+                {
+                    "Catalog Name": catalog,
+                    "Schema Name": i.get("name"),
+                    "Schema Full Name": i.get("fullName"),
+                    "Storage Location": i.get("storageLocation"),
+                }
+            )
+
+    if rows:
+        df = pd.DataFrame(rows, columns=list(columns.keys()))
 
     return df
 
@@ -192,22 +192,22 @@ def discover_tables(
 
     df = _create_dataframe(columns=columns)
 
-    dfs = []
+    rows = []
     for r in responses:
         for i in r.get("value", []):
-            new_data = {
-                "Catalog Name": catalog,
-                "Schema Name": schema,
-                "Table Name": i.get("name"),
-                "Table Full Name": i.get("fullName"),
-                "Storage Location": i.get("storageLocation"),
-                "Table Type": i.get("tableType"),
-                "Data Source Format": i.get("dataSourceFormat"),
-            }
-
-            dfs.append(pd.DataFrame(new_data, index=[0]))
-
-    if dfs:
-        df = pd.concat(dfs, ignore_index=True)
+            rows.append(
+                {
+                    "Catalog Name": catalog,
+                    "Schema Name": schema,
+                    "Table Name": i.get("name"),
+                    "Table Full Name": i.get("fullName"),
+                    "Storage Location": i.get("storageLocation"),
+                    "Table Type": i.get("tableType"),
+                    "Data Source Format": i.get("dataSourceFormat"),
+                }
+            )
+
+    if rows:
+        df = pd.DataFrame(rows, columns=list(columns.keys()))
 
     return df

sempy_labs/mirrored_azure_databricks_catalog/_refresh_catalog_metadata.py

@@ -1,6 +1,6 @@
 from uuid import UUID
 from typing import Optional
-from sempy_labs._helper_functions import (
+from .._helper_functions import (
     resolve_workspace_name_and_id,
     resolve_item_name_and_id,
     _base_api,