normalized-schema-generator 0.3.0__tar.gz → 0.4.1__tar.gz

normalized_schema_generator-0.4.1/.git

@@ -0,0 +1 @@
+gitdir: /Users/hippo/Library/CloudStorage/OneDrive-Microsoft/Github/rosikm_normalized-schema-generator/.git/worktrees/rosikm-solid-fishstick

{normalized_schema_generator-0.3.0 → normalized_schema_generator-0.4.1}/PKG-INFO

@@ -1,13 +1,12 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.3
 Name: normalized-schema-generator
-Version: 0.3.0
+Version: 0.4.1
 Summary: An anywidget for Microsoft Fabric notebooks that visually configures normalized schema JSON for Process Mining data import.
 Project-URL: Homepage, https://github.com/rosikm/normalized-schema-generator
 Project-URL: Repository, https://github.com/rosikm/normalized-schema-generator
 Project-URL: Issues, https://github.com/rosikm/normalized-schema-generator/issues
 Author: rosikm
 License: MIT
-License-File: LICENSE
 Keywords: anywidget,fabric,jupyter,process-mining,widget
 Classifier: Development Status :: 4 - Beta
 Classifier: Framework :: Jupyter
@@ -44,6 +43,14 @@ w
 
 The widget auto-detects the Fabric environment and populates lakehouses, tables, and columns using `notebookutils` and Spark.
 
+For schema-enabled lakehouses, it uses the [OneLake Delta metadata API](https://learn.microsoft.com/fabric/onelake/table-apis/delta-table-apis-overview)
+to discover schemas, tables, and columns. This works even when the notebook's
+default lakehouse uses the legacy Spark catalog. Authentication uses the current
+notebook identity via `notebookutils.credentials.getToken("storage")`; no extra
+credentials are required. That identity must have permission to read the selected
+lakehouse's tables. Discovery errors are shown in the widget instead of appearing
+as an empty table list.
+
 After configuring your schema, retrieve the generated JSON:
 
 ```python
@@ -61,7 +68,7 @@ jupyter lab
 
 ## Features
 
-- **Lakehouse selection** from the current workspace
+- **Lakehouse selection** from the current workspace, including schema-enabled lakehouses
 - **Data source toggle**: Delta Tables or Files (CSV / Parquet)
 - **Events table** (required): column checklist with data type badges, include/exclude toggles
 - **Cases table** (optional): same column management, with required Events-to-Cases FK relationship

{normalized_schema_generator-0.3.0 → normalized_schema_generator-0.4.1}/README.md

@@ -23,6 +23,14 @@ w
 
 The widget auto-detects the Fabric environment and populates lakehouses, tables, and columns using `notebookutils` and Spark.
 
+For schema-enabled lakehouses, it uses the [OneLake Delta metadata API](https://learn.microsoft.com/fabric/onelake/table-apis/delta-table-apis-overview)
+to discover schemas, tables, and columns. This works even when the notebook's
+default lakehouse uses the legacy Spark catalog. Authentication uses the current
+notebook identity via `notebookutils.credentials.getToken("storage")`; no extra
+credentials are required. That identity must have permission to read the selected
+lakehouse's tables. Discovery errors are shown in the widget instead of appearing
+as an empty table list.
+
 After configuring your schema, retrieve the generated JSON:
 
 ```python
@@ -40,7 +48,7 @@ jupyter lab
 
 ## Features
 
-- **Lakehouse selection** from the current workspace
+- **Lakehouse selection** from the current workspace, including schema-enabled lakehouses
 - **Data source toggle**: Delta Tables or Files (CSV / Parquet)
 - **Events table** (required): column checklist with data type badges, include/exclude toggles
 - **Cases table** (optional): same column management, with required Events-to-Cases FK relationship

{normalized_schema_generator-0.3.0 → normalized_schema_generator-0.4.1}/pyproject.toml

@@ -1,10 +1,10 @@
 [build-system]
-requires = ["hatchling"]
+requires = ["hatchling<1.27"]
 build-backend = "hatchling.build"
 
 [project]
 name = "normalized-schema-generator"
-version = "0.3.0"
+version = "0.4.1"
 description = "An anywidget for Microsoft Fabric notebooks that visually configures normalized schema JSON for Process Mining data import."
 readme = "README.md"
 license = {text = "MIT"}

{normalized_schema_generator-0.3.0 → normalized_schema_generator-0.4.1}/src/normalized_schema_generator/widget.js

@@ -96,7 +96,7 @@ export function render({ model, el }) {
     return h("div", { className: "nsw-header" },
       h("div", { style: "display:flex;align-items:baseline;gap:8px" },
         h("h2", { textContent: "Normalized Schema Generator" }),
-        h("span", { className: "nsw-version", textContent: "v0.3.0" })
+        h("span", { className: "nsw-version", textContent: "v0.4.1" })
       ),
       h("p", { textContent: "Configure your process mining star-schema and generate the dataSource JSON." })
     );
@@ -117,8 +117,8 @@ export function render({ model, el }) {
       h("span", { className: "nsw-label", textContent: "Lakehouse" }),
       selectEl(lhOpts, selId, (val) => {
         const lh = lakehouses.find(l => l.id === val);
-        model.set("selected_lakehouse_id", val);
         model.set("selected_lakehouse_name", lh ? lh.displayName : "");
+        model.set("selected_lakehouse_id", val);
         model.save_changes();
       }, "Select a lakehouse...", lhLoading)
     );

{normalized_schema_generator-0.3.0 → normalized_schema_generator-0.4.1}/src/normalized_schema_generator/widget.py

@@ -18,6 +18,9 @@ For local development (outside Fabric), mock data is used automatically.
 
 import json
 import pathlib
+from urllib.error import HTTPError, URLError
+from urllib.parse import quote, urlencode
+from urllib.request import Request, urlopen
 
 import anywidget
 import traitlets
@@ -37,6 +40,11 @@ except ImportError:
     _runtime_utils = None
     _fs_utils = None
 
+try:
+    from notebookutils import credentials as _credentials_utils
+except ImportError:
+    _credentials_utils = None
+
 try:
     from pyspark.sql import SparkSession
     _spark = SparkSession.builder.getOrCreate()
@@ -112,23 +120,128 @@ def _list_files_recursive(base_path, lakehouse_id, file_type, prefix="", depth=0
     return results
 
 
+def _quote_spark_identifier(identifier):
+    return f"`{identifier.replace('`', '``')}`"
+
+
+def _onelake_metadata(lakehouse_id, resource, **params):
+    """Read catalog metadata independently of the notebook's Spark catalog."""
+    workspace_id = _get_fabric_context()["workspace_id"]
+    if not workspace_id:
+        raise RuntimeError("The current Fabric workspace ID is unavailable.")
+    if _credentials_utils is None:
+        raise RuntimeError("NotebookUtils credentials are required for OneLake metadata.")
+    token = _credentials_utils.getToken("storage")
+    if not token:
+        raise RuntimeError("NotebookUtils did not return a OneLake access token.")
+    base_url = (
+        "https://onelake.table.fabric.microsoft.com/delta/"
+        f"{quote(workspace_id, safe='')}/{quote(lakehouse_id, safe='')}"
+        "/api/2.1/unity-catalog/"
+    )
+    query = urlencode({"catalog_name": lakehouse_id, **params})
+    request = Request(base_url + resource + "?" + query, headers={
+        "Authorization": f"Bearer {token}",
+        "Accept": "application/json",
+    })
+    try:
+        with urlopen(request, timeout=30) as response:
+            payload = json.load(response)
+    except HTTPError as exc:
+        raise RuntimeError(
+            f"OneLake metadata request failed (HTTP {exc.code}). "
+            "Check lakehouse access and OneLake permissions."
+        ) from None
+    except URLError:
+        raise RuntimeError("Unable to connect to the OneLake metadata API.") from None
+    if not isinstance(payload, dict):
+        raise RuntimeError("The OneLake metadata API returned an invalid response.")
+    return payload
+
+
+def _onelake_items(lakehouse_id, resource, **params):
+    page_token = None
+    seen_tokens = set()
+    while True:
+        query = dict(params)
+        if page_token:
+            query["page_token"] = page_token
+        payload = _onelake_metadata(lakehouse_id, resource, **query)
+        items = payload.get(resource)
+        if not isinstance(items, list) or any(not isinstance(item, dict) for item in items):
+            raise RuntimeError(f"The OneLake metadata API returned invalid {resource}.")
+        yield from items
+        page_token = payload.get("next_page_token")
+        if not page_token:
+            break
+        if not isinstance(page_token, str) or page_token in seen_tokens:
+            raise RuntimeError("The OneLake metadata API returned an invalid page token.")
+        seen_tokens.add(page_token)
+
+
+def _list_schema_tables(lakehouse_id):
+    results = []
+    for schema in _onelake_items(lakehouse_id, "schemas"):
+        schema_name = schema["name"]
+        for table in _onelake_items(lakehouse_id, "tables", schema_name=schema_name):
+            table_name = table["name"]
+            results.append({
+                "name": f"{schema_name}.{table_name}",
+                "path": f"/{lakehouse_id}/Tables/{schema_name}/{table_name}",
+                "type": "delta",
+            })
+    return results
+
+
+def _get_schema_columns(table_name, lakehouse_id):
+    schema, name = table_name.split(".", 1)
+    full_name = quote(f"{lakehouse_id}.{schema}.{name}", safe="")
+    table = _onelake_metadata(
+        lakehouse_id, f"tables/{full_name}", schema_name=schema
+    )
+    columns = table.get("columns")
+    if not isinstance(columns, list):
+        raise RuntimeError("The OneLake metadata API did not return table columns.")
+    return [
+        {
+            "name": column["name"],
+            "dtype": _spark_type_to_simple(
+                column.get("type_text") or column["type_name"]
+            ),
+        }
+        for column in columns
+    ]
+
+
 def _list_tables(lakehouse_name, lakehouse_id, file_type):
     if FABRIC_AVAILABLE:
-        try:
-            if file_type == 2:
+        if file_type == 2:
+            try:
                 tables = _lh_utils.listTables(lakehouse_name)
-                return [
-                    {
-                        "name": t.name,
-                        "path": f"/{lakehouse_id}/Tables/dbo/{t.name}",
+            except Exception:
+                tables = None
+
+            if tables:
+                results = []
+                for table in tables:
+                    name = table.name
+                    schema = (
+                        getattr(table, "schemaName", None)
+                        or getattr(table, "schema", None)
+                    )
+                    if schema:
+                        name = f"{schema}.{name}"
+                    else:
+                        schema = "dbo"
+                    results.append({
+                        "name": name,
+                        "path": f"/{lakehouse_id}/Tables/{schema}/{table.name}",
                         "type": "delta",
-                    }
-                    for t in tables
-                ]
-            else:
-                return _list_files_recursive("Files", lakehouse_id, file_type)
-        except Exception:
-            return []
+                    })
+                return results
+
+            return _list_schema_tables(lakehouse_id)
+        return _list_files_recursive("Files", lakehouse_id, file_type)
     # Mock data
     if file_type == 2:
         return [
@@ -172,29 +285,32 @@ def _spark_type_to_simple(dtype_str):
 
 
 def _get_columns(table_name, lakehouse_name, lakehouse_id, file_type):
+    if FABRIC_AVAILABLE and file_type == 2 and "." in table_name:
+        return _get_schema_columns(table_name, lakehouse_id)
+    if FABRIC_AVAILABLE and _spark is None:
+        raise RuntimeError("Spark is required to load columns for this source.")
     if FABRIC_AVAILABLE and _spark:
-        try:
-            if file_type == 2:
-                full_name = f"`{lakehouse_name}`.`{table_name}`"
-                df = _spark.sql(f"DESCRIBE TABLE {full_name}")
-                rows = df.collect()
-                return [
-                    {"name": r["col_name"], "dtype": _spark_type_to_simple(r["data_type"])}
-                    for r in rows
-                    if r["col_name"] and not r["col_name"].startswith("#")
-                ]
+        if file_type == 2:
+            full_name = ".".join(
+                _quote_spark_identifier(part) for part in (lakehouse_name, table_name)
+            )
+            df = _spark.sql(f"DESCRIBE TABLE {full_name}")
+            rows = df.collect()
+            return [
+                {"name": r["col_name"], "dtype": _spark_type_to_simple(r["data_type"])}
+                for r in rows
+                if r["col_name"] and not r["col_name"].startswith("#")
+            ]
+        else:
+            base = f"Files/{table_name}"
+            if file_type == 0:
+                df = _spark.read.option("header", "true").option("inferSchema", "true").csv(base)
             else:
-                base = f"Files/{table_name}"
-                if file_type == 0:
-                    df = _spark.read.option("header", "true").option("inferSchema", "true").csv(base)
-                else:
-                    df = _spark.read.parquet(base)
-                return [
-                    {"name": f.name, "dtype": _spark_type_to_simple(f.dataType.simpleString())}
-                    for f in df.schema.fields
-                ]
-        except Exception:
-            return []
+                df = _spark.read.parquet(base)
+            return [
+                {"name": f.name, "dtype": _spark_type_to_simple(f.dataType.simpleString())}
+                for f in df.schema.fields
+            ]
     # Mock data
     _mock = {
         "events": [
@@ -274,6 +390,7 @@ class NormalizedSchemaWidget(anywidget.AnyWidget):
     config_json = traitlets.Unicode("").tag(sync=True)
     loading_status = traitlets.Unicode("").tag(sync=True)
     error_message = traitlets.Unicode("").tag(sync=True)
+    _data_error = traitlets.Unicode("")
 
     # Fabric context
     workspace_id = traitlets.Unicode("").tag(sync=True)
@@ -290,10 +407,17 @@ class NormalizedSchemaWidget(anywidget.AnyWidget):
 
     @traitlets.observe("selected_lakehouse_id")
     def _on_lakehouse_change(self, change):
+        self._data_error = ""
+        self.error_message = ""
         lh_id = change["new"]
         if not lh_id:
             self.available_tables = []
+            self.config_json = ""
             return
+        for lakehouse in self.lakehouses:
+            if lakehouse["id"] == lh_id:
+                self.selected_lakehouse_name = lakehouse["displayName"]
+                break
         self.loading_status = "Loading tables..."
         self.events_table_name = ""
         self.events_columns = []
@@ -307,8 +431,8 @@ class NormalizedSchemaWidget(anywidget.AnyWidget):
                 self.selected_lakehouse_name, lh_id, self.data_source_file_type
             )
         except Exception as e:
-            self.error_message = f"Failed to load tables: {e}"
             self.available_tables = []
+            self._data_error = f"Failed to load tables: {e}"
         self.loading_status = ""
         self._generate_config()
 
@@ -316,6 +440,8 @@ class NormalizedSchemaWidget(anywidget.AnyWidget):
     def _on_file_type_change(self, change):
         if not self.selected_lakehouse_id:
             return
+        self._data_error = ""
+        self.error_message = ""
         self.loading_status = "Reloading tables..."
         self.events_table_name = ""
         self.events_columns = []
@@ -331,13 +457,15 @@ class NormalizedSchemaWidget(anywidget.AnyWidget):
                 change["new"],
             )
         except Exception as e:
-            self.error_message = f"Failed to load tables: {e}"
             self.available_tables = []
+            self._data_error = f"Failed to load tables: {e}"
         self.loading_status = ""
         self._generate_config()
 
     @traitlets.observe("events_table_name")
     def _on_events_table_change(self, change):
+        self._data_error = ""
+        self.error_message = ""
         name = change["new"]
         if not name:
             self.events_columns = []
@@ -354,13 +482,15 @@ class NormalizedSchemaWidget(anywidget.AnyWidget):
                 for c in raw
             ]
         except Exception as e:
-            self.error_message = f"Failed to load columns: {e}"
             self.events_columns = []
+            self._data_error = f"Failed to load columns: {e}"
         self.loading_status = ""
         self._generate_config()
 
     @traitlets.observe("cases_table_name")
     def _on_cases_table_change(self, change):
+        self._data_error = ""
+        self.error_message = ""
         name = change["new"]
         if not name:
             self.cases_columns = []
@@ -379,8 +509,8 @@ class NormalizedSchemaWidget(anywidget.AnyWidget):
                 for c in raw
             ]
         except Exception as e:
-            self.error_message = f"Failed to load columns: {e}"
             self.cases_columns = []
+            self._data_error = f"Failed to load columns: {e}"
         self.loading_status = ""
         self._generate_config()
 
@@ -394,6 +524,8 @@ class NormalizedSchemaWidget(anywidget.AnyWidget):
         if len(parts) < 2:
             return
         table_name = parts[1]
+        self._data_error = ""
+        self.error_message = ""
         try:
             raw = _get_columns(
                 table_name, self.selected_lakehouse_name,
@@ -404,8 +536,9 @@ class NormalizedSchemaWidget(anywidget.AnyWidget):
                 for c in raw
             ]
         except Exception as e:
-            self.error_message = f"Failed to load columns for {table_name}: {e}"
             self.fetch_columns_response = []
+            self._data_error = f"Failed to load columns for {table_name}: {e}"
+        self._generate_config()
 
     @traitlets.observe("events_columns", "events_joins", "cases_enabled",
                        "cases_columns", "cases_joins", "cases_relationship")
@@ -448,8 +581,8 @@ class NormalizedSchemaWidget(anywidget.AnyWidget):
         }.get(dtype, "String")
 
     def _generate_config(self, sync_attributes=True):
-        self.error_message = ""
-        if not self.selected_lakehouse_id or not self.events_table_name:
+        self.error_message = self._data_error
+        if self._data_error or not self.selected_lakehouse_id or not self.events_table_name:
             self.config_json = ""
             return
 

normalized_schema_generator-0.4.1/tests/test_widget_tables.py

@@ -0,0 +1,316 @@
+import io
+import json
+import unittest
+from types import SimpleNamespace
+from urllib.error import HTTPError, URLError
+from urllib.parse import parse_qs, urlsplit
+from unittest.mock import Mock, patch
+
+with patch.dict("sys.modules", {"pyspark.sql": None}):
+    from normalized_schema_generator import widget
+
+
+class LakehouseTableTests(unittest.TestCase):
+    def setUp(self):
+        self.lakehouse = Mock()
+        self.lakehouse.listTables.side_effect = RuntimeError(
+            "UnsupportedOperationForSchemasEnabledLakehouse"
+        )
+        self.lakehouse.list.return_value = [
+            SimpleNamespace(id="lakehouse-id", displayName="Sales"),
+        ]
+        self.spark = Mock()
+        self.spark.sql.side_effect = RuntimeError(
+            "[SCHEMA_NOT_FOUND] The schema spark_catalog cannot be found."
+        )
+        self.credentials = Mock()
+        self.credentials.getToken.return_value = "test-access-token"
+        patches = {
+            "FABRIC_AVAILABLE": True,
+            "_spark": self.spark,
+            "_lh_utils": self.lakehouse,
+            "_credentials_utils": self.credentials,
+            "_get_fabric_context": Mock(return_value={
+                "workspace_id": "workspace-id", "tenant_id": "tenant-id",
+            }),
+        }
+        for name, value in patches.items():
+            self.enter_patch(name, value)
+        self.urlopen = self.enter_patch("urlopen", Mock())
+
+    def enter_patch(self, name, value):
+        patcher = patch.object(widget, name, value)
+        self.addCleanup(patcher.stop)
+        return patcher.start()
+
+    def respond(self, *payloads):
+        self.urlopen.side_effect = [
+            io.BytesIO(json.dumps(payload).encode("utf-8"))
+            for payload in payloads
+        ]
+
+    def request_query(self, index=0):
+        request = self.urlopen.call_args_list[index].args[0]
+        return parse_qs(urlsplit(request.full_url).query)
+
+    def new_widget(self):
+        instance = widget.NormalizedSchemaWidget()
+        self.addCleanup(instance.close)
+        return instance
+
+    def test_schema_discovery_does_not_use_incompatible_spark_catalog(self):
+        self.respond(
+            {"schemas": [{"name": "bronze"}, {"name": "dbo"}]},
+            {"tables": [{"name": "events", "data_source_format": "DELTA"}]},
+            {"tables": [{"name": "events", "data_source_format": "DELTA"}]},
+        )
+
+        tables = widget._list_tables("Sales", "lakehouse-id", 2)
+
+        self.assertEqual(tables, [
+            {
+                "name": "bronze.events",
+                "path": "/lakehouse-id/Tables/bronze/events",
+                "type": "delta",
+            },
+            {
+                "name": "dbo.events",
+                "path": "/lakehouse-id/Tables/dbo/events",
+                "type": "delta",
+            },
+        ])
+        self.spark.sql.assert_not_called()
+        self.credentials.getToken.assert_called_with("storage")
+        request = self.urlopen.call_args_list[0].args[0]
+        self.assertEqual(urlsplit(request.full_url).path,
+            "/delta/workspace-id/lakehouse-id/api/2.1/unity-catalog/schemas")
+        self.assertEqual(request.get_header("Authorization"), "Bearer test-access-token")
+        self.assertEqual(self.request_query(), {"catalog_name": ["lakehouse-id"]})
+        self.assertEqual(self.request_query(1)["schema_name"], ["bronze"])
+        self.assertEqual(self.urlopen.call_args.kwargs["timeout"], 30)
+
+    def test_empty_notebookutils_listing_also_uses_onelake(self):
+        self.lakehouse.listTables.side_effect = None
+        self.lakehouse.listTables.return_value = []
+        self.respond(
+            {"schemas": [{"name": "bronze"}]},
+            {"tables": [{"name": "events", "data_source_format": "DELTA"}]},
+        )
+        self.assertEqual(
+            widget._list_tables("Sales", "lakehouse-id", 2)[0]["name"],
+            "bronze.events",
+        )
+
+    def test_schema_and_table_pagination(self):
+        self.respond(
+            {"schemas": [{"name": "bronze"}], "next_page_token": "schema page 2"},
+            {"tables": [{"name": "events", "data_source_format": "DELTA"}],
+             "next_page_token": "table page +2"},
+            {"tables": [{"name": "cases", "data_source_format": "DELTA"}]},
+            {"schemas": [{"name": "silver"}]},
+            {"tables": [{"name": "events", "data_source_format": "DELTA"}]},
+        )
+        tables = widget._list_tables("Sales", "lakehouse-id", 2)
+
+        self.assertEqual([table["name"] for table in tables],
+                         ["bronze.events", "bronze.cases", "silver.events"])
+        self.assertEqual(self.request_query(2)["page_token"], ["table page +2"])
+        self.assertEqual(self.request_query(3)["page_token"], ["schema page 2"])
+        self.assertNotIn("page_token", self.request_query(4))
+
+    def test_repeated_pagination_token_raises_instead_of_looping(self):
+        self.respond(
+            {"schemas": [], "next_page_token": "same"},
+            {"schemas": [], "next_page_token": "same"},
+        )
+        with self.assertRaisesRegex(RuntimeError, "invalid page token"):
+            widget._list_tables("Sales", "lakehouse-id", 2)
+
+    def test_empty_lakehouse_is_a_valid_empty_list(self):
+        self.respond({"schemas": [{"name": "dbo"}]}, {"tables": []})
+        self.assertEqual(widget._list_tables("Sales", "lakehouse-id", 2), [])
+
+    def test_invalid_metadata_is_not_treated_as_empty_lakehouse(self):
+        for payload in ([], {}, {"schemas": None}, {"schemas": ["dbo"]}):
+            with self.subTest(payload=payload):
+                self.respond(payload)
+                with self.assertRaisesRegex(RuntimeError, "invalid"):
+                    widget._list_tables("Sales", "lakehouse-id", 2)
+
+    def test_http_error_is_actionable_and_does_not_expose_token(self):
+        self.urlopen.side_effect = HTTPError(
+            "https://onelake.table.fabric.microsoft.com", 403, "Forbidden", {}, None
+        )
+        with self.assertRaisesRegex(RuntimeError, "HTTP 403") as caught:
+            widget._list_tables("Sales", "lakehouse-id", 2)
+        self.assertNotIn("test-access-token", str(caught.exception))
+
+    def test_connection_error_is_not_an_empty_list(self):
+        self.urlopen.side_effect = URLError("Connection refused")
+        with self.assertRaisesRegex(RuntimeError, "Unable to connect"):
+            widget._list_tables("Sales", "lakehouse-id", 2)
+
+    def test_missing_workspace_or_credentials_fail_explicitly(self):
+        with patch.object(widget, "_get_fabric_context", return_value={
+            "workspace_id": "", "tenant_id": "",
+        }):
+            with self.assertRaisesRegex(RuntimeError, "workspace ID"):
+                widget._list_tables("Sales", "lakehouse-id", 2)
+        with patch.object(widget, "_credentials_utils", None):
+            with self.assertRaisesRegex(RuntimeError, "credentials"):
+                widget._list_tables("Sales", "lakehouse-id", 2)
+        self.urlopen.assert_not_called()
+
+    def test_missing_access_token_fails_before_request(self):
+        self.credentials.getToken.return_value = ""
+        with self.assertRaisesRegex(RuntimeError, "access token"):
+            widget._list_tables("Sales", "lakehouse-id", 2)
+        self.urlopen.assert_not_called()
+
+    def test_schema_columns_do_not_depend_on_spark(self):
+        self.respond({"columns": [
+            {"name": "CaseID", "type_text": None, "type_name": "long"},
+            {"name": "Cost", "type_text": "decimal(18,2)", "type_name": "decimal"},
+            {"name": "Activity", "type_name": "string"},
+            {"name": "Started", "type_name": "timestamp"},
+        ]})
+        with patch.object(widget, "_spark", None):
+            columns = widget._get_columns(
+                "bronze.events", "Sales", "lakehouse-id", 2
+            )
+        self.assertEqual(columns, [
+            {"name": "CaseID", "dtype": "int"},
+            {"name": "Cost", "dtype": "double"},
+            {"name": "Activity", "dtype": "string"},
+            {"name": "Started", "dtype": "timestamp"},
+        ])
+        request = self.urlopen.call_args.args[0]
+        self.assertTrue(urlsplit(request.full_url).path.endswith(
+            "/tables/lakehouse-id.bronze.events"
+        ))
+        self.assertEqual(self.request_query()["schema_name"], ["bronze"])
+        self.spark.sql.assert_not_called()
+
+    def test_missing_columns_fail_explicitly(self):
+        self.respond({"columns": None})
+        with self.assertRaisesRegex(RuntimeError, "did not return table columns"):
+            widget._get_columns("bronze.events", "Sales", "lakehouse-id", 2)
+
+    def test_legacy_listing_does_not_call_onelake(self):
+        self.lakehouse.listTables.side_effect = None
+        self.lakehouse.listTables.return_value = [SimpleNamespace(name="events")]
+
+        self.assertEqual(widget._list_tables("Legacy", "lakehouse-id", 2), [{
+            "name": "events",
+            "path": "/lakehouse-id/Tables/dbo/events",
+            "type": "delta",
+        }])
+        self.urlopen.assert_not_called()
+        self.spark.sql.assert_not_called()
+
+    def test_legacy_columns_preserve_spark_identifier_escaping(self):
+        self.spark.sql.side_effect = None
+        self.spark.sql.return_value.collect.return_value = [
+            {"col_name": "CaseID", "data_type": "bigint"},
+        ]
+        self.assertEqual(widget._get_columns("events", "Sales`LH", "lakehouse-id", 2),
+                         [{"name": "CaseID", "dtype": "int"}])
+        self.spark.sql.assert_called_once_with("DESCRIBE TABLE `Sales``LH`.`events`")
+        self.urlopen.assert_not_called()
+
+    def test_legacy_column_errors_propagate(self):
+        with self.assertRaisesRegex(RuntimeError, "SCHEMA_NOT_FOUND"):
+            widget._get_columns("events", "Sales", "lakehouse-id", 2)
+
+    def test_table_error_survives_config_generation_and_recovers_on_reload(self):
+        instance = self.new_widget()
+        self.urlopen.side_effect = HTTPError(
+            "https://onelake.table.fabric.microsoft.com", 403, "Forbidden", {}, None
+        )
+        instance.selected_lakehouse_name = "Stale selection"
+        instance.selected_lakehouse_id = "lakehouse-id"
+        self.assertEqual(instance.selected_lakehouse_name, "Sales")
+        self.assertEqual(instance.available_tables, [])
+        self.assertIn("HTTP 403", instance.error_message)
+        self.assertEqual(instance.loading_status, "")
+        instance._generate_config()
+        self.assertIn("HTTP 403", instance.error_message)
+        self.assertEqual(instance.config_json, "")
+
+        self.respond(
+            {"schemas": [{"name": "bronze"}]},
+            {"tables": [{"name": "events", "data_source_format": "DELTA"}]},
+        )
+        instance._on_lakehouse_change({"new": "lakehouse-id"})
+        self.assertEqual(instance.error_message, "")
+        self.assertEqual(instance.available_tables[0]["name"], "bronze.events")
+
+    def test_source_type_reload_error_remains_visible(self):
+        instance = self.new_widget()
+        with patch.object(widget, "_list_tables", return_value=[]):
+            instance.selected_lakehouse_id = "lakehouse-id"
+        with patch.object(widget, "_list_tables", side_effect=RuntimeError("Access denied")):
+            instance.data_source_file_type = 1
+        self.assertIn("Access denied", instance.error_message)
+        self.assertEqual(instance.loading_status, "")
+
+    def test_column_errors_remain_visible_for_events_cases_and_references(self):
+        for target in ("events", "cases", "reference"):
+            with self.subTest(target=target):
+                instance = self.new_widget()
+                with patch.object(widget, "_list_tables", return_value=[]):
+                    instance.selected_lakehouse_id = "lakehouse-id"
+                with patch.object(widget, "_get_columns",
+                                  side_effect=RuntimeError("Metadata denied")):
+                    if target == "reference":
+                        instance.fetch_columns_response = [{"name": "stale"}]
+                        instance.fetch_columns_request = "1:bronze.activity"
+                        self.assertEqual(instance.fetch_columns_response, [])
+                    else:
+                        setattr(instance, f"{target}_table_name", "bronze.events")
+                        self.assertEqual(getattr(instance, f"{target}_columns"), [])
+                instance._generate_config()
+                self.assertIn("Metadata denied", instance.error_message)
+                self.assertEqual(instance.config_json, "")
+                self.assertEqual(instance.loading_status, "")
+
+    def test_schema_selection_generates_qualified_dataset_and_join_paths(self):
+        instance = self.new_widget()
+        self.respond(
+            {"schemas": [{"name": "bronze"}, {"name": "silver"}]},
+            {"tables": [
+                {"name": "events", "data_source_format": "DELTA"},
+                {"name": "cases", "data_source_format": "DELTA"},
+            ]},
+            {"tables": [{"name": "events", "data_source_format": "DELTA"}]},
+            {"columns": [{"name": "CaseID", "type_name": "long"}]},
+            {"columns": [{"name": "CaseID", "type_name": "long"}]},
+            {"columns": [{"name": "CaseID", "type_name": "long"}]},
+        )
+        instance.selected_lakehouse_id = "lakehouse-id"
+        instance.events_table_name = "bronze.events"
+        instance.cases_enabled = True
+        instance.cases_table_name = "bronze.cases"
+        instance.cases_relationship = {"fk_column": "CaseID", "pk_column": "CaseID"}
+        instance.fetch_columns_request = "1:silver.events"
+        instance.events_joins = [{
+            "target_table": "silver.events", "source_column": "CaseID",
+            "target_column": "CaseID", "join_key_type": "Integer",
+            "target_columns": [{"name": "CaseID", "dtype": "int", "included": True}],
+        }]
+
+        self.assertEqual(instance.error_message, "")
+        config = json.loads(instance.config_json)
+        datasets = config["inputDataBinding"]["dataSource"]["datasets"]
+        self.assertEqual([(d["Name"], d["Path"]) for d in datasets], [
+            ("bronze.events", "/lakehouse-id/Tables/bronze/events"),
+            ("bronze.cases", "/lakehouse-id/Tables/bronze/cases"),
+            ("silver.events", "/lakehouse-id/Tables/silver/events"),
+        ])
+        self.assertEqual([j["TargetDatasetName"] for j in datasets[0]["Join"]],
+                         ["bronze.cases", "silver.events"])
+        self.spark.sql.assert_not_called()
+
+
+if __name__ == "__main__":
+    unittest.main()