normalized-schema-generator 0.3.0__tar.gz → 0.4.0__tar.gz

normalized_schema_generator-0.4.0/.git

@@ -0,0 +1 @@
+gitdir: /Users/hippo/Library/CloudStorage/OneDrive-Microsoft/Github/rosikm_normalized-schema-generator/.git/worktrees/rosikm-solid-fishstick

{normalized_schema_generator-0.3.0 → normalized_schema_generator-0.4.0}/PKG-INFO

@@ -1,13 +1,12 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.3
 Name: normalized-schema-generator
-Version: 0.3.0
+Version: 0.4.0
 Summary: An anywidget for Microsoft Fabric notebooks that visually configures normalized schema JSON for Process Mining data import.
 Project-URL: Homepage, https://github.com/rosikm/normalized-schema-generator
 Project-URL: Repository, https://github.com/rosikm/normalized-schema-generator
 Project-URL: Issues, https://github.com/rosikm/normalized-schema-generator/issues
 Author: rosikm
 License: MIT
-License-File: LICENSE
 Keywords: anywidget,fabric,jupyter,process-mining,widget
 Classifier: Development Status :: 4 - Beta
 Classifier: Framework :: Jupyter
@@ -61,7 +60,7 @@ jupyter lab
 
 ## Features
 
-- **Lakehouse selection** from the current workspace
+- **Lakehouse selection** from the current workspace, including schema-enabled lakehouses
 - **Data source toggle**: Delta Tables or Files (CSV / Parquet)
 - **Events table** (required): column checklist with data type badges, include/exclude toggles
 - **Cases table** (optional): same column management, with required Events-to-Cases FK relationship

{normalized_schema_generator-0.3.0 → normalized_schema_generator-0.4.0}/README.md

@@ -40,7 +40,7 @@ jupyter lab
 
 ## Features
 
-- **Lakehouse selection** from the current workspace
+- **Lakehouse selection** from the current workspace, including schema-enabled lakehouses
 - **Data source toggle**: Delta Tables or Files (CSV / Parquet)
 - **Events table** (required): column checklist with data type badges, include/exclude toggles
 - **Cases table** (optional): same column management, with required Events-to-Cases FK relationship

{normalized_schema_generator-0.3.0 → normalized_schema_generator-0.4.0}/pyproject.toml

@@ -1,10 +1,10 @@
 [build-system]
-requires = ["hatchling"]
+requires = ["hatchling<1.27"]
 build-backend = "hatchling.build"
 
 [project]
 name = "normalized-schema-generator"
-version = "0.3.0"
+version = "0.4.0"
 description = "An anywidget for Microsoft Fabric notebooks that visually configures normalized schema JSON for Process Mining data import."
 readme = "README.md"
 license = {text = "MIT"}

{normalized_schema_generator-0.3.0 → normalized_schema_generator-0.4.0}/src/normalized_schema_generator/widget.js

@@ -96,7 +96,7 @@ export function render({ model, el }) {
     return h("div", { className: "nsw-header" },
       h("div", { style: "display:flex;align-items:baseline;gap:8px" },
         h("h2", { textContent: "Normalized Schema Generator" }),
-        h("span", { className: "nsw-version", textContent: "v0.3.0" })
+        h("span", { className: "nsw-version", textContent: "v0.4.0" })
       ),
       h("p", { textContent: "Configure your process mining star-schema and generate the dataSource JSON." })
     );

{normalized_schema_generator-0.3.0 → normalized_schema_generator-0.4.0}/src/normalized_schema_generator/widget.py

@@ -112,23 +112,90 @@ def _list_files_recursive(base_path, lakehouse_id, file_type, prefix="", depth=0
     return results
 
 
+def _quote_spark_identifier(identifier):
+    return f"`{identifier.replace('`', '``')}`"
+
+
+def _row_value(row, *names):
+    if hasattr(row, "asDict"):
+        row = row.asDict()
+    if isinstance(row, dict):
+        for name in names:
+            if name in row:
+                return row[name]
+        return None
+    for name in names:
+        if hasattr(row, name):
+            return getattr(row, name)
+    try:
+        return row[0]
+    except (IndexError, KeyError, TypeError):
+        return None
+
+
+def _list_schema_tables(lakehouse_name, lakehouse_id):
+    """Discover tables via Spark for schema-enabled lakehouses."""
+    if _spark is None:
+        return []
+
+    lakehouse = _quote_spark_identifier(lakehouse_name)
+    schemas = _spark.sql(f"SHOW SCHEMAS IN {lakehouse}").collect()
+    results = []
+    for row in schemas:
+        schema = _row_value(row, "namespace", "databaseName", "schemaName")
+        if not schema:
+            continue
+        schema_identifier = _quote_spark_identifier(schema)
+        tables = _spark.sql(
+            f"SHOW TABLES IN {lakehouse}.{schema_identifier}"
+        ).collect()
+        for table_row in tables:
+            table_name = _row_value(table_row, "tableName", "table_name", "name")
+            if not table_name:
+                continue
+            results.append({
+                "name": f"{schema}.{table_name}",
+                "path": f"/{lakehouse_id}/Tables/{schema}/{table_name}",
+                "type": "delta",
+            })
+    return results
+
+
 def _list_tables(lakehouse_name, lakehouse_id, file_type):
     if FABRIC_AVAILABLE:
-        try:
-            if file_type == 2:
+        if file_type == 2:
+            try:
                 tables = _lh_utils.listTables(lakehouse_name)
-                return [
-                    {
-                        "name": t.name,
-                        "path": f"/{lakehouse_id}/Tables/dbo/{t.name}",
+            except Exception:
+                tables = None
+
+            if tables:
+                results = []
+                for table in tables:
+                    name = table.name
+                    schema = (
+                        getattr(table, "schemaName", None)
+                        or getattr(table, "schema", None)
+                    )
+                    if schema:
+                        name = f"{schema}.{name}"
+                    else:
+                        schema = "dbo"
+                    results.append({
+                        "name": name,
+                        "path": f"/{lakehouse_id}/Tables/{schema}/{table.name}",
                         "type": "delta",
-                    }
-                    for t in tables
-                ]
-            else:
-                return _list_files_recursive("Files", lakehouse_id, file_type)
-        except Exception:
-            return []
+                    })
+                return results
+
+            schema_tables = _list_schema_tables(lakehouse_name, lakehouse_id)
+            if schema_tables or tables == []:
+                return schema_tables
+            raise RuntimeError(
+                f"Unable to list tables in lakehouse {lakehouse_name!r}; "
+                "NotebookUtils listing failed and Spark catalog discovery is unavailable."
+            )
+        return _list_files_recursive("Files", lakehouse_id, file_type)
     # Mock data
     if file_type == 2:
         return [
@@ -175,7 +242,11 @@ def _get_columns(table_name, lakehouse_name, lakehouse_id, file_type):
     if FABRIC_AVAILABLE and _spark:
         try:
             if file_type == 2:
-                full_name = f"`{lakehouse_name}`.`{table_name}`"
+                name_parts = table_name.split(".", 1)
+                identifiers = [lakehouse_name, *name_parts]
+                full_name = ".".join(
+                    _quote_spark_identifier(part) for part in identifiers
+                )
                 df = _spark.sql(f"DESCRIBE TABLE {full_name}")
                 rows = df.collect()
                 return [

normalized_schema_generator-0.4.0/tests/test_widget_tables.py

@@ -0,0 +1,92 @@
+import unittest
+from types import SimpleNamespace
+from unittest.mock import patch
+
+from normalized_schema_generator import widget
+
+
+class FakeSpark:
+    def __init__(self):
+        self.queries = []
+
+    def sql(self, query):
+        self.queries.append(query)
+        if query == "SHOW SCHEMAS IN `Sales`":
+            rows = [SimpleNamespace(namespace="dbo"), SimpleNamespace(namespace="marketing")]
+        elif query == "SHOW TABLES IN `Sales`.`dbo`":
+            rows = [SimpleNamespace(tableName="customers")]
+        elif query == "SHOW TABLES IN `Sales`.`marketing`":
+            rows = [SimpleNamespace(tableName="orders")]
+        elif query == "DESCRIBE TABLE `Sales`.`marketing`.`orders`":
+            rows = [{"col_name": "order_id", "data_type": "int"}]
+        else:
+            raise AssertionError(f"Unexpected Spark SQL query: {query}")
+        return SimpleNamespace(collect=lambda: rows)
+
+
+class LakehouseTableTests(unittest.TestCase):
+    def test_schema_catalog_fallback_lists_tables_with_schema_paths(self):
+        spark = FakeSpark()
+        lakehouse_utils = SimpleNamespace(
+            listTables=lambda _name: (_ for _ in ()).throw(
+                RuntimeError("schemas are unsupported")
+            )
+        )
+
+        with patch.object(widget, "FABRIC_AVAILABLE", True):
+            with patch.object(widget, "_lh_utils", lakehouse_utils):
+                with patch.object(widget, "_spark", spark):
+                    tables = widget._list_tables("Sales", "lakehouse-id", 2)
+
+        self.assertEqual(
+            tables,
+            [
+                {
+                    "name": "dbo.customers",
+                    "path": "/lakehouse-id/Tables/dbo/customers",
+                    "type": "delta",
+                },
+                {
+                    "name": "marketing.orders",
+                    "path": "/lakehouse-id/Tables/marketing/orders",
+                    "type": "delta",
+                },
+            ],
+        )
+
+    def test_schema_qualified_table_name_is_used_for_column_lookup(self):
+        spark = FakeSpark()
+
+        with patch.object(widget, "FABRIC_AVAILABLE", True):
+            with patch.object(widget, "_spark", spark):
+                columns = widget._get_columns(
+                    "marketing.orders", "Sales", "lakehouse-id", 2
+                )
+
+        self.assertEqual(columns, [{"name": "order_id", "dtype": "int"}])
+        self.assertIn(
+            "DESCRIBE TABLE `Sales`.`marketing`.`orders`",
+            spark.queries,
+        )
+
+    def test_legacy_listing_keeps_table_names_and_dbo_paths(self):
+        lakehouse_utils = SimpleNamespace(
+            listTables=lambda _name: [SimpleNamespace(name="events")]
+        )
+
+        with patch.object(widget, "FABRIC_AVAILABLE", True):
+            with patch.object(widget, "_lh_utils", lakehouse_utils):
+                tables = widget._list_tables("Legacy", "lakehouse-id", 2)
+
+        self.assertEqual(
+            tables,
+            [{
+                "name": "events",
+                "path": "/lakehouse-id/Tables/dbo/events",
+                "type": "delta",
+            }],
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()