normalized-schema-generator 0.2.0__tar.gz → 0.4.0__tar.gz

normalized_schema_generator-0.4.0/.git

@@ -0,0 +1 @@
+gitdir: /Users/hippo/Library/CloudStorage/OneDrive-Microsoft/Github/rosikm_normalized-schema-generator/.git/worktrees/rosikm-solid-fishstick

{normalized_schema_generator-0.2.0 → normalized_schema_generator-0.4.0}/PKG-INFO

@@ -1,13 +1,12 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.3
 Name: normalized-schema-generator
-Version: 0.2.0
+Version: 0.4.0
 Summary: An anywidget for Microsoft Fabric notebooks that visually configures normalized schema JSON for Process Mining data import.
 Project-URL: Homepage, https://github.com/rosikm/normalized-schema-generator
 Project-URL: Repository, https://github.com/rosikm/normalized-schema-generator
 Project-URL: Issues, https://github.com/rosikm/normalized-schema-generator/issues
 Author: rosikm
 License: MIT
-License-File: LICENSE
 Keywords: anywidget,fabric,jupyter,process-mining,widget
 Classifier: Development Status :: 4 - Beta
 Classifier: Framework :: Jupyter
@@ -61,7 +60,7 @@ jupyter lab
 
 ## Features
 
-- **Lakehouse selection** from the current workspace
+- **Lakehouse selection** from the current workspace, including schema-enabled lakehouses
 - **Data source toggle**: Delta Tables or Files (CSV / Parquet)
 - **Events table** (required): column checklist with data type badges, include/exclude toggles
 - **Cases table** (optional): same column management, with required Events-to-Cases FK relationship

{normalized_schema_generator-0.2.0 → normalized_schema_generator-0.4.0}/README.md

@@ -40,7 +40,7 @@ jupyter lab
 
 ## Features
 
-- **Lakehouse selection** from the current workspace
+- **Lakehouse selection** from the current workspace, including schema-enabled lakehouses
 - **Data source toggle**: Delta Tables or Files (CSV / Parquet)
 - **Events table** (required): column checklist with data type badges, include/exclude toggles
 - **Cases table** (optional): same column management, with required Events-to-Cases FK relationship

{normalized_schema_generator-0.2.0 → normalized_schema_generator-0.4.0}/pyproject.toml

@@ -1,10 +1,10 @@
 [build-system]
-requires = ["hatchling"]
+requires = ["hatchling<1.27"]
 build-backend = "hatchling.build"
 
 [project]
 name = "normalized-schema-generator"
-version = "0.2.0"
+version = "0.4.0"
 description = "An anywidget for Microsoft Fabric notebooks that visually configures normalized schema JSON for Process Mining data import."
 readme = "README.md"
 license = {text = "MIT"}

{normalized_schema_generator-0.2.0 → normalized_schema_generator-0.4.0}/src/normalized_schema_generator/__init__.py

@@ -3,4 +3,4 @@
 from .widget import NormalizedSchemaWidget
 
 __all__ = ["NormalizedSchemaWidget"]
-__version__ = "0.2.0"
+__version__ = "0.3.0"

{normalized_schema_generator-0.2.0 → normalized_schema_generator-0.4.0}/src/normalized_schema_generator/widget.css

@@ -5,8 +5,9 @@
 
 /* Header */
 .nsw-header { margin-bottom: 16px; }
-.nsw-header h2 { font-size: 20px; font-weight: 600; margin: 0 0 2px; color: var(--f-text); }
-.nsw-header p { font-size: 12px; color: var(--f-text2); margin: 0; }
+.nsw-header h2 { font-size: 20px; font-weight: 600; margin: 0; color: var(--f-text); }
+.nsw-header p { font-size: 12px; color: var(--f-text2); margin: 2px 0 0; }
+.nsw-version { font-size: 11px; color: var(--f-text3); font-weight: 400; }
 
 /* Cards */
 .nsw-card { border: 1px solid var(--f-border); border-radius: var(--f-radius); padding: 16px; margin-bottom: 12px; background: var(--f-bg); box-shadow: var(--f-shadow4); }
@@ -96,7 +97,7 @@
 /* JSON output */
 .nsw-json-wrap { position: relative; }
 .nsw-json { font-family: var(--f-mono); font-size: 11.5px; background: var(--f-surface); border: 1px solid var(--f-border); border-radius: var(--f-radius-sm); padding: 12px; max-height: 400px; overflow: auto; white-space: pre; line-height: 1.45; color: var(--f-text); }
-.nsw-copy-btn { position: absolute; top: 8px; right: 8px; }
+.nsw-json-actions { position: absolute; top: 8px; right: 16px; display: flex; gap: 4px; align-items: center; }
 .nsw-copied { font-size: 11px; color: var(--f-success); font-weight: 600; padding: 3px 10px; }
 
 /* Collapse toggle */
@@ -120,6 +121,42 @@
 /* Cases placeholder */
 .nsw-cases-placeholder { display: flex; align-items: center; justify-content: center; padding: 16px; border: 1.5px dashed #E8AB02; border-radius: var(--f-radius); margin-bottom: 12px; }
 
+/* Attribute mapping */
+.nsw-card.attrs { border-left: 3px solid #8764B8; }
+.nsw-attr-list { }
+.nsw-attr-row { display: flex; align-items: center; gap: 8px; padding: 5px 0; border-bottom: 1px solid #f2f2f2; min-height: 34px; }
+.nsw-attr-row:last-child { border-bottom: none; }
+.nsw-attr-row:hover { background: var(--f-surface); }
+.nsw-attr-name { font-family: var(--f-mono); font-size: 12px; flex: 1; min-width: 100px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; color: var(--f-text); }
+
+/* Icon button group */
+.nsw-icon-group { display: inline-flex; gap: 1px; background: var(--f-surface); border-radius: var(--f-radius-sm); padding: 2px; border: 1px solid var(--f-border); }
+.nsw-icon-btn { width: 26px; height: 24px; border: none; background: transparent; cursor: pointer; border-radius: 3px; color: var(--f-text3); display: inline-flex; align-items: center; justify-content: center; padding: 0; transition: all 0.1s; }
+.nsw-icon-btn:hover { color: var(--f-text); background: rgba(0,0,0,.06); }
+.nsw-icon-btn.active { background: var(--f-brand); color: #fff; }
+.nsw-icon-btn svg { width: 14px; height: 14px; fill: none; stroke: currentColor; stroke-width: 1.8; stroke-linecap: round; stroke-linejoin: round; }
+.nsw-icon-btn[title]:hover::after { content: attr(title); position: absolute; bottom: 100%; left: 50%; transform: translateX(-50%); background: var(--f-text); color: #fff; font-size: 10px; padding: 2px 6px; border-radius: 3px; white-space: nowrap; pointer-events: none; z-index: 10; }
+.nsw-icon-btn { position: relative; }
+
+/* Data type pills */
+.nsw-dtype-group { display: inline-flex; gap: 1px; background: var(--f-surface); border-radius: var(--f-radius-sm); padding: 2px; border: 1px solid var(--f-border); }
+.nsw-dtype-btn { height: 22px; padding: 0 6px; border: none; background: transparent; cursor: pointer; border-radius: 3px; font-family: var(--f-font); font-size: 10px; font-weight: 600; color: var(--f-text3); letter-spacing: 0.03em; transition: all 0.1s; }
+.nsw-dtype-btn:hover { color: var(--f-text); background: rgba(0,0,0,.06); }
+.nsw-dtype-btn.active { color: #fff; }
+.nsw-dtype-btn.active.t-str { background: var(--f-brand); }
+.nsw-dtype-btn.active.t-int { background: var(--f-success); }
+.nsw-dtype-btn.active.t-flt { background: var(--f-warn); }
+.nsw-dtype-btn.active.t-date { background: #7B1FA2; }
+.nsw-dtype-btn.active.t-bool { background: var(--f-danger); }
+
+/* First/Last mini toggle */
+.nsw-fl-group { display: inline-flex; gap: 1px; background: var(--f-surface); border-radius: var(--f-radius-sm); padding: 2px; border: 1px solid var(--f-border); }
+.nsw-fl-btn { height: 22px; padding: 0 8px; border: none; background: transparent; cursor: pointer; border-radius: 3px; font-family: var(--f-font); font-size: 10px; font-weight: 500; color: var(--f-text3); transition: all 0.1s; }
+.nsw-fl-btn:hover { color: var(--f-text); background: rgba(0,0,0,.06); }
+.nsw-fl-btn.active { background: var(--f-brand); color: #fff; }
+.nsw-fl-group.disabled { opacity: 0.35; pointer-events: none; }
+.nsw-fl-group.disabled .nsw-fl-btn { cursor: default; }
+
 /* Scrollbar */
-.nsw-col-list::-webkit-scrollbar, .nsw-json::-webkit-scrollbar { width: 6px; }
-.nsw-col-list::-webkit-scrollbar-thumb, .nsw-json::-webkit-scrollbar-thumb { background: var(--f-border-strong); border-radius: 3px; }
+.nsw-col-list::-webkit-scrollbar, .nsw-json::-webkit-scrollbar, .nsw-attr-list::-webkit-scrollbar { width: 6px; }
+.nsw-col-list::-webkit-scrollbar-thumb, .nsw-json::-webkit-scrollbar-thumb, .nsw-attr-list::-webkit-scrollbar-thumb { background: var(--f-border-strong); border-radius: 3px; }

{normalized_schema_generator-0.2.0 → normalized_schema_generator-0.4.0}/src/normalized_schema_generator/widget.js

@@ -63,6 +63,7 @@ export function render({ model, el }) {
     eventsRef: { visible: false, editIndex: -1, targetTable: "", sourceCol: "", targetCol: "", keyType: "Integer", exportName: "", targetCols: [], loading: false },
     casesRef:  { visible: false, editIndex: -1, targetTable: "", sourceCol: "", targetCol: "", keyType: "Integer", exportName: "", targetCols: [], loading: false },
     pendingRefFor: null,
+    attrsOpen: false,
     outputOpen: true,
     copied: false,
   };
@@ -86,13 +87,17 @@ export function render({ model, el }) {
     root.appendChild(buildLakehouseCard());
     root.appendChild(buildEventsCard());
     root.appendChild(buildCasesCard());
+    root.appendChild(buildAttributesCard());
     root.appendChild(buildOutputCard());
   }
 
   /* ── header ──────────────────────────────────────────── */
   function buildHeader() {
     return h("div", { className: "nsw-header" },
-      h("h2", { textContent: "Normalized Schema Generator" }),
+      h("div", { style: "display:flex;align-items:baseline;gap:8px" },
+        h("h2", { textContent: "Normalized Schema Generator" }),
+        h("span", { className: "nsw-version", textContent: "v0.4.0" })
+      ),
       h("p", { textContent: "Configure your process mining star-schema and generate the dataSource JSON." })
     );
   }
@@ -106,6 +111,7 @@ export function render({ model, el }) {
 
     // Lakehouse selector
     const status = model.get("loading_status") || "";
+    const lhLoading = !!(status && (status.includes("tables") || status.includes("Reloading")));
     const lhOpts = lakehouses.map(l => ({ value: l.id, label: l.displayName }));
     const lhRow = h("div", { className: "nsw-row" },
       h("span", { className: "nsw-label", textContent: "Lakehouse" }),
@@ -114,9 +120,9 @@ export function render({ model, el }) {
         model.set("selected_lakehouse_id", val);
         model.set("selected_lakehouse_name", lh ? lh.displayName : "");
         model.save_changes();
-      }, "Select a lakehouse...")
+      }, "Select a lakehouse...", lhLoading)
     );
-    if (status && status.includes("tables")) {
+    if (lhLoading) {
       lhRow.appendChild(h("span", { className: "nsw-loading-inline", textContent: status }));
     }
     card.appendChild(lhRow);
@@ -333,6 +339,8 @@ export function render({ model, el }) {
 
     // Table selector
     const loadingStatus = model.get("loading_status") || "";
+    const isLoadingTables = loadingStatus.includes("tables") || loadingStatus.includes("Reloading");
+    const isLoadingCols = selTable && columns.length === 0 && loadingStatus.includes("columns");
     const tOpts = tables.map(t => ({ value: t.name, label: t.name }));
     const tblRow = h("div", { className: "nsw-row" },
       h("span", { className: "nsw-label", textContent: "Table" }),
@@ -341,9 +349,11 @@ export function render({ model, el }) {
         model.set("events_joins", []);
         model.save_changes();
         resetRefForm(S.eventsRef);
-      }, selLH ? "Select events table..." : "Select a lakehouse first", !selLH)
+      }, isLoadingTables ? "Loading tables..." : (selLH ? "Select events table..." : "Select a lakehouse first"), !selLH || isLoadingTables || isLoadingCols)
     );
-    if (selTable && columns.length === 0 && loadingStatus.includes("columns")) {
+    if (isLoadingTables) {
+      tblRow.appendChild(h("span", { className: "nsw-loading-inline", textContent: "" }));
+    } else if (isLoadingCols) {
       tblRow.appendChild(h("span", { className: "nsw-loading-inline", textContent: "Loading columns..." }));
     }
     card.appendChild(tblRow);
@@ -497,6 +507,8 @@ export function render({ model, el }) {
     const selLH = model.get("selected_lakehouse_id") || "";
 
     const casesLoadingStatus = model.get("loading_status") || "";
+    const casesLoadingTables = casesLoadingStatus.includes("tables") || casesLoadingStatus.includes("Reloading");
+    const casesLoadingCols = selTable && columns.length === 0 && casesLoadingStatus.includes("columns");
     const tOpts = tables.map(t => ({ value: t.name, label: t.name }));
     const casesTblRow = h("div", { className: "nsw-row" },
       h("span", { className: "nsw-label", textContent: "Table" }),
@@ -506,9 +518,11 @@ export function render({ model, el }) {
         model.set("cases_relationship", {});
         model.save_changes();
         resetRefForm(S.casesRef);
-      }, selLH ? "Select cases table..." : "Select a lakehouse first", !selLH)
+      }, casesLoadingTables ? "Loading tables..." : (selLH ? "Select cases table..." : "Select a lakehouse first"), !selLH || casesLoadingTables || casesLoadingCols)
     );
-    if (selTable && columns.length === 0 && casesLoadingStatus.includes("columns")) {
+    if (casesLoadingTables) {
+      casesTblRow.appendChild(h("span", { className: "nsw-loading-inline", textContent: "" }));
+    } else if (casesLoadingCols) {
       casesTblRow.appendChild(h("span", { className: "nsw-loading-inline", textContent: "Loading columns..." }));
     }
     card.appendChild(casesTblRow);
@@ -564,7 +578,125 @@ export function render({ model, el }) {
     return card;
   }
 
-  /* ── Section D: JSON output ──────────────────────────── */
+  /* ── SVG icons for attribute types (16x16 viewBox) ──── */
+  const ICONS = {
+    caseid: '<svg viewBox="0 0 16 16"><path d="M10 2a2 2 0 0 1 2 2c0 .74-.4 1.39-1 1.73V7h1a2 2 0 0 1 2 2v.27c.6.34 1 .99 1 1.73a2 2 0 0 1-4 0c0-.74.4-1.39 1-1.73V9a1 1 0 0 0-1-1H6a1 1 0 0 0-1 1v.27c.6.34 1 .99 1 1.73a2 2 0 0 1-4 0c0-.74.4-1.39 1-1.73V9a2 2 0 0 1 2-2h1V5.73C5.4 5.39 5 4.74 5 4a2 2 0 0 1 2-2"/><circle cx="10" cy="4" r="1"/><circle cx="6" cy="11" r="1"/><circle cx="14" cy="11" r="1"/>',
+    activity: '<svg viewBox="0 0 16 16"><polygon points="5,2 14,8 5,14"/>',
+    start: '<svg viewBox="0 0 16 16"><circle cx="8" cy="8" r="5.5" fill="none"/><path d="M8 4v4l3 1.5"/>',
+    end: '<svg viewBox="0 0 16 16"><circle cx="8" cy="8" r="5.5" fill="none"/><path d="M5.5 8.5l2 2 3.5-4"/>',
+    resource: '<svg viewBox="0 0 16 16"><circle cx="8" cy="5" r="2.5" fill="none"/><path d="M3 14c0-2.8 2.2-5 5-5s5 2.2 5 5"/>',
+    evtattr: '<svg viewBox="0 0 16 16"><path d="M2 4h12M2 8h12M2 12h8"/>',
+    caseattr: '<svg viewBox="0 0 16 16"><path d="M2 3h5l1.5 2H14v8H2z"/>',
+    fin_evt: '<svg viewBox="0 0 16 16"><path d="M2 4h7M2 8h7M2 12h5"/><text x="13" y="10" text-anchor="middle" font-size="7" font-weight="700" fill="currentColor" stroke="none">$</text>',
+    fin_case: '<svg viewBox="0 0 16 16"><path d="M2 3h5l1.5 2H11v5H2z"/><text x="14" y="12" text-anchor="middle" font-size="7" font-weight="700" fill="currentColor" stroke="none">$</text>',
+    fin_res: '<svg viewBox="0 0 16 16"><circle cx="6" cy="5" r="2.5" fill="none"/><path d="M1 14c0-2.8 2.2-5 5-5s5 2.2 5 5"/><text x="14" y="7" text-anchor="middle" font-size="7" font-weight="700" fill="currentColor" stroke="none">$</text>',
+  };
+
+  /* Attribute type definitions: key, icon, tooltip, ImportType, Level, FinanceImportType */
+  const ATTR_TYPES = [
+    { key: "caseid",   icon: ICONS.caseid,   tip: "Case ID",              it: "Case",     lv: "Event", fi: "None" },
+    { key: "activity", icon: ICONS.activity,  tip: "Activity",             it: "Activity",  lv: "Event", fi: "None" },
+    { key: "start",    icon: ICONS.start,     tip: "Start timestamp",      it: "Start",     lv: "Event", fi: "None" },
+    { key: "end",      icon: ICONS.end,       tip: "End timestamp",        it: "End",       lv: "Event", fi: "None" },
+    { key: "resource", icon: ICONS.resource,  tip: "Resource",             it: "Resource",  lv: "Event", fi: "None" },
+    { key: "evtattr",  icon: ICONS.evtattr,   tip: "Event-level attribute",it: "Other",     lv: "Event", fi: "None" },
+    { key: "caseattr", icon: ICONS.caseattr,  tip: "Case-level attribute", it: "Other",     lv: "Case",  fi: "None" },
+    { key: "fin_evt",  icon: ICONS.fin_evt,   tip: "Finance per event",    it: "Other",     lv: "Event", fi: "PerEvent" },
+    { key: "fin_case", icon: ICONS.fin_case,  tip: "Finance per case",     it: "Other",     lv: "Case",  fi: "PerCase" },
+    { key: "fin_res",  icon: ICONS.fin_res,   tip: "Finance per resource", it: "Other",     lv: "Event", fi: "PerResource" },
+  ];
+
+  function attrTypeKey(a) {
+    for (const t of ATTR_TYPES) {
+      if (t.it === a.ImportType && t.lv === a.Level && t.fi === a.FinanceImportType) return t.key;
+    }
+    return "evtattr";
+  }
+
+  const DTYPES = [
+    { key: "String",  label: "STR",  cls: "t-str" },
+    { key: "Integer", label: "INT",  cls: "t-int" },
+    { key: "Float",   label: "FLT",  cls: "t-flt" },
+    { key: "Date",    label: "DATE", cls: "t-date" },
+    { key: "Boolean", label: "BOOL", cls: "t-bool" },
+  ];
+
+  /* ── Section D: Attribute Mapping ────────────────────── */
+  function buildAttributesCard() {
+    const attrs = model.get("attributes_config") || [];
+    if (attrs.length === 0) return h("div", null);
+
+    const card = h("div", { className: "nsw-card attrs" });
+
+    const toggleBtn = h("button", { className: "nsw-collapse-btn", onClick: () => { S.attrsOpen = !S.attrsOpen; rebuild(); } },
+      h("span", { className: "arrow " + (S.attrsOpen ? "open" : ""), textContent: "▶" }),
+      h("span", { textContent: "Attribute Mapping" }),
+      h("span", { style: "font-weight:400;color:var(--f-text3);font-size:12px;margin-left:4px", textContent: "(" + attrs.length + ")" })
+    );
+    card.appendChild(toggleBtn);
+
+    if (!S.attrsOpen) return card;
+
+    function updAttr(i, fields) {
+      const updated = JSON.parse(JSON.stringify(attrs));
+      Object.assign(updated[i], fields);
+      model.set("attributes_config", updated);
+      model.save_changes();
+    }
+
+    const list = h("div", { className: "nsw-attr-list" });
+    attrs.forEach((attr, i) => {
+      const curType = attrTypeKey(attr);
+      const isCaseLevel = attr.Level === "Case";
+
+      // Attribute type icon group
+      const typeGroup = h("div", { className: "nsw-icon-group" });
+      for (const t of ATTR_TYPES) {
+        const btn = h("button", {
+          className: "nsw-icon-btn" + (curType === t.key ? " active" : ""),
+          title: t.tip,
+          innerHTML: t.icon,
+          onClick: () => updAttr(i, { ImportType: t.it, Level: t.lv, FinanceImportType: t.fi })
+        });
+        typeGroup.appendChild(btn);
+      }
+
+      // First/Last toggle (always visible, disabled when not case-level)
+      const curFL = attr.CaseLevelAttributeHandling || "First";
+      const flGroup = h("div", { className: "nsw-fl-group" + (isCaseLevel ? "" : " disabled") });
+      for (const v of ["First", "Last"]) {
+        flGroup.appendChild(h("button", {
+          className: "nsw-fl-btn" + (isCaseLevel && curFL === v ? " active" : ""),
+          textContent: v,
+          onClick: isCaseLevel ? () => updAttr(i, { CaseLevelAttributeHandling: v }) : null
+        }));
+      }
+
+      // Data type pills
+      const dtGroup = h("div", { className: "nsw-dtype-group" });
+      const curDt = attr.SourceDataType || "String";
+      for (const d of DTYPES) {
+        dtGroup.appendChild(h("button", {
+          className: "nsw-dtype-btn " + d.cls + (curDt === d.key ? " active" : ""),
+          textContent: d.label,
+          onClick: () => updAttr(i, { SourceDataType: d.key })
+        }));
+      }
+
+      const row = h("div", { className: "nsw-attr-row" },
+        h("span", { className: "nsw-attr-name", textContent: attr.Name }),
+        typeGroup,
+        flGroup,
+        dtGroup
+      );
+      list.appendChild(row);
+    });
+
+    card.appendChild(list);
+    return card;
+  }
+
+  /* ── Section E: JSON output ──────────────────────────── */
   function buildOutputCard() {
     const card = h("div", { className: "nsw-card output" });
     const json = model.get("config_json") || "";
@@ -581,11 +713,12 @@ export function render({ model, el }) {
       } else {
         const wrap = h("div", { className: "nsw-json-wrap" });
         wrap.appendChild(h("div", { className: "nsw-json", textContent: json }));
+        const btnGroup = h("div", { className: "nsw-json-actions" });
         if (S.copied) {
-          wrap.appendChild(h("span", { className: "nsw-copy-btn nsw-copied", textContent: "Copied!" }));
+          btnGroup.appendChild(h("span", { className: "nsw-copied", textContent: "Copied!" }));
         } else {
-          wrap.appendChild(h("button", {
-            className: "nsw-btn outline nsw-copy-btn",
+          btnGroup.appendChild(h("button", {
+            className: "nsw-btn outline sm",
             textContent: "Copy",
             onClick: () => {
               navigator.clipboard.writeText(json).then(() => {
@@ -596,6 +729,20 @@ export function render({ model, el }) {
             }
           }));
         }
+        btnGroup.appendChild(h("button", {
+          className: "nsw-btn outline sm",
+          textContent: "Save",
+          onClick: () => {
+            const blob = new Blob([json], { type: "application/json" });
+            const url = URL.createObjectURL(blob);
+            const a = document.createElement("a");
+            a.href = url;
+            a.download = "normalized_config.json";
+            a.click();
+            URL.revokeObjectURL(url);
+          }
+        }));
+        wrap.appendChild(btnGroup);
         card.appendChild(wrap);
       }
     }
@@ -612,7 +759,7 @@ export function render({ model, el }) {
     "lakehouses", "selected_lakehouse_id", "data_source_file_type",
     "available_tables", "events_table_name", "events_columns", "events_joins",
     "cases_enabled", "cases_table_name", "cases_columns", "cases_joins", "cases_relationship",
-    "config_json", "loading_status", "error_message"
+    "attributes_config", "config_json", "loading_status", "error_message"
   ];
   for (const t of watchedTraits) {
     model.on("change:" + t, rebuild);

{normalized_schema_generator-0.2.0 → normalized_schema_generator-0.4.0}/src/normalized_schema_generator/widget.py

@@ -112,23 +112,90 @@ def _list_files_recursive(base_path, lakehouse_id, file_type, prefix="", depth=0
     return results
 
 
+def _quote_spark_identifier(identifier):
+    return f"`{identifier.replace('`', '``')}`"
+
+
+def _row_value(row, *names):
+    if hasattr(row, "asDict"):
+        row = row.asDict()
+    if isinstance(row, dict):
+        for name in names:
+            if name in row:
+                return row[name]
+        return None
+    for name in names:
+        if hasattr(row, name):
+            return getattr(row, name)
+    try:
+        return row[0]
+    except (IndexError, KeyError, TypeError):
+        return None
+
+
+def _list_schema_tables(lakehouse_name, lakehouse_id):
+    """Discover tables via Spark for schema-enabled lakehouses."""
+    if _spark is None:
+        return []
+
+    lakehouse = _quote_spark_identifier(lakehouse_name)
+    schemas = _spark.sql(f"SHOW SCHEMAS IN {lakehouse}").collect()
+    results = []
+    for row in schemas:
+        schema = _row_value(row, "namespace", "databaseName", "schemaName")
+        if not schema:
+            continue
+        schema_identifier = _quote_spark_identifier(schema)
+        tables = _spark.sql(
+            f"SHOW TABLES IN {lakehouse}.{schema_identifier}"
+        ).collect()
+        for table_row in tables:
+            table_name = _row_value(table_row, "tableName", "table_name", "name")
+            if not table_name:
+                continue
+            results.append({
+                "name": f"{schema}.{table_name}",
+                "path": f"/{lakehouse_id}/Tables/{schema}/{table_name}",
+                "type": "delta",
+            })
+    return results
+
+
 def _list_tables(lakehouse_name, lakehouse_id, file_type):
     if FABRIC_AVAILABLE:
-        try:
-            if file_type == 2:
+        if file_type == 2:
+            try:
                 tables = _lh_utils.listTables(lakehouse_name)
-                return [
-                    {
-                        "name": t.name,
-                        "path": f"/{lakehouse_id}/Tables/dbo/{t.name}",
+            except Exception:
+                tables = None
+
+            if tables:
+                results = []
+                for table in tables:
+                    name = table.name
+                    schema = (
+                        getattr(table, "schemaName", None)
+                        or getattr(table, "schema", None)
+                    )
+                    if schema:
+                        name = f"{schema}.{name}"
+                    else:
+                        schema = "dbo"
+                    results.append({
+                        "name": name,
+                        "path": f"/{lakehouse_id}/Tables/{schema}/{table.name}",
                         "type": "delta",
-                    }
-                    for t in tables
-                ]
-            else:
-                return _list_files_recursive("Files", lakehouse_id, file_type)
-        except Exception:
-            return []
+                    })
+                return results
+
+            schema_tables = _list_schema_tables(lakehouse_name, lakehouse_id)
+            if schema_tables or tables == []:
+                return schema_tables
+            raise RuntimeError(
+                f"Unable to list tables in lakehouse {lakehouse_name!r}; "
+                "NotebookUtils listing failed and Spark catalog discovery is unavailable."
+            )
+        return _list_files_recursive("Files", lakehouse_id, file_type)
     # Mock data
     if file_type == 2:
         return [
@@ -175,7 +242,11 @@ def _get_columns(table_name, lakehouse_name, lakehouse_id, file_type):
     if FABRIC_AVAILABLE and _spark:
         try:
             if file_type == 2:
-                full_name = f"`{lakehouse_name}`.`{table_name}`"
+                name_parts = table_name.split(".", 1)
+                identifiers = [lakehouse_name, *name_parts]
+                full_name = ".".join(
+                    _quote_spark_identifier(part) for part in identifiers
+                )
                 df = _spark.sql(f"DESCRIBE TABLE {full_name}")
                 rows = df.collect()
                 return [

normalized_schema_generator-0.4.0/tests/test_widget_tables.py

@@ -0,0 +1,92 @@
+import unittest
+from types import SimpleNamespace
+from unittest.mock import patch
+
+from normalized_schema_generator import widget
+
+
+class FakeSpark:
+    def __init__(self):
+        self.queries = []
+
+    def sql(self, query):
+        self.queries.append(query)
+        if query == "SHOW SCHEMAS IN `Sales`":
+            rows = [SimpleNamespace(namespace="dbo"), SimpleNamespace(namespace="marketing")]
+        elif query == "SHOW TABLES IN `Sales`.`dbo`":
+            rows = [SimpleNamespace(tableName="customers")]
+        elif query == "SHOW TABLES IN `Sales`.`marketing`":
+            rows = [SimpleNamespace(tableName="orders")]
+        elif query == "DESCRIBE TABLE `Sales`.`marketing`.`orders`":
+            rows = [{"col_name": "order_id", "data_type": "int"}]
+        else:
+            raise AssertionError(f"Unexpected Spark SQL query: {query}")
+        return SimpleNamespace(collect=lambda: rows)
+
+
+class LakehouseTableTests(unittest.TestCase):
+    def test_schema_catalog_fallback_lists_tables_with_schema_paths(self):
+        spark = FakeSpark()
+        lakehouse_utils = SimpleNamespace(
+            listTables=lambda _name: (_ for _ in ()).throw(
+                RuntimeError("schemas are unsupported")
+            )
+        )
+
+        with patch.object(widget, "FABRIC_AVAILABLE", True):
+            with patch.object(widget, "_lh_utils", lakehouse_utils):
+                with patch.object(widget, "_spark", spark):
+                    tables = widget._list_tables("Sales", "lakehouse-id", 2)
+
+        self.assertEqual(
+            tables,
+            [
+                {
+                    "name": "dbo.customers",
+                    "path": "/lakehouse-id/Tables/dbo/customers",
+                    "type": "delta",
+                },
+                {
+                    "name": "marketing.orders",
+                    "path": "/lakehouse-id/Tables/marketing/orders",
+                    "type": "delta",
+                },
+            ],
+        )
+
+    def test_schema_qualified_table_name_is_used_for_column_lookup(self):
+        spark = FakeSpark()
+
+        with patch.object(widget, "FABRIC_AVAILABLE", True):
+            with patch.object(widget, "_spark", spark):
+                columns = widget._get_columns(
+                    "marketing.orders", "Sales", "lakehouse-id", 2
+                )
+
+        self.assertEqual(columns, [{"name": "order_id", "dtype": "int"}])
+        self.assertIn(
+            "DESCRIBE TABLE `Sales`.`marketing`.`orders`",
+            spark.queries,
+        )
+
+    def test_legacy_listing_keeps_table_names_and_dbo_paths(self):
+        lakehouse_utils = SimpleNamespace(
+            listTables=lambda _name: [SimpleNamespace(name="events")]
+        )
+
+        with patch.object(widget, "FABRIC_AVAILABLE", True):
+            with patch.object(widget, "_lh_utils", lakehouse_utils):
+                tables = widget._list_tables("Legacy", "lakehouse-id", 2)
+
+        self.assertEqual(
+            tables,
+            [{
+                "name": "events",
+                "path": "/lakehouse-id/Tables/dbo/events",
+                "type": "delta",
+            }],
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()