normalized-schema-generator 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- normalized_schema_generator/__init__.py +6 -0
- normalized_schema_generator/widget.css +125 -0
- normalized_schema_generator/widget.js +635 -0
- normalized_schema_generator/widget.py +573 -0
- normalized_schema_generator-0.1.0.dist-info/METADATA +85 -0
- normalized_schema_generator-0.1.0.dist-info/RECORD +8 -0
- normalized_schema_generator-0.1.0.dist-info/WHEEL +4 -0
- normalized_schema_generator-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,573 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Normalized Schema Generator Widget for Microsoft Fabric Notebooks.
|
|
3
|
+
|
|
4
|
+
An anywidget-based interactive widget that lets users visually configure
|
|
5
|
+
a normalized schema JSON config for Process Mining data import. Replaces
|
|
6
|
+
manual JSON authoring by providing dropdowns, toggles, and inline forms.
|
|
7
|
+
|
|
8
|
+
Usage in a Fabric notebook:
|
|
9
|
+
from normalized_schema_generator import NormalizedSchemaWidget
|
|
10
|
+
w = NormalizedSchemaWidget()
|
|
11
|
+
w # display the widget
|
|
12
|
+
|
|
13
|
+
# After configuration, retrieve the generated JSON:
|
|
14
|
+
print(w.config_json)
|
|
15
|
+
|
|
16
|
+
For local development (outside Fabric), mock data is used automatically.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
import json
|
|
20
|
+
import pathlib
|
|
21
|
+
|
|
22
|
+
import anywidget
|
|
23
|
+
import traitlets
|
|
24
|
+
|
|
25
|
+
# ── Fabric availability detection ────────────────────────────────────────
|
|
26
|
+
|
|
27
|
+
FABRIC_AVAILABLE = False
|
|
28
|
+
_spark = None
|
|
29
|
+
|
|
30
|
+
try:
|
|
31
|
+
from notebookutils import lakehouse as _lh_utils
|
|
32
|
+
from notebookutils import runtime as _runtime_utils
|
|
33
|
+
from notebookutils import fs as _fs_utils
|
|
34
|
+
FABRIC_AVAILABLE = True
|
|
35
|
+
except ImportError:
|
|
36
|
+
_lh_utils = None
|
|
37
|
+
_runtime_utils = None
|
|
38
|
+
_fs_utils = None
|
|
39
|
+
|
|
40
|
+
try:
|
|
41
|
+
from pyspark.sql import SparkSession
|
|
42
|
+
_spark = SparkSession.builder.getOrCreate()
|
|
43
|
+
except Exception:
|
|
44
|
+
pass
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
# ── Fabric integration / mock helpers ────────────────────────────────────
|
|
48
|
+
|
|
49
|
+
def _get_fabric_context():
|
|
50
|
+
if FABRIC_AVAILABLE:
|
|
51
|
+
result = {"workspace_id": "", "tenant_id": ""}
|
|
52
|
+
# Workspace ID: from runtime context (Py4J Java Map, bracket access)
|
|
53
|
+
try:
|
|
54
|
+
result["workspace_id"] = str(_runtime_utils.context["currentWorkspaceId"])
|
|
55
|
+
except Exception:
|
|
56
|
+
pass
|
|
57
|
+
# Tenant ID: not in runtime context — use Spark trident config
|
|
58
|
+
if _spark:
|
|
59
|
+
try:
|
|
60
|
+
result["tenant_id"] = _spark.conf.get("trident.tenant.id")
|
|
61
|
+
except Exception:
|
|
62
|
+
pass
|
|
63
|
+
return result
|
|
64
|
+
return {
|
|
65
|
+
"workspace_id": "eb5b82ad-3bfe-4976-9830-107e982eb72f",
|
|
66
|
+
"tenant_id": "a1165230-a86d-4887-9000-276fea697c53",
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _list_lakehouses():
|
|
71
|
+
if FABRIC_AVAILABLE:
|
|
72
|
+
try:
|
|
73
|
+
items = _lh_utils.list()
|
|
74
|
+
return [
|
|
75
|
+
{"id": str(item.id), "displayName": item.displayName}
|
|
76
|
+
for item in items
|
|
77
|
+
]
|
|
78
|
+
except Exception:
|
|
79
|
+
return []
|
|
80
|
+
return [
|
|
81
|
+
{"id": "2de91b1d-28a0-4bbb-839c-3ba65414497a", "displayName": "ProcessMining_LH"},
|
|
82
|
+
{"id": "f8c3a1b2-1234-5678-9abc-def012345678", "displayName": "Sales_LH"},
|
|
83
|
+
{"id": "a0b1c2d3-aaaa-bbbb-cccc-ddddeeeeeeee", "displayName": "HR_LH"},
|
|
84
|
+
]
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _list_files_recursive(base_path, lakehouse_id, file_type, prefix="", depth=0, max_depth=3):
|
|
88
|
+
"""Recursively list data files under the Files section."""
|
|
89
|
+
results = []
|
|
90
|
+
if depth > max_depth:
|
|
91
|
+
return results
|
|
92
|
+
exts = (".csv",) if file_type == 0 else (".parquet", ".snappy.parquet")
|
|
93
|
+
try:
|
|
94
|
+
items = _fs_utils.ls(base_path)
|
|
95
|
+
except Exception:
|
|
96
|
+
return results
|
|
97
|
+
for item in items:
|
|
98
|
+
name = item.name.rstrip("/")
|
|
99
|
+
if name.startswith("_") or name.startswith("."):
|
|
100
|
+
continue
|
|
101
|
+
rel = f"{prefix}/{name}" if prefix else name
|
|
102
|
+
if item.isDir:
|
|
103
|
+
results.extend(
|
|
104
|
+
_list_files_recursive(item.path, lakehouse_id, file_type, rel, depth + 1, max_depth)
|
|
105
|
+
)
|
|
106
|
+
elif any(name.lower().endswith(ext) for ext in exts):
|
|
107
|
+
results.append({
|
|
108
|
+
"name": rel,
|
|
109
|
+
"path": f"/{lakehouse_id}/Files/{rel}",
|
|
110
|
+
"type": "file",
|
|
111
|
+
})
|
|
112
|
+
return results
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _list_tables(lakehouse_name, lakehouse_id, file_type):
|
|
116
|
+
if FABRIC_AVAILABLE:
|
|
117
|
+
try:
|
|
118
|
+
if file_type == 2:
|
|
119
|
+
tables = _lh_utils.listTables(lakehouse_name)
|
|
120
|
+
return [
|
|
121
|
+
{
|
|
122
|
+
"name": t.name,
|
|
123
|
+
"path": f"/{lakehouse_id}/Tables/dbo/{t.name}",
|
|
124
|
+
"type": "delta",
|
|
125
|
+
}
|
|
126
|
+
for t in tables
|
|
127
|
+
]
|
|
128
|
+
else:
|
|
129
|
+
return _list_files_recursive("Files", lakehouse_id, file_type)
|
|
130
|
+
except Exception:
|
|
131
|
+
return []
|
|
132
|
+
# Mock data
|
|
133
|
+
if file_type == 2:
|
|
134
|
+
return [
|
|
135
|
+
{"name": "events", "path": f"/{lakehouse_id}/Tables/dbo/events", "type": "delta"},
|
|
136
|
+
{"name": "cases", "path": f"/{lakehouse_id}/Tables/dbo/cases", "type": "delta"},
|
|
137
|
+
{"name": "activity", "path": f"/{lakehouse_id}/Tables/dbo/activity", "type": "delta"},
|
|
138
|
+
{"name": "resource", "path": f"/{lakehouse_id}/Tables/dbo/resource", "type": "delta"},
|
|
139
|
+
{"name": "vendors", "path": f"/{lakehouse_id}/Tables/dbo/vendors", "type": "delta"},
|
|
140
|
+
{"name": "departments", "path": f"/{lakehouse_id}/Tables/dbo/departments", "type": "delta"},
|
|
141
|
+
]
|
|
142
|
+
else:
|
|
143
|
+
return [
|
|
144
|
+
{"name": "events", "path": f"/{lakehouse_id}/Files/events", "type": "folder"},
|
|
145
|
+
{"name": "cases", "path": f"/{lakehouse_id}/Files/cases", "type": "folder"},
|
|
146
|
+
{"name": "activity", "path": f"/{lakehouse_id}/Files/activity", "type": "folder"},
|
|
147
|
+
{"name": "resource", "path": f"/{lakehouse_id}/Files/resource", "type": "folder"},
|
|
148
|
+
]
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _spark_type_to_simple(dtype_str):
|
|
152
|
+
dtype_str = dtype_str.lower().strip()
|
|
153
|
+
# Integer family
|
|
154
|
+
if dtype_str in ("int", "integer", "bigint", "long", "short", "smallint", "tinyint", "byte"):
|
|
155
|
+
return "int"
|
|
156
|
+
# Float family
|
|
157
|
+
if dtype_str in ("float", "double", "decimal", "numeric", "real", "number") or dtype_str.startswith("decimal"):
|
|
158
|
+
return "double"
|
|
159
|
+
# String family
|
|
160
|
+
if dtype_str in ("string", "varchar", "char", "nvarchar", "text", "nchar") or dtype_str.startswith("varchar") or dtype_str.startswith("char") or dtype_str.startswith("nvarchar"):
|
|
161
|
+
return "string"
|
|
162
|
+
# Boolean
|
|
163
|
+
if dtype_str in ("boolean", "bool", "bit"):
|
|
164
|
+
return "boolean"
|
|
165
|
+
# Date/time family
|
|
166
|
+
if dtype_str in ("timestamp", "timestamp_ntz", "timestamp_ltz", "date", "datetime", "datetime2", "datetimeoffset", "time"):
|
|
167
|
+
return "timestamp"
|
|
168
|
+
# Binary / other — map to string for display
|
|
169
|
+
if dtype_str in ("binary", "varbinary", "array", "map", "struct", "void", "null"):
|
|
170
|
+
return "string"
|
|
171
|
+
return dtype_str
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _get_columns(table_name, lakehouse_name, lakehouse_id, file_type):
|
|
175
|
+
if FABRIC_AVAILABLE and _spark:
|
|
176
|
+
try:
|
|
177
|
+
if file_type == 2:
|
|
178
|
+
full_name = f"`{lakehouse_name}`.`{table_name}`"
|
|
179
|
+
df = _spark.sql(f"DESCRIBE TABLE {full_name}")
|
|
180
|
+
rows = df.collect()
|
|
181
|
+
return [
|
|
182
|
+
{"name": r["col_name"], "dtype": _spark_type_to_simple(r["data_type"])}
|
|
183
|
+
for r in rows
|
|
184
|
+
if r["col_name"] and not r["col_name"].startswith("#")
|
|
185
|
+
]
|
|
186
|
+
else:
|
|
187
|
+
base = f"Files/{table_name}"
|
|
188
|
+
if file_type == 0:
|
|
189
|
+
df = _spark.read.option("header", "true").option("inferSchema", "true").csv(base)
|
|
190
|
+
else:
|
|
191
|
+
df = _spark.read.parquet(base)
|
|
192
|
+
return [
|
|
193
|
+
{"name": f.name, "dtype": _spark_type_to_simple(f.dataType.simpleString())}
|
|
194
|
+
for f in df.schema.fields
|
|
195
|
+
]
|
|
196
|
+
except Exception:
|
|
197
|
+
return []
|
|
198
|
+
# Mock data
|
|
199
|
+
_mock = {
|
|
200
|
+
"events": [
|
|
201
|
+
{"name": "CaseID", "dtype": "int"},
|
|
202
|
+
{"name": "Activity_id", "dtype": "int"},
|
|
203
|
+
{"name": "Resource_id", "dtype": "int"},
|
|
204
|
+
{"name": "StartTimestamp", "dtype": "timestamp"},
|
|
205
|
+
{"name": "EndTimestamp", "dtype": "timestamp"},
|
|
206
|
+
{"name": "EventCost", "dtype": "double"},
|
|
207
|
+
],
|
|
208
|
+
"cases": [
|
|
209
|
+
{"name": "CaseID", "dtype": "int"},
|
|
210
|
+
{"name": "CustomerName", "dtype": "string"},
|
|
211
|
+
{"name": "CustomerSegment", "dtype": "string"},
|
|
212
|
+
{"name": "InvoiceTotalAmountWithoutVAT", "dtype": "double"},
|
|
213
|
+
{"name": "Region", "dtype": "string"},
|
|
214
|
+
],
|
|
215
|
+
"activity": [
|
|
216
|
+
{"name": "Activity_id", "dtype": "int"},
|
|
217
|
+
{"name": "Activity", "dtype": "string"},
|
|
218
|
+
],
|
|
219
|
+
"resource": [
|
|
220
|
+
{"name": "Resource_id", "dtype": "int"},
|
|
221
|
+
{"name": "Resource", "dtype": "string"},
|
|
222
|
+
],
|
|
223
|
+
"vendors": [
|
|
224
|
+
{"name": "Vendor_id", "dtype": "int"},
|
|
225
|
+
{"name": "VendorName", "dtype": "string"},
|
|
226
|
+
{"name": "VendorCountry", "dtype": "string"},
|
|
227
|
+
],
|
|
228
|
+
"departments": [
|
|
229
|
+
{"name": "Dept_id", "dtype": "int"},
|
|
230
|
+
{"name": "DeptName", "dtype": "string"},
|
|
231
|
+
],
|
|
232
|
+
}
|
|
233
|
+
return _mock.get(table_name, [
|
|
234
|
+
{"name": "id", "dtype": "int"},
|
|
235
|
+
{"name": "value", "dtype": "string"},
|
|
236
|
+
])
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
# ── Widget class ─────────────────────────────────────────────────────────
|
|
240
|
+
|
|
241
|
+
_HERE = pathlib.Path(__file__).parent
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
class NormalizedSchemaWidget(anywidget.AnyWidget):
|
|
245
|
+
_esm = _HERE / "widget.js"
|
|
246
|
+
_css = _HERE / "widget.css"
|
|
247
|
+
|
|
248
|
+
# Lakehouse & source config
|
|
249
|
+
lakehouses = traitlets.List([]).tag(sync=True)
|
|
250
|
+
selected_lakehouse_id = traitlets.Unicode("").tag(sync=True)
|
|
251
|
+
selected_lakehouse_name = traitlets.Unicode("").tag(sync=True)
|
|
252
|
+
data_source_file_type = traitlets.Int(2).tag(sync=True)
|
|
253
|
+
available_tables = traitlets.List([]).tag(sync=True)
|
|
254
|
+
|
|
255
|
+
# Events table
|
|
256
|
+
events_table_name = traitlets.Unicode("").tag(sync=True)
|
|
257
|
+
events_columns = traitlets.List([]).tag(sync=True)
|
|
258
|
+
events_joins = traitlets.List([]).tag(sync=True)
|
|
259
|
+
|
|
260
|
+
# Cases table
|
|
261
|
+
cases_enabled = traitlets.Bool(False).tag(sync=True)
|
|
262
|
+
cases_table_name = traitlets.Unicode("").tag(sync=True)
|
|
263
|
+
cases_columns = traitlets.List([]).tag(sync=True)
|
|
264
|
+
cases_joins = traitlets.List([]).tag(sync=True)
|
|
265
|
+
# Events→Cases FK relationship: {fk_column, pk_column, join_key_type}
|
|
266
|
+
cases_relationship = traitlets.Dict({}).tag(sync=True)
|
|
267
|
+
|
|
268
|
+
# Communication & output
|
|
269
|
+
fetch_columns_request = traitlets.Unicode("").tag(sync=True)
|
|
270
|
+
fetch_columns_response = traitlets.List([]).tag(sync=True)
|
|
271
|
+
config_json = traitlets.Unicode("").tag(sync=True)
|
|
272
|
+
loading_status = traitlets.Unicode("").tag(sync=True)
|
|
273
|
+
error_message = traitlets.Unicode("").tag(sync=True)
|
|
274
|
+
|
|
275
|
+
# Fabric context
|
|
276
|
+
workspace_id = traitlets.Unicode("").tag(sync=True)
|
|
277
|
+
tenant_id = traitlets.Unicode("").tag(sync=True)
|
|
278
|
+
|
|
279
|
+
def __init__(self, **kwargs):
|
|
280
|
+
super().__init__(**kwargs)
|
|
281
|
+
ctx = _get_fabric_context()
|
|
282
|
+
self.workspace_id = ctx["workspace_id"]
|
|
283
|
+
self.tenant_id = ctx["tenant_id"]
|
|
284
|
+
self.lakehouses = _list_lakehouses()
|
|
285
|
+
|
|
286
|
+
# ── Observers ────────────────────────────────────────
|
|
287
|
+
|
|
288
|
+
@traitlets.observe("selected_lakehouse_id")
|
|
289
|
+
def _on_lakehouse_change(self, change):
|
|
290
|
+
lh_id = change["new"]
|
|
291
|
+
if not lh_id:
|
|
292
|
+
self.available_tables = []
|
|
293
|
+
return
|
|
294
|
+
self.loading_status = "Loading tables..."
|
|
295
|
+
self.events_table_name = ""
|
|
296
|
+
self.events_columns = []
|
|
297
|
+
self.events_joins = []
|
|
298
|
+
self.cases_table_name = ""
|
|
299
|
+
self.cases_columns = []
|
|
300
|
+
self.cases_joins = []
|
|
301
|
+
self.cases_relationship = {}
|
|
302
|
+
try:
|
|
303
|
+
self.available_tables = _list_tables(
|
|
304
|
+
self.selected_lakehouse_name, lh_id, self.data_source_file_type
|
|
305
|
+
)
|
|
306
|
+
except Exception as e:
|
|
307
|
+
self.error_message = f"Failed to load tables: {e}"
|
|
308
|
+
self.available_tables = []
|
|
309
|
+
self.loading_status = ""
|
|
310
|
+
self._generate_config()
|
|
311
|
+
|
|
312
|
+
@traitlets.observe("data_source_file_type")
|
|
313
|
+
def _on_file_type_change(self, change):
|
|
314
|
+
if not self.selected_lakehouse_id:
|
|
315
|
+
return
|
|
316
|
+
self.loading_status = "Reloading tables..."
|
|
317
|
+
self.events_table_name = ""
|
|
318
|
+
self.events_columns = []
|
|
319
|
+
self.events_joins = []
|
|
320
|
+
self.cases_table_name = ""
|
|
321
|
+
self.cases_columns = []
|
|
322
|
+
self.cases_joins = []
|
|
323
|
+
self.cases_relationship = {}
|
|
324
|
+
try:
|
|
325
|
+
self.available_tables = _list_tables(
|
|
326
|
+
self.selected_lakehouse_name,
|
|
327
|
+
self.selected_lakehouse_id,
|
|
328
|
+
change["new"],
|
|
329
|
+
)
|
|
330
|
+
except Exception as e:
|
|
331
|
+
self.error_message = f"Failed to load tables: {e}"
|
|
332
|
+
self.available_tables = []
|
|
333
|
+
self.loading_status = ""
|
|
334
|
+
self._generate_config()
|
|
335
|
+
|
|
336
|
+
@traitlets.observe("events_table_name")
|
|
337
|
+
def _on_events_table_change(self, change):
|
|
338
|
+
name = change["new"]
|
|
339
|
+
if not name:
|
|
340
|
+
self.events_columns = []
|
|
341
|
+
self._generate_config()
|
|
342
|
+
return
|
|
343
|
+
self.loading_status = "Loading columns..."
|
|
344
|
+
try:
|
|
345
|
+
raw = _get_columns(
|
|
346
|
+
name, self.selected_lakehouse_name,
|
|
347
|
+
self.selected_lakehouse_id, self.data_source_file_type,
|
|
348
|
+
)
|
|
349
|
+
self.events_columns = [
|
|
350
|
+
{"name": c["name"], "dtype": c["dtype"], "included": True}
|
|
351
|
+
for c in raw
|
|
352
|
+
]
|
|
353
|
+
except Exception as e:
|
|
354
|
+
self.error_message = f"Failed to load columns: {e}"
|
|
355
|
+
self.events_columns = []
|
|
356
|
+
self.loading_status = ""
|
|
357
|
+
self._generate_config()
|
|
358
|
+
|
|
359
|
+
@traitlets.observe("cases_table_name")
|
|
360
|
+
def _on_cases_table_change(self, change):
|
|
361
|
+
name = change["new"]
|
|
362
|
+
if not name:
|
|
363
|
+
self.cases_columns = []
|
|
364
|
+
self.cases_relationship = {}
|
|
365
|
+
self._generate_config()
|
|
366
|
+
return
|
|
367
|
+
self.cases_relationship = {}
|
|
368
|
+
self.loading_status = "Loading columns..."
|
|
369
|
+
try:
|
|
370
|
+
raw = _get_columns(
|
|
371
|
+
name, self.selected_lakehouse_name,
|
|
372
|
+
self.selected_lakehouse_id, self.data_source_file_type,
|
|
373
|
+
)
|
|
374
|
+
self.cases_columns = [
|
|
375
|
+
{"name": c["name"], "dtype": c["dtype"], "included": True}
|
|
376
|
+
for c in raw
|
|
377
|
+
]
|
|
378
|
+
except Exception as e:
|
|
379
|
+
self.error_message = f"Failed to load columns: {e}"
|
|
380
|
+
self.cases_columns = []
|
|
381
|
+
self.loading_status = ""
|
|
382
|
+
self._generate_config()
|
|
383
|
+
|
|
384
|
+
@traitlets.observe("fetch_columns_request")
|
|
385
|
+
def _on_fetch_columns_request(self, change):
|
|
386
|
+
req = change["new"]
|
|
387
|
+
if not req:
|
|
388
|
+
return
|
|
389
|
+
# Format: "timestamp:tablename"
|
|
390
|
+
parts = req.split(":", 1)
|
|
391
|
+
if len(parts) < 2:
|
|
392
|
+
return
|
|
393
|
+
table_name = parts[1]
|
|
394
|
+
try:
|
|
395
|
+
raw = _get_columns(
|
|
396
|
+
table_name, self.selected_lakehouse_name,
|
|
397
|
+
self.selected_lakehouse_id, self.data_source_file_type,
|
|
398
|
+
)
|
|
399
|
+
self.fetch_columns_response = [
|
|
400
|
+
{"name": c["name"], "dtype": c["dtype"]}
|
|
401
|
+
for c in raw
|
|
402
|
+
]
|
|
403
|
+
except Exception as e:
|
|
404
|
+
self.error_message = f"Failed to load columns for {table_name}: {e}"
|
|
405
|
+
self.fetch_columns_response = []
|
|
406
|
+
|
|
407
|
+
@traitlets.observe("events_columns", "events_joins", "cases_enabled",
|
|
408
|
+
"cases_columns", "cases_joins", "cases_relationship")
|
|
409
|
+
def _on_data_change(self, change):
|
|
410
|
+
self._generate_config()
|
|
411
|
+
|
|
412
|
+
# ── JSON generation ──────────────────────────────────
|
|
413
|
+
|
|
414
|
+
def _find_table_path(self, table_name):
|
|
415
|
+
for t in self.available_tables:
|
|
416
|
+
if t["name"] == table_name:
|
|
417
|
+
return t["path"]
|
|
418
|
+
return ""
|
|
419
|
+
|
|
420
|
+
def _generate_config(self):
|
|
421
|
+
self.error_message = ""
|
|
422
|
+
if not self.selected_lakehouse_id or not self.events_table_name:
|
|
423
|
+
self.config_json = ""
|
|
424
|
+
return
|
|
425
|
+
|
|
426
|
+
try:
|
|
427
|
+
datasets = []
|
|
428
|
+
join_dataset_names = set()
|
|
429
|
+
|
|
430
|
+
# ── Events→Cases relationship join ──
|
|
431
|
+
cases_rel = self.cases_relationship
|
|
432
|
+
has_cases_rel = (
|
|
433
|
+
self.cases_enabled
|
|
434
|
+
and self.cases_table_name
|
|
435
|
+
and cases_rel.get("fk_column")
|
|
436
|
+
and cases_rel.get("pk_column")
|
|
437
|
+
)
|
|
438
|
+
|
|
439
|
+
# ── Events dataset (Kind=0) ──
|
|
440
|
+
events_fk_cols = {j["source_column"] for j in self.events_joins}
|
|
441
|
+
if has_cases_rel:
|
|
442
|
+
events_fk_cols.add(cases_rel["fk_column"])
|
|
443
|
+
events_columns = [
|
|
444
|
+
{"Name": c["name"]}
|
|
445
|
+
for c in self.events_columns
|
|
446
|
+
if c.get("included") and c["name"] not in events_fk_cols
|
|
447
|
+
]
|
|
448
|
+
events_join_arr = []
|
|
449
|
+
# Add the Events→Cases join first
|
|
450
|
+
if has_cases_rel:
|
|
451
|
+
events_join_arr.append({
|
|
452
|
+
"SourceColumnName": cases_rel["fk_column"],
|
|
453
|
+
"TargetColumnName": cases_rel["pk_column"],
|
|
454
|
+
"TargetDatasetName": self.cases_table_name,
|
|
455
|
+
"JoinKeyType": cases_rel.get("join_key_type", "Integer"),
|
|
456
|
+
})
|
|
457
|
+
for j in self.events_joins:
|
|
458
|
+
entry = {
|
|
459
|
+
"SourceColumnName": j["source_column"],
|
|
460
|
+
"TargetColumnName": j["target_column"],
|
|
461
|
+
"TargetDatasetName": j["target_table"],
|
|
462
|
+
"JoinKeyType": j["join_key_type"],
|
|
463
|
+
}
|
|
464
|
+
if j.get("export_name"):
|
|
465
|
+
entry["ExportName"] = j["export_name"]
|
|
466
|
+
events_join_arr.append(entry)
|
|
467
|
+
join_dataset_names.add(j["target_table"])
|
|
468
|
+
|
|
469
|
+
datasets.append({
|
|
470
|
+
"Kind": 0,
|
|
471
|
+
"Name": self.events_table_name,
|
|
472
|
+
"Path": self._find_table_path(self.events_table_name),
|
|
473
|
+
"Columns": events_columns,
|
|
474
|
+
"Join": events_join_arr if events_join_arr else None,
|
|
475
|
+
})
|
|
476
|
+
|
|
477
|
+
# ── Cases dataset (Kind=1) ──
|
|
478
|
+
if self.cases_enabled and self.cases_table_name:
|
|
479
|
+
cases_fk_cols = {j["source_column"] for j in self.cases_joins}
|
|
480
|
+
cases_columns = [
|
|
481
|
+
{"Name": c["name"]}
|
|
482
|
+
for c in self.cases_columns
|
|
483
|
+
if c.get("included") and c["name"] not in cases_fk_cols
|
|
484
|
+
]
|
|
485
|
+
cases_join_arr = []
|
|
486
|
+
for j in self.cases_joins:
|
|
487
|
+
entry = {
|
|
488
|
+
"SourceColumnName": j["source_column"],
|
|
489
|
+
"TargetColumnName": j["target_column"],
|
|
490
|
+
"TargetDatasetName": j["target_table"],
|
|
491
|
+
"JoinKeyType": j["join_key_type"],
|
|
492
|
+
}
|
|
493
|
+
if j.get("export_name"):
|
|
494
|
+
entry["ExportName"] = j["export_name"]
|
|
495
|
+
cases_join_arr.append(entry)
|
|
496
|
+
join_dataset_names.add(j["target_table"])
|
|
497
|
+
|
|
498
|
+
datasets.append({
|
|
499
|
+
"Kind": 1,
|
|
500
|
+
"Name": self.cases_table_name,
|
|
501
|
+
"Path": self._find_table_path(self.cases_table_name),
|
|
502
|
+
"Columns": cases_columns,
|
|
503
|
+
"Join": cases_join_arr if cases_join_arr else None,
|
|
504
|
+
})
|
|
505
|
+
|
|
506
|
+
# ── Join datasets (Kind=2) ──
|
|
507
|
+
join_table_cols = {}
|
|
508
|
+
for j in list(self.events_joins) + list(self.cases_joins):
|
|
509
|
+
tname = j["target_table"]
|
|
510
|
+
if tname not in join_table_cols:
|
|
511
|
+
join_table_cols[tname] = {}
|
|
512
|
+
for c in j.get("target_columns", []):
|
|
513
|
+
if c["name"] not in join_table_cols[tname]:
|
|
514
|
+
join_table_cols[tname][c["name"]] = c
|
|
515
|
+
elif c.get("included"):
|
|
516
|
+
join_table_cols[tname][c["name"]]["included"] = True
|
|
517
|
+
|
|
518
|
+
for tname in sorted(join_dataset_names):
|
|
519
|
+
cols_map = join_table_cols.get(tname, {})
|
|
520
|
+
join_cols = [
|
|
521
|
+
{"Name": c["name"]}
|
|
522
|
+
for c in cols_map.values()
|
|
523
|
+
if c.get("included")
|
|
524
|
+
]
|
|
525
|
+
datasets.append({
|
|
526
|
+
"Kind": 2,
|
|
527
|
+
"Name": tname,
|
|
528
|
+
"Path": self._find_table_path(tname),
|
|
529
|
+
"Columns": join_cols,
|
|
530
|
+
"Join": None,
|
|
531
|
+
})
|
|
532
|
+
|
|
533
|
+
# ── Assemble full config ──
|
|
534
|
+
config = {
|
|
535
|
+
"inputDataBinding": {
|
|
536
|
+
"productFlavor": "processadvisor",
|
|
537
|
+
"dataSource": {
|
|
538
|
+
"dataSourceSchemaType": 1,
|
|
539
|
+
"dataSourceType": 2,
|
|
540
|
+
"dataSourceFileType": self.data_source_file_type,
|
|
541
|
+
"dataSourceId": "oneLakeDataSourceIdPlaceholder",
|
|
542
|
+
"path": "",
|
|
543
|
+
"oneLakeConnectionSetupProperties": {
|
|
544
|
+
"tenantId": self.tenant_id,
|
|
545
|
+
"workspaceId": self.workspace_id,
|
|
546
|
+
"lakehouseId": self.selected_lakehouse_id,
|
|
547
|
+
},
|
|
548
|
+
"datasets": datasets,
|
|
549
|
+
},
|
|
550
|
+
"miningMetadata": {
|
|
551
|
+
"ImportConfiguration": {"Attributes": []},
|
|
552
|
+
"Views": [],
|
|
553
|
+
"ProcessExtendedMetadata": {
|
|
554
|
+
"CalculatedMetrics": [],
|
|
555
|
+
"BusinessRules": [],
|
|
556
|
+
"ProcessMapHierarchies": [],
|
|
557
|
+
"CaseCategorization": {
|
|
558
|
+
"CaseStates": [],
|
|
559
|
+
"CaseAnnotations": [],
|
|
560
|
+
},
|
|
561
|
+
},
|
|
562
|
+
"ReportData": {},
|
|
563
|
+
"Calendars": [],
|
|
564
|
+
},
|
|
565
|
+
},
|
|
566
|
+
"reportConfiguration": {"isEmbeddedReport": True},
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
self.config_json = json.dumps(config, indent=2)
|
|
570
|
+
|
|
571
|
+
except Exception as e:
|
|
572
|
+
self.error_message = f"Config generation error: {e}"
|
|
573
|
+
self.config_json = ""
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: normalized-schema-generator
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: An anywidget for Microsoft Fabric notebooks that visually configures normalized schema JSON for Process Mining data import.
|
|
5
|
+
Project-URL: Homepage, https://github.com/rosikm/normalized-schema-generator
|
|
6
|
+
Project-URL: Repository, https://github.com/rosikm/normalized-schema-generator
|
|
7
|
+
Project-URL: Issues, https://github.com/rosikm/normalized-schema-generator/issues
|
|
8
|
+
Author: rosikm
|
|
9
|
+
License: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: anywidget,fabric,jupyter,process-mining,widget
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Framework :: Jupyter
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Topic :: Scientific/Engineering
|
|
17
|
+
Requires-Python: >=3.9
|
|
18
|
+
Requires-Dist: anywidget>=0.9.0
|
|
19
|
+
Requires-Dist: traitlets>=5.0
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
|
|
22
|
+
# Normalized Schema Generator
|
|
23
|
+
|
|
24
|
+
An [anywidget](https://anywidget.dev)-based Jupyter widget for **Microsoft Fabric** notebooks that visually configures a normalized schema JSON config for **Process Mining** data import.
|
|
25
|
+
|
|
26
|
+
Replaces manual JSON authoring by providing dropdowns, toggles, and inline forms to select lakehouses, tables, columns, and FK/PK relationships, then auto-generates the `dataSource` section of the config.
|
|
27
|
+
|
|
28
|
+
## Install
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
pip install normalized-schema-generator
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## Usage
|
|
35
|
+
|
|
36
|
+
### In a Fabric notebook
|
|
37
|
+
|
|
38
|
+
```python
|
|
39
|
+
from normalized_schema_generator import NormalizedSchemaWidget
|
|
40
|
+
|
|
41
|
+
w = NormalizedSchemaWidget()
|
|
42
|
+
w
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
The widget auto-detects the Fabric environment and populates lakehouses, tables, and columns using `notebookutils` and Spark.
|
|
46
|
+
|
|
47
|
+
After configuring your schema, retrieve the generated JSON:
|
|
48
|
+
|
|
49
|
+
```python
|
|
50
|
+
print(w.config_json)
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
### Local development
|
|
54
|
+
|
|
55
|
+
Outside Fabric, the widget uses mock data automatically — no `notebookutils` or Spark needed:
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
pip install normalized-schema-generator jupyterlab
|
|
59
|
+
jupyter lab
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
## Features
|
|
63
|
+
|
|
64
|
+
- **Lakehouse selection** from the current workspace
|
|
65
|
+
- **Data source toggle**: Delta Tables or Files (CSV / Parquet)
|
|
66
|
+
- **Events table** (required): column checklist with data type badges, include/exclude toggles
|
|
67
|
+
- **Cases table** (optional): same column management, with required Events-to-Cases FK relationship
|
|
68
|
+
- **Reference management**: add/edit/remove FK joins to lookup tables, with target column selection
|
|
69
|
+
- **Auto-generated JSON**: full `inputDataBinding` config with `dataSource`, connection properties, and dataset definitions
|
|
70
|
+
- **Copy to clipboard** for the generated JSON
|
|
71
|
+
- **Fluent 2 styling** matching the Fabric design language
|
|
72
|
+
|
|
73
|
+
## JSON output
|
|
74
|
+
|
|
75
|
+
The widget generates a complete normalized import config following the [Process Mining Normalized Import specification](https://learn.microsoft.com/en-us/power-automate/process-mining-overview):
|
|
76
|
+
|
|
77
|
+
- `dataSourceSchemaType: 1` (Normalized)
|
|
78
|
+
- `dataSourceType: 2` (OneLake)
|
|
79
|
+
- Proper `datasets[]` with Kind 0 (Event), 1 (Case), 2 (Join)
|
|
80
|
+
- FK columns excluded from `Columns[]` (exposed via `Join[].ExportName`)
|
|
81
|
+
- No nested joins on lookup datasets
|
|
82
|
+
|
|
83
|
+
## License
|
|
84
|
+
|
|
85
|
+
MIT
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
normalized_schema_generator/__init__.py,sha256=2oC4ifjiN4IXJnHeR2AD9gUzAIgQHVUlItC0XxcjrR0,189
|
|
2
|
+
normalized_schema_generator/widget.css,sha256=Uf4e7dH11KGy8kfLwU8c4m5ASr38znmLHZtuoYaar9A,10637
|
|
3
|
+
normalized_schema_generator/widget.js,sha256=tl6nMValu2iwh45Qu27ldwiguxp3TlAQ-GXIYRAcz-c,26631
|
|
4
|
+
normalized_schema_generator/widget.py,sha256=opEp1W0CPJ3FxO-jAPNVMtbFK5t-g0bpqVPrX8iIEXs,22340
|
|
5
|
+
normalized_schema_generator-0.1.0.dist-info/METADATA,sha256=JhRZpf3m5I6_GXfGom9XO4O9uGF8bn9HSLqaPyk_fO4,3080
|
|
6
|
+
normalized_schema_generator-0.1.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
|
|
7
|
+
normalized_schema_generator-0.1.0.dist-info/licenses/LICENSE,sha256=vRiDhEUJV1wye5jgS_LlZIbK2oFjQFpXF2pUuN-TIp4,1063
|
|
8
|
+
normalized_schema_generator-0.1.0.dist-info/RECORD,,
|