modulearn 0.1.1__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- modulearn-0.2.0/.github/workflows/ci.yml +29 -0
- modulearn-0.2.0/CHANGELOG.md +6 -0
- {modulearn-0.1.1 → modulearn-0.2.0}/PKG-INFO +13 -1
- {modulearn-0.1.1 → modulearn-0.2.0}/README.md +12 -0
- modulearn-0.2.0/docs/roadmap.html +368 -0
- {modulearn-0.1.1 → modulearn-0.2.0}/examples/quickstart.py +10 -4
- {modulearn-0.1.1 → modulearn-0.2.0}/modulearn/__init__.py +1 -1
- {modulearn-0.1.1 → modulearn-0.2.0}/modulearn/compiler.py +84 -4
- {modulearn-0.1.1 → modulearn-0.2.0}/modulearn/demo.py +11 -4
- {modulearn-0.1.1 → modulearn-0.2.0}/modulearn/server.py +18 -2
- {modulearn-0.1.1 → modulearn-0.2.0}/modulearn/static/graph.html +5 -0
- {modulearn-0.1.1 → modulearn-0.2.0}/modulearn/static/graph.js +5 -2
- {modulearn-0.1.1 → modulearn-0.2.0}/pyproject.toml +1 -1
- modulearn-0.2.0/tests/test_robustness.py +138 -0
- modulearn-0.2.0/tests/test_run_errors.py +80 -0
- {modulearn-0.1.1 → modulearn-0.2.0}/.gitignore +0 -0
- {modulearn-0.1.1 → modulearn-0.2.0}/LICENSE +0 -0
- {modulearn-0.1.1 → modulearn-0.2.0}/PUBLISHING.md +0 -0
- {modulearn-0.1.1 → modulearn-0.2.0}/docs/implementation-guide.html +0 -0
- {modulearn-0.1.1 → modulearn-0.2.0}/modulearn/__main__.py +0 -0
- {modulearn-0.1.1 → modulearn-0.2.0}/modulearn/cli.py +0 -0
- {modulearn-0.1.1 → modulearn-0.2.0}/modulearn/registry.py +0 -0
- {modulearn-0.1.1 → modulearn-0.2.0}/modulearn/static/graph-engine.js +0 -0
- {modulearn-0.1.1 → modulearn-0.2.0}/tests/test_smoke.py +0 -0
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: [main]
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
test:
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
strategy:
|
|
13
|
+
fail-fast: false
|
|
14
|
+
matrix:
|
|
15
|
+
python-version: ["3.10", "3.11", "3.12", "3.13"]
|
|
16
|
+
|
|
17
|
+
steps:
|
|
18
|
+
- uses: actions/checkout@v4
|
|
19
|
+
|
|
20
|
+
- name: Set up Python ${{ matrix.python-version }}
|
|
21
|
+
uses: actions/setup-python@v5
|
|
22
|
+
with:
|
|
23
|
+
python-version: ${{ matrix.python-version }}
|
|
24
|
+
|
|
25
|
+
- name: Install package with dev extras
|
|
26
|
+
run: pip install -e ".[dev]"
|
|
27
|
+
|
|
28
|
+
- name: Run tests
|
|
29
|
+
run: pytest -q
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
## 0.2.0 — 2026-07-23
|
|
2
|
+
- Surface on_train crashes on the canvas (error + full traceback)
|
|
3
|
+
- Loss node now shapes the training curve
|
|
4
|
+
- CompiledGraph.check_shape() validates loaded array shape
|
|
5
|
+
- Enforce hyperparameter min/max at compile
|
|
6
|
+
- GitHub Actions CI across Python 3.10–3.13
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: modulearn
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: A visual, Unreal-Blueprints-style node-graph editor for building and launching ML training runs.
|
|
5
5
|
Project-URL: Homepage, https://github.com/IsaiahKoamalu/modulearn
|
|
6
6
|
Project-URL: Source, https://github.com/IsaiahKoamalu/modulearn
|
|
@@ -81,6 +81,7 @@ reg.add_loss([Param("loss", "kind", "enum", "mse", choices=["mse", "cross_entrop
|
|
|
81
81
|
|
|
82
82
|
def on_train(compiled, reporter):
|
|
83
83
|
# compiled.dataset, .model, .model_params, .hyperparameters, .loss_params, .transforms
|
|
84
|
+
# compiled.check_shape(X, y) # optional: fail early if a loaded array's shape drifts
|
|
84
85
|
reporter.state(epochs=100)
|
|
85
86
|
for e in range(100):
|
|
86
87
|
reporter.metric(epoch=e, train=..., val=...) # drives the live chart
|
|
@@ -119,6 +120,11 @@ app = create_app(reg, on_train, title="My Project")
|
|
|
119
120
|
- **Composable transforms.** Chain `Dataset → Transform → … → Model`; the compiler
|
|
120
121
|
walks the chain into `compiled.transforms`. Mark a transform `live=False` to show
|
|
121
122
|
it in the palette but have the compiler refuse it until your backend is ready.
|
|
123
|
+
- **Guarded at the boundary.** Hyperparameters are range-checked at compile against
|
|
124
|
+
the `min`/`max` you declared, so `on_train` never sees a value the UI called
|
|
125
|
+
impossible (`epochs=0`, a negative `lr`). And `compiled.check_shape(X, y)` fails
|
|
126
|
+
early — naming the dataset — when a loaded array doesn't match its declared
|
|
127
|
+
features/targets, instead of dying cryptically deep inside your model.
|
|
122
128
|
- **Stateless live window.** Training runs in a background thread and persists to
|
|
123
129
|
`runs/<id>/`. Closing the browser never stops a run; reopening shows true state.
|
|
124
130
|
|
|
@@ -133,6 +139,12 @@ Your `on_train` reports through `reporter`, and the editor reads these keys:
|
|
|
133
139
|
|
|
134
140
|
Anything else you write is stored and returned by the API, just not charted.
|
|
135
141
|
|
|
142
|
+
If your `on_train` raises, ModuLearn catches it and records `phase="error"` with a
|
|
143
|
+
one-line `error` summary (led by the exception type) and the full `traceback` — both
|
|
144
|
+
render right in the editor panel, so a crashing loader or training loop is visible on
|
|
145
|
+
the canvas instead of hanging or failing silently. The traceback also prints to the
|
|
146
|
+
server log.
|
|
147
|
+
|
|
136
148
|
## Editor niceties
|
|
137
149
|
|
|
138
150
|
- Searchable palette with collapsible category dropdowns; `⌘K` / `/` to focus,
|
|
@@ -56,6 +56,7 @@ reg.add_loss([Param("loss", "kind", "enum", "mse", choices=["mse", "cross_entrop
|
|
|
56
56
|
|
|
57
57
|
def on_train(compiled, reporter):
|
|
58
58
|
# compiled.dataset, .model, .model_params, .hyperparameters, .loss_params, .transforms
|
|
59
|
+
# compiled.check_shape(X, y) # optional: fail early if a loaded array's shape drifts
|
|
59
60
|
reporter.state(epochs=100)
|
|
60
61
|
for e in range(100):
|
|
61
62
|
reporter.metric(epoch=e, train=..., val=...) # drives the live chart
|
|
@@ -94,6 +95,11 @@ app = create_app(reg, on_train, title="My Project")
|
|
|
94
95
|
- **Composable transforms.** Chain `Dataset → Transform → … → Model`; the compiler
|
|
95
96
|
walks the chain into `compiled.transforms`. Mark a transform `live=False` to show
|
|
96
97
|
it in the palette but have the compiler refuse it until your backend is ready.
|
|
98
|
+
- **Guarded at the boundary.** Hyperparameters are range-checked at compile against
|
|
99
|
+
the `min`/`max` you declared, so `on_train` never sees a value the UI called
|
|
100
|
+
impossible (`epochs=0`, a negative `lr`). And `compiled.check_shape(X, y)` fails
|
|
101
|
+
early — naming the dataset — when a loaded array doesn't match its declared
|
|
102
|
+
features/targets, instead of dying cryptically deep inside your model.
|
|
97
103
|
- **Stateless live window.** Training runs in a background thread and persists to
|
|
98
104
|
`runs/<id>/`. Closing the browser never stops a run; reopening shows true state.
|
|
99
105
|
|
|
@@ -108,6 +114,12 @@ Your `on_train` reports through `reporter`, and the editor reads these keys:
|
|
|
108
114
|
|
|
109
115
|
Anything else you write is stored and returned by the API, just not charted.
|
|
110
116
|
|
|
117
|
+
If your `on_train` raises, ModuLearn catches it and records `phase="error"` with a
|
|
118
|
+
one-line `error` summary (led by the exception type) and the full `traceback` — both
|
|
119
|
+
render right in the editor panel, so a crashing loader or training loop is visible on
|
|
120
|
+
the canvas instead of hanging or failing silently. The traceback also prints to the
|
|
121
|
+
server log.
|
|
122
|
+
|
|
111
123
|
## Editor niceties
|
|
112
124
|
|
|
113
125
|
- Searchable palette with collapsible category dropdowns; `⌘K` / `/` to focus,
|
|
@@ -0,0 +1,368 @@
|
|
|
1
|
+
<!doctype html>
|
|
2
|
+
<html lang="en">
|
|
3
|
+
<head>
|
|
4
|
+
<meta charset="utf-8">
|
|
5
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
6
|
+
<title>ModuLearn — Improvement Roadmap</title>
|
|
7
|
+
<style>
|
|
8
|
+
:root{
|
|
9
|
+
--bg:#14181f; --panel:#1b212b; --panel2:#20262f; --edge:#2a323f;
|
|
10
|
+
--ink:#e6edf3; --muted:#8b98a9; --faint:#5f6b7a; --accent:#a8542f;
|
|
11
|
+
/* the app's port-type palette — reused here so each work-area IS a port type */
|
|
12
|
+
--robust:#c8663c; --testing:#6fb98f; --features:#d8a24a; --dx:#5aa9e6; --hygiene:#c98bdb;
|
|
13
|
+
--mono:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;
|
|
14
|
+
--sans:ui-sans-serif,system-ui,-apple-system,sans-serif;
|
|
15
|
+
}
|
|
16
|
+
*{box-sizing:border-box}
|
|
17
|
+
html,body{margin:0}
|
|
18
|
+
body{
|
|
19
|
+
background:var(--bg); color:var(--ink); font:14px/1.55 var(--sans);
|
|
20
|
+
/* the blueprint grid — same 50px lattice graph-engine.js paints on the canvas */
|
|
21
|
+
background-image:
|
|
22
|
+
linear-gradient(rgba(255,255,255,.022) 1px,transparent 1px),
|
|
23
|
+
linear-gradient(90deg,rgba(255,255,255,.022) 1px,transparent 1px);
|
|
24
|
+
background-size:50px 50px;
|
|
25
|
+
-webkit-font-smoothing:antialiased;
|
|
26
|
+
}
|
|
27
|
+
a{color:var(--dx)}
|
|
28
|
+
|
|
29
|
+
.wrap{max-width:1120px; margin:0 auto; padding:0 20px}
|
|
30
|
+
|
|
31
|
+
/* ---- header: the thesis ---- */
|
|
32
|
+
header{padding:64px 0 30px}
|
|
33
|
+
.kicker{font:600 11px/1 var(--mono); letter-spacing:.28em; text-transform:uppercase;
|
|
34
|
+
color:var(--muted); display:flex; align-items:center; gap:10px}
|
|
35
|
+
.kicker .dot{width:8px; height:8px; border-radius:50%; background:var(--accent);
|
|
36
|
+
box-shadow:0 0 0 4px rgba(168,84,47,.18)}
|
|
37
|
+
h1{font:600 clamp(34px,6vw,60px)/1.02 var(--sans); letter-spacing:-.02em; margin:18px 0 0}
|
|
38
|
+
h1 .soft{color:var(--muted)}
|
|
39
|
+
.lede{margin:18px 0 0; max-width:65ch; color:#c3ccd8; font-size:16px}
|
|
40
|
+
.lede b{color:var(--ink); font-weight:600}
|
|
41
|
+
|
|
42
|
+
/* ---- progress: the Train-node bar motif ---- */
|
|
43
|
+
.progress{margin:34px 0 6px; display:flex; align-items:center; gap:16px; flex-wrap:wrap}
|
|
44
|
+
.track{flex:1; min-width:220px; height:10px; border-radius:6px; background:#0f131a;
|
|
45
|
+
border:1px solid var(--edge); overflow:hidden; position:relative}
|
|
46
|
+
.fill{height:100%; width:0; background:linear-gradient(90deg,var(--accent),#c8663c);
|
|
47
|
+
border-radius:6px; transition:width .5s cubic-bezier(.2,.7,.2,1)}
|
|
48
|
+
.count{font:600 13px/1 var(--mono); color:var(--muted); white-space:nowrap}
|
|
49
|
+
.count b{color:var(--ink); font-size:16px}
|
|
50
|
+
|
|
51
|
+
/* ---- filter rail ---- */
|
|
52
|
+
.rail{position:sticky; top:0; z-index:5; padding:14px 0; margin-top:22px;
|
|
53
|
+
background:linear-gradient(var(--bg) 74%,transparent);
|
|
54
|
+
display:flex; gap:8px; flex-wrap:wrap; align-items:center}
|
|
55
|
+
.rail .lbl{font:600 10px/1 var(--mono); letter-spacing:.18em; text-transform:uppercase;
|
|
56
|
+
color:var(--faint); margin-right:4px}
|
|
57
|
+
.chip{font:600 12px/1 var(--sans); color:var(--muted); background:var(--panel);
|
|
58
|
+
border:1px solid var(--edge); border-radius:20px; padding:7px 12px; cursor:pointer;
|
|
59
|
+
display:inline-flex; align-items:center; gap:7px; user-select:none;
|
|
60
|
+
transition:border-color .12s,color .12s,background .12s}
|
|
61
|
+
.chip:hover{border-color:var(--muted)}
|
|
62
|
+
.chip .pd{width:8px; height:8px; border-radius:50%; background:var(--c,var(--muted))}
|
|
63
|
+
.chip[aria-pressed="true"]{color:var(--ink); background:#243040; border-color:var(--muted)}
|
|
64
|
+
.chip.off{opacity:.4}
|
|
65
|
+
|
|
66
|
+
/* ---- area sections ---- */
|
|
67
|
+
section{margin:40px 0 0}
|
|
68
|
+
.area{display:flex; align-items:baseline; gap:12px; padding-bottom:12px;
|
|
69
|
+
border-bottom:1px solid var(--edge)}
|
|
70
|
+
.area h2{font:600 14px/1 var(--mono); letter-spacing:.06em; text-transform:uppercase;
|
|
71
|
+
margin:0; display:flex; align-items:center; gap:10px}
|
|
72
|
+
.area h2::before{content:""; width:11px; height:11px; border-radius:50%;
|
|
73
|
+
background:var(--c); box-shadow:0 0 10px -1px var(--c)}
|
|
74
|
+
.area .tally{font:600 12px/1 var(--mono); color:var(--faint); margin-left:auto}
|
|
75
|
+
.area .blurb{color:var(--muted); font-size:13px}
|
|
76
|
+
|
|
77
|
+
/* ---- node cards ---- */
|
|
78
|
+
.grid{display:grid; grid-template-columns:repeat(auto-fill,minmax(320px,1fr));
|
|
79
|
+
gap:14px; margin-top:16px}
|
|
80
|
+
.node{background:var(--panel); border:1px solid var(--edge); border-radius:9px;
|
|
81
|
+
overflow:hidden; transition:border-color .14s,transform .14s,opacity .2s;
|
|
82
|
+
display:flex; flex-direction:column}
|
|
83
|
+
.node:hover{border-color:#3d4756; transform:translateY(-2px)}
|
|
84
|
+
.node .bar{display:flex; align-items:center; gap:10px; padding:11px 13px;
|
|
85
|
+
background:var(--panel2); border-bottom:1px solid var(--edge)}
|
|
86
|
+
.node .port{width:11px; height:11px; border-radius:50%; background:var(--c);
|
|
87
|
+
flex:none; box-shadow:0 0 0 3px color-mix(in srgb,var(--c) 22%,transparent)}
|
|
88
|
+
.node .ttl{font:600 12.5px/1.25 var(--mono); letter-spacing:.01em; color:var(--ink)}
|
|
89
|
+
/* the done toggle — a check node you wire "shipped" ------------------------ */
|
|
90
|
+
.chk{margin-left:auto; width:22px; height:22px; border-radius:6px; flex:none;
|
|
91
|
+
border:1.5px solid var(--edge); background:#0f131a; cursor:pointer; position:relative;
|
|
92
|
+
transition:border-color .12s,background .12s}
|
|
93
|
+
.chk:hover{border-color:var(--testing)}
|
|
94
|
+
.chk::after{content:"✓"; position:absolute; inset:0; display:grid; place-items:center;
|
|
95
|
+
font-size:13px; color:#0f131a; opacity:0; transform:scale(.5); transition:.14s}
|
|
96
|
+
.node.done .chk{background:var(--testing); border-color:var(--testing)}
|
|
97
|
+
.node.done .chk::after{opacity:1; transform:none}
|
|
98
|
+
|
|
99
|
+
.node .body{padding:12px 13px; display:flex; flex-direction:column; gap:10px; flex:1}
|
|
100
|
+
.tags{display:flex; gap:7px; flex-wrap:wrap}
|
|
101
|
+
.tag{font:700 10px/1 var(--mono); letter-spacing:.06em; text-transform:uppercase;
|
|
102
|
+
padding:4px 7px; border-radius:5px; border:1px solid transparent}
|
|
103
|
+
.p1{color:#ffb3ae; background:rgba(248,81,73,.12); border-color:rgba(248,81,73,.3)}
|
|
104
|
+
.p2{color:#f0c674; background:rgba(216,162,74,.12); border-color:rgba(216,162,74,.3)}
|
|
105
|
+
.p3{color:var(--muted); background:rgba(139,152,169,.1); border-color:rgba(139,152,169,.25)}
|
|
106
|
+
.eff{color:var(--muted); background:#0f131a; border-color:var(--edge)}
|
|
107
|
+
.desc{color:#c3ccd8; font-size:13px; margin:0}
|
|
108
|
+
|
|
109
|
+
.more{margin-top:auto}
|
|
110
|
+
.more summary{list-style:none; cursor:pointer; font:600 11px/1 var(--mono);
|
|
111
|
+
letter-spacing:.04em; color:var(--muted); display:inline-flex; align-items:center; gap:6px;
|
|
112
|
+
padding:5px 0; user-select:none}
|
|
113
|
+
.more summary::-webkit-details-marker{display:none}
|
|
114
|
+
.more summary::before{content:"▸"; color:var(--c); transition:transform .15s}
|
|
115
|
+
.more[open] summary::before{transform:rotate(90deg)}
|
|
116
|
+
.why{margin:8px 0 2px; padding:11px 12px; border-left:2px solid var(--c);
|
|
117
|
+
background:#0f131a; border-radius:0 6px 6px 0; font-size:12.5px; color:#b7c1cd}
|
|
118
|
+
.why b{color:var(--ink)}
|
|
119
|
+
|
|
120
|
+
.node.done{opacity:.5}
|
|
121
|
+
.node.done .ttl{text-decoration:line-through; text-decoration-color:var(--faint)}
|
|
122
|
+
.node.hide{display:none}
|
|
123
|
+
|
|
124
|
+
footer{margin:64px 0; color:var(--faint); font:12px/1.6 var(--mono);
|
|
125
|
+
border-top:1px solid var(--edge); padding-top:20px}
|
|
126
|
+
.empty{color:var(--muted); padding:30px 0; font-size:14px; display:none}
|
|
127
|
+
|
|
128
|
+
@media (prefers-reduced-motion:reduce){*{transition:none!important}}
|
|
129
|
+
</style>
|
|
130
|
+
</head>
|
|
131
|
+
<body>
|
|
132
|
+
<div class="wrap">
|
|
133
|
+
<header>
|
|
134
|
+
<div class="kicker"><span class="dot"></span>ModuLearn · working doc</div>
|
|
135
|
+
<h1>Improvement<br><span class="soft">roadmap.</span></h1>
|
|
136
|
+
<p class="lede">Everything worth doing to take ModuLearn from <b>works</b> to <b>polished</b> —
|
|
137
|
+
laid out on the same blueprint canvas it edits. Each card is a node; the colored
|
|
138
|
+
port marks its work area. <b>Check one off</b> and the bar fills — your progress lives
|
|
139
|
+
in this browser.</p>
|
|
140
|
+
|
|
141
|
+
<div class="progress">
|
|
142
|
+
<div class="track"><div class="fill" id="fill"></div></div>
|
|
143
|
+
<div class="count"><b id="doneN">0</b> / <span id="totalN">0</span> shipped</div>
|
|
144
|
+
</div>
|
|
145
|
+
</header>
|
|
146
|
+
|
|
147
|
+
<div class="rail" id="rail">
|
|
148
|
+
<span class="lbl">Filter</span>
|
|
149
|
+
<!-- chips injected -->
|
|
150
|
+
</div>
|
|
151
|
+
|
|
152
|
+
<div id="board"></div>
|
|
153
|
+
<div class="empty" id="empty">Nothing matches this filter.</div>
|
|
154
|
+
|
|
155
|
+
<footer>
|
|
156
|
+
ModuLearn roadmap · lives at <code>docs/roadmap.html</code> · priorities and effort are
|
|
157
|
+
estimates, not commitments · check state is local to this browser.
|
|
158
|
+
</footer>
|
|
159
|
+
</div>
|
|
160
|
+
|
|
161
|
+
<script>
|
|
162
|
+
// ---- work areas = port types (colors match the app's typed ports) -----------
|
|
163
|
+
const AREAS = {
|
|
164
|
+
robust: {name:"Robustness", c:"var(--robust)", blurb:"The parts a real user hits first."},
|
|
165
|
+
testing: {name:"Testing & CI", c:"var(--testing)", blurb:"Confidence that a change didn't break the last one."},
|
|
166
|
+
features:{name:"Features", c:"var(--features)",blurb:"Grow what the editor can actually do."},
|
|
167
|
+
dx: {name:"Developer UX", c:"var(--dx)", blurb:"Make building an app on it pleasant."},
|
|
168
|
+
hygiene: {name:"Project hygiene", c:"var(--hygiene)", blurb:"The unglamorous bits a published package owes its users."},
|
|
169
|
+
};
|
|
170
|
+
|
|
171
|
+
// ---- the roadmap itself ------------------------------------------------------
|
|
172
|
+
const ITEMS = [
|
|
173
|
+
// Robustness
|
|
174
|
+
{area:"robust", prio:1, eff:"M", title:"Surface on_train crashes on the canvas",
|
|
175
|
+
desc:"When a user's loader or training loop throws, the validation panel should show a clear error — not hang or fail silently.",
|
|
176
|
+
why:"<b>The #1 first-run failure.</b> A missing Parquet path or a shape mismatch is the most likely thing to go wrong, and right now the failure path is unverified. Catch the exception in the run thread, write it to the run's state as phase=error with the message, and render it in the panel."},
|
|
177
|
+
{area:"robust", prio:1, eff:"M", title:"Make the loss node actually do something",
|
|
178
|
+
desc:"The loss node is declared and wired, but on_train ignores compiled.loss_params.",
|
|
179
|
+
why:"<b>Declared-but-unused wiring is a lie the UI tells.</b> Either consume loss_params in the demo/example trainers, or mark it clearly as advisory. A wire that changes nothing erodes trust in every other wire."},
|
|
180
|
+
{area:"robust", prio:2, eff:"M", title:"Validate feature/target shape at compile time",
|
|
181
|
+
desc:"The Registry's features/targets lists are declarative only — nothing checks they match what the loader returns.",
|
|
182
|
+
why:"A mismatch currently fails deep inside sklearn with a cryptic message. Compare declared feature count against the loaded array once and fail early with a message that names the dataset."},
|
|
183
|
+
{area:"robust", prio:2, eff:"S", title:"Enforce hyperparameter bounds at the boundary",
|
|
184
|
+
desc:"Confirm min/max on hyperparameters are enforced before values reach on_train (e.g. epochs=0, negative lr).",
|
|
185
|
+
why:"The canvas advertises min/max; the compiler should guarantee them so on_train never receives a value it declared impossible."},
|
|
186
|
+
|
|
187
|
+
// Testing & CI
|
|
188
|
+
{area:"testing", prio:1, eff:"S", title:"GitHub Actions CI",
|
|
189
|
+
desc:"Run the test suite on every push and pull request.",
|
|
190
|
+
why:"<b>Nothing runs the tests before they ship to PyPI.</b> A tiny workflow (checkout → install → pytest across 3.10–3.13) is the highest-leverage safety net here."},
|
|
191
|
+
{area:"testing", prio:1, eff:"M", title:"Cover the compiler's type rules",
|
|
192
|
+
desc:"Tests for the typed-port matching: valid wires compile, mismatched families/subtypes raise, all errors collected together.",
|
|
193
|
+
why:"Type-checked wiring is ModuLearn's core promise. It has almost no tests. This is where a regression would be most silently damaging."},
|
|
194
|
+
{area:"testing", prio:2, eff:"S", title:"serialize / rebuild round-trip test",
|
|
195
|
+
desc:"Assert a graph survives serialize() → rebuild() → serialize() unchanged.",
|
|
196
|
+
why:"Export/import, undo/redo, and load-run all lean on this being lossless. One test locks it down."},
|
|
197
|
+
{area:"testing", prio:2, eff:"M", title:"Server endpoint tests",
|
|
198
|
+
desc:"Exercise /api/nodes, /api/graph/compile, /api/graph/start against a toy registry with FastAPI's TestClient.",
|
|
199
|
+
why:"The HTTP contract between canvas and backend is untested. TestClient makes this fast and dependency-free."},
|
|
200
|
+
{area:"testing", prio:2, eff:"L", title:"Automate the canvas-engine checks",
|
|
201
|
+
desc:"Fold the CDP smoke checks (add/connect/serialize/drag/zoom) into a repeatable headless test.",
|
|
202
|
+
why:"graph-engine.js has only ever been verified by hand. Ambient, but it's the part with no net at all."},
|
|
203
|
+
|
|
204
|
+
// Features
|
|
205
|
+
{area:"features", prio:1, eff:"L", title:"Regression support",
|
|
206
|
+
desc:"An MLPRegressor path with a numeric metric (MAE/RMSE) alongside the current classifier flow.",
|
|
207
|
+
why:"<b>You already downloaded a house-prices dataset.</b> The stack is classification-only today (log_loss + accuracy). This is the most motivated next feature you have."},
|
|
208
|
+
{area:"features", prio:2, eff:"M", title:"Run comparison overlay",
|
|
209
|
+
desc:"Plot several past runs' curves on one chart to compare hyperparameters.",
|
|
210
|
+
why:"Sketched in the implementation guide. The payoff of a visual tool is seeing runs side by side, not one at a time."},
|
|
211
|
+
{area:"features", prio:2, eff:"M", title:"Stop / re-run a training job",
|
|
212
|
+
desc:"A control to cancel an in-flight run and to re-launch an edited graph.",
|
|
213
|
+
why:"Right now a run only ends on its own. Interrupting a bad run is table-stakes for iterating."},
|
|
214
|
+
{area:"features", prio:3, eff:"L", title:"Hyperparameter sweep",
|
|
215
|
+
desc:"Fan a graph out over a range of one hyperparameter and launch the batch.",
|
|
216
|
+
why:"Natural once run-comparison exists — the two together make the tool genuinely useful for tuning."},
|
|
217
|
+
|
|
218
|
+
// Developer UX
|
|
219
|
+
{area:"dx", prio:1, eff:"M", title:"modulearn init scaffold",
|
|
220
|
+
desc:"Generate a starter project (registry + loader + on_train + app) so nobody hand-assembles one.",
|
|
221
|
+
why:"You built ~/modTest by hand. A one-command scaffold is the difference between a library and a framework people reach for."},
|
|
222
|
+
{area:"dx", prio:2, eff:"S", title:"--reload for live development",
|
|
223
|
+
desc:"Pass through uvicorn's reload so editing an app file restarts the server.",
|
|
224
|
+
why:"Iterating on a registry currently means kill-and-restart. Small change, constant daily payoff."},
|
|
225
|
+
{area:"dx", prio:2, eff:"S", title:"Sharper `modulearn run` errors",
|
|
226
|
+
desc:"When a target file has no app, or import fails, point at the fix precisely.",
|
|
227
|
+
why:"The errors are okay but generic. The moment a newcomer's file doesn't load is exactly when the message has to be good."},
|
|
228
|
+
{area:"dx", prio:3, eff:"M", title:"Trim Registry boilerplate",
|
|
229
|
+
desc:"Explore a lighter API — decorators (@reg.dataset) or a config object — over repeated add_* calls.",
|
|
230
|
+
why:"Speculative, low priority: only worth it once real usage shows which parts of the pattern actually chafe."},
|
|
231
|
+
|
|
232
|
+
// Hygiene
|
|
233
|
+
{area:"hygiene", prio:1, eff:"S", title:"Lead the README with pip install modulearn",
|
|
234
|
+
desc:"The install block still shows the from-source `pip install -e .` path.",
|
|
235
|
+
why:"It's on PyPI now. The first thing a reader copies should be the one-liner that works for them, not the contributor path."},
|
|
236
|
+
{area:"hygiene", prio:2, eff:"S", title:"Single-source the version",
|
|
237
|
+
desc:"Version lives in both pyproject.toml and __init__.py — derive one from the other.",
|
|
238
|
+
why:"Two sources of truth drift. A release where they disagree is confusing and avoidable."},
|
|
239
|
+
{area:"hygiene", prio:2, eff:"S", title:"CHANGELOG + tags that match PyPI",
|
|
240
|
+
desc:"Keep a changelog and git-tag each release so GitHub and PyPI stay in lockstep.",
|
|
241
|
+
why:"0.1.1 shipped to PyPI; the repo should always be able to answer 'what changed and when.'"},
|
|
242
|
+
{area:"hygiene", prio:3, eff:"S", title:"CI status badge",
|
|
243
|
+
desc:"Surface the CI result in the README once the workflow exists.",
|
|
244
|
+
why:"Cheap signal of health for anyone evaluating the package. Depends on CI landing first."},
|
|
245
|
+
];
|
|
246
|
+
|
|
247
|
+
// ---- state ------------------------------------------------------------------
|
|
248
|
+
const KEY = "modulearn.roadmap.done";
|
|
249
|
+
const done = new Set(JSON.parse(localStorage.getItem(KEY) || "[]"));
|
|
250
|
+
const id = it => it.area + ":" + it.title;
|
|
251
|
+
const save = () => localStorage.setItem(KEY, JSON.stringify([...done]));
|
|
252
|
+
|
|
253
|
+
const areaOn = new Set(Object.keys(AREAS)); // active area filters
|
|
254
|
+
let prioOn = new Set([1, 2, 3]); // active priority filters
|
|
255
|
+
|
|
256
|
+
// ---- render -----------------------------------------------------------------
|
|
257
|
+
const board = document.getElementById("board");
|
|
258
|
+
const PRIO = {1:"P1 · now", 2:"P2 · next", 3:"P3 · later"};
|
|
259
|
+
|
|
260
|
+
function cardHTML(it){
|
|
261
|
+
return `<article class="node" style="--c:${AREAS[it.area].c}" data-id="${id(it)}"
|
|
262
|
+
data-area="${it.area}" data-prio="${it.prio}">
|
|
263
|
+
<div class="bar">
|
|
264
|
+
<span class="port"></span>
|
|
265
|
+
<span class="ttl">${it.title}</span>
|
|
266
|
+
<button class="chk" title="Mark shipped" aria-label="Mark shipped"></button>
|
|
267
|
+
</div>
|
|
268
|
+
<div class="body">
|
|
269
|
+
<div class="tags">
|
|
270
|
+
<span class="tag p${it.prio}">${PRIO[it.prio]}</span>
|
|
271
|
+
<span class="tag eff">${it.eff} effort</span>
|
|
272
|
+
</div>
|
|
273
|
+
<p class="desc">${it.desc}</p>
|
|
274
|
+
<details class="more">
|
|
275
|
+
<summary>why it matters</summary>
|
|
276
|
+
<div class="why">${it.why}</div>
|
|
277
|
+
</details>
|
|
278
|
+
</div>
|
|
279
|
+
</article>`;
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
function render(){
|
|
283
|
+
board.innerHTML = "";
|
|
284
|
+
for (const key of Object.keys(AREAS)){
|
|
285
|
+
const items = ITEMS.filter(i => i.area === key);
|
|
286
|
+
const sec = document.createElement("section");
|
|
287
|
+
sec.dataset.area = key;
|
|
288
|
+
const shipped = items.filter(i => done.has(id(i))).length;
|
|
289
|
+
sec.innerHTML = `
|
|
290
|
+
<div class="area" style="--c:${AREAS[key].c}">
|
|
291
|
+
<h2>${AREAS[key].name}</h2>
|
|
292
|
+
<span class="blurb">${AREAS[key].blurb}</span>
|
|
293
|
+
<span class="tally">${shipped}/${items.length}</span>
|
|
294
|
+
</div>
|
|
295
|
+
<div class="grid">${items.map(cardHTML).join("")}</div>`;
|
|
296
|
+
board.appendChild(sec);
|
|
297
|
+
}
|
|
298
|
+
document.querySelectorAll(".node").forEach(n => {
|
|
299
|
+
if (done.has(n.dataset.id)) n.classList.add("done");
|
|
300
|
+
n.querySelector(".chk").addEventListener("click", () => toggle(n));
|
|
301
|
+
});
|
|
302
|
+
applyFilter();
|
|
303
|
+
updateProgress();
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
function toggle(node){
|
|
307
|
+
const key = node.dataset.id;
|
|
308
|
+
if (done.has(key)) done.delete(key); else done.add(key);
|
|
309
|
+
node.classList.toggle("done", done.has(key));
|
|
310
|
+
save();
|
|
311
|
+
// refresh the section tally without a full re-render
|
|
312
|
+
const sec = node.closest("section");
|
|
313
|
+
const items = ITEMS.filter(i => i.area === sec.dataset.area);
|
|
314
|
+
sec.querySelector(".tally").textContent =
|
|
315
|
+
`${items.filter(i => done.has(id(i))).length}/${items.length}`;
|
|
316
|
+
updateProgress();
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
function updateProgress(){
|
|
320
|
+
const n = done.size, total = ITEMS.length;
|
|
321
|
+
document.getElementById("fill").style.width = (100 * n / total) + "%";
|
|
322
|
+
document.getElementById("doneN").textContent = n;
|
|
323
|
+
document.getElementById("totalN").textContent = total;
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
// ---- filters ----------------------------------------------------------------
|
|
327
|
+
function buildRail(){
|
|
328
|
+
const rail = document.getElementById("rail");
|
|
329
|
+
for (const [key, a] of Object.entries(AREAS)){
|
|
330
|
+
const c = document.createElement("button");
|
|
331
|
+
c.className = "chip"; c.dataset.area = key; c.setAttribute("aria-pressed","true");
|
|
332
|
+
c.innerHTML = `<span class="pd" style="--c:${a.c}"></span>${a.name}`;
|
|
333
|
+
c.onclick = () => { c.setAttribute("aria-pressed", areaOn.has(key) ? "false":"true");
|
|
334
|
+
c.classList.toggle("off", areaOn.has(key));
|
|
335
|
+
areaOn.has(key) ? areaOn.delete(key) : areaOn.add(key); applyFilter(); };
|
|
336
|
+
rail.appendChild(c);
|
|
337
|
+
}
|
|
338
|
+
const sep = document.createElement("span"); sep.className="lbl"; sep.textContent="Priority";
|
|
339
|
+
sep.style.marginLeft = "8px"; rail.appendChild(sep);
|
|
340
|
+
for (const p of [1,2,3]){
|
|
341
|
+
const c = document.createElement("button");
|
|
342
|
+
c.className="chip"; c.dataset.prio=p; c.setAttribute("aria-pressed","true");
|
|
343
|
+
c.textContent = "P"+p;
|
|
344
|
+
c.onclick = () => { c.setAttribute("aria-pressed", prioOn.has(p) ? "false":"true");
|
|
345
|
+
c.classList.toggle("off", prioOn.has(p));
|
|
346
|
+
prioOn.has(p) ? prioOn.delete(p) : prioOn.add(p); applyFilter(); };
|
|
347
|
+
rail.appendChild(c);
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
function applyFilter(){
|
|
352
|
+
let shown = 0;
|
|
353
|
+
document.querySelectorAll(".node").forEach(n => {
|
|
354
|
+
const ok = areaOn.has(n.dataset.area) && prioOn.has(+n.dataset.prio);
|
|
355
|
+
n.classList.toggle("hide", !ok); if (ok) shown++;
|
|
356
|
+
});
|
|
357
|
+
document.querySelectorAll("section").forEach(s => {
|
|
358
|
+
const any = [...s.querySelectorAll(".node")].some(n => !n.classList.contains("hide"));
|
|
359
|
+
s.style.display = any ? "" : "none";
|
|
360
|
+
});
|
|
361
|
+
document.getElementById("empty").style.display = shown ? "none" : "block";
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
buildRail();
|
|
365
|
+
render();
|
|
366
|
+
</script>
|
|
367
|
+
</body>
|
|
368
|
+
</html>
|
|
@@ -65,15 +65,21 @@ def on_train(compiled, reporter):
|
|
|
65
65
|
epochs = int(hp.get("epochs", 60))
|
|
66
66
|
lr = float(hp.get("lr", 1e-3))
|
|
67
67
|
random.seed(int(hp.get("seed", 0)))
|
|
68
|
-
|
|
68
|
+
|
|
69
|
+
# Consume the wired Loss node so the connection means something: cross-entropy
|
|
70
|
+
# starts high and decays fast toward a higher floor; mse rides lower with a
|
|
71
|
+
# smaller generalization gap. Swap your real objective in here the same way.
|
|
72
|
+
loss_kind = compiled.loss_params.get("loss", "cross_entropy")
|
|
73
|
+
amp, floor_base, gap = (0.8, 0.05, 0.03) if loss_kind == "mse" else (1.4, 0.15, 0.05)
|
|
74
|
+
reporter.state(epochs=epochs, loss=loss_kind)
|
|
69
75
|
|
|
70
76
|
# A toy loss curve: exponential decay toward a floor, shaped by lr, plus noise.
|
|
71
77
|
best = float("inf")
|
|
72
|
-
floor =
|
|
78
|
+
floor = floor_base + random.random() * 0.1
|
|
73
79
|
for e in range(epochs):
|
|
74
80
|
t = e / max(1, epochs - 1)
|
|
75
|
-
train = floor + (
|
|
76
|
-
val = train +
|
|
81
|
+
train = floor + (amp * math.exp(-4 * lr * 100 * t)) + random.uniform(-0.02, 0.02)
|
|
82
|
+
val = train + gap + random.uniform(0, 0.04) # a little generalization gap
|
|
77
83
|
best = min(best, val)
|
|
78
84
|
reporter.metric(epoch=e, train=round(train, 4), val=round(val, 4))
|
|
79
85
|
reporter.state(epoch=e, best_val=round(best, 4))
|
|
@@ -33,7 +33,11 @@ can flag every problem at once) on: unknown/duplicate node, a link to a missing
|
|
|
33
33
|
or port, a **type mismatch** on any wire (scalars being field-specific catches
|
|
34
34
|
lr→epochs), not exactly one Train node, an unconnected/mis-sourced ``model`` input,
|
|
35
35
|
a Model→Transform→…→Dataset chain that doesn't terminate at a Dataset, a wired
|
|
36
|
-
``experimental`` node, a feature-only dataset,
|
|
36
|
+
``experimental`` node, a feature-only dataset, a hyperparameter value outside its
|
|
37
|
+
declared ``min``/``max``, or any host semantic check failing.
|
|
38
|
+
|
|
39
|
+
Shape validation of *loaded data* happens later, not here (the compiler never sees
|
|
40
|
+
your arrays): call :meth:`CompiledGraph.check_shape` at the top of ``on_train``.
|
|
37
41
|
"""
|
|
38
42
|
from __future__ import annotations
|
|
39
43
|
|
|
@@ -72,6 +76,58 @@ class CompiledGraph:
|
|
|
72
76
|
"loss_params": self.loss_params, "hyperparameters": self.hyperparameters,
|
|
73
77
|
}
|
|
74
78
|
|
|
79
|
+
# -- declared dataset shape (from the dataset node's metadata) ---------------
|
|
80
|
+
@property
|
|
81
|
+
def dataset_features(self) -> list[str]:
|
|
82
|
+
"""The feature columns the dataset node declared."""
|
|
83
|
+
return list(self.meta.get("dataset_meta", {}).get("features", []))
|
|
84
|
+
|
|
85
|
+
@property
|
|
86
|
+
def dataset_targets(self) -> list[str]:
|
|
87
|
+
"""The target columns the dataset node declared."""
|
|
88
|
+
return list(self.meta.get("dataset_meta", {}).get("targets", []))
|
|
89
|
+
|
|
90
|
+
def check_shape(self, X, y=None) -> None:
|
|
91
|
+
"""Fail early, with a message naming the dataset, when a loaded array's
|
|
92
|
+
width doesn't match what the dataset node declared.
|
|
93
|
+
|
|
94
|
+
Call this at the top of your ``on_train``, right after loading (and before
|
|
95
|
+
applying transforms, which may change the column count) — a mismatch then
|
|
96
|
+
surfaces cleanly on the canvas instead of dying deep inside your model with
|
|
97
|
+
a cryptic error. ``X`` is compared against :attr:`dataset_features`, and
|
|
98
|
+
``y`` (when given) against :attr:`dataset_targets`. Raises
|
|
99
|
+
:class:`ValueError`; no-ops when the dataset declared no features/targets or
|
|
100
|
+
the array's width can't be determined. Accepts numpy/pandas objects (via
|
|
101
|
+
``.shape``) and plain lists of rows, with no hard dependency on either."""
|
|
102
|
+
want_x = len(self.dataset_features)
|
|
103
|
+
if want_x:
|
|
104
|
+
got = _ncols(X)
|
|
105
|
+
if got is not None and got != want_x:
|
|
106
|
+
raise ValueError(
|
|
107
|
+
f"dataset {self.dataset!r} declares {want_x} feature(s) "
|
|
108
|
+
f"{self.dataset_features} but the loaded X has {got} column(s)")
|
|
109
|
+
want_y = len(self.dataset_targets)
|
|
110
|
+
if want_y and y is not None:
|
|
111
|
+
got = _ncols(y)
|
|
112
|
+
if got is not None and got != want_y:
|
|
113
|
+
raise ValueError(
|
|
114
|
+
f"dataset {self.dataset!r} declares {want_y} target(s) "
|
|
115
|
+
f"{self.dataset_targets} but the loaded y has {got} column(s)")
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _ncols(a) -> int | None:
|
|
119
|
+
"""Best-effort column count of a 2-D array-like, or ``None`` if it can't be
|
|
120
|
+
told. Reads numpy/pandas ``.shape`` without importing them, and falls back to
|
|
121
|
+
a list-of-rows; a flat 1-D sequence counts as a single column."""
|
|
122
|
+
shape = getattr(a, "shape", None)
|
|
123
|
+
if shape is not None:
|
|
124
|
+
return int(shape[1]) if len(shape) >= 2 else 1
|
|
125
|
+
if isinstance(a, (list, tuple)):
|
|
126
|
+
if not a:
|
|
127
|
+
return None
|
|
128
|
+
return len(a[0]) if isinstance(a[0], (list, tuple)) else 1
|
|
129
|
+
return None
|
|
130
|
+
|
|
75
131
|
|
|
76
132
|
def _index_catalog(reg: Registry) -> dict[str, dict]:
|
|
77
133
|
idx = {}
|
|
@@ -198,12 +254,15 @@ def compile_graph(graph: dict, reg: Registry,
|
|
|
198
254
|
loss_params = _params_with_defaults(catalog, gnodes[loss_link[0]])
|
|
199
255
|
|
|
200
256
|
hyper: dict = {}
|
|
257
|
+
bound_errs: list[str] = []
|
|
201
258
|
for port_name, (src_id, _sp) in incoming[train_id].items():
|
|
202
259
|
if port_name in ("model", "loss"):
|
|
203
260
|
continue
|
|
204
261
|
hp = _params_with_defaults(catalog, gnodes[src_id])
|
|
205
262
|
if port_name in hp:
|
|
206
|
-
|
|
263
|
+
val = hp[port_name]
|
|
264
|
+
hyper[port_name] = val
|
|
265
|
+
bound_errs.extend(_bound_errors(catalog, gnodes[src_id], port_name, val))
|
|
207
266
|
|
|
208
267
|
compiled = CompiledGraph(
|
|
209
268
|
run_id=run_id or f"{ds_key}_{uuid.uuid4().hex[:8]}",
|
|
@@ -213,8 +272,8 @@ def compile_graph(graph: dict, reg: Registry,
|
|
|
213
272
|
meta={"dataset_meta": ds_meta, "model_meta": model_meta},
|
|
214
273
|
)
|
|
215
274
|
|
|
216
|
-
# -- semantic checks: built-in
|
|
217
|
-
sem: list[str] = list(_builtin_checks(compiled))
|
|
275
|
+
# -- semantic checks: hyperparameter bounds, built-in rules, then host rules -
|
|
276
|
+
sem: list[str] = bound_errs + list(_builtin_checks(compiled))
|
|
218
277
|
for check in reg.semantic_checks:
|
|
219
278
|
sem.extend(check(compiled) or [])
|
|
220
279
|
if sem:
|
|
@@ -223,6 +282,27 @@ def compile_graph(graph: dict, reg: Registry,
|
|
|
223
282
|
return compiled
|
|
224
283
|
|
|
225
284
|
|
|
285
|
+
def _bound_errors(catalog: dict, gnode: dict, port_name: str, val) -> list[str]:
|
|
286
|
+
"""Enforce a hyperparameter's declared ``min``/``max`` at the boundary, so
|
|
287
|
+
``on_train`` never receives a value the UI advertised as impossible (e.g.
|
|
288
|
+
``epochs=0`` or a negative ``lr`` from a hand-edited or imported graph). The
|
|
289
|
+
canvas widgets clamp interactively; this is the guarantee behind them. Bools
|
|
290
|
+
are left alone (``True``/``False`` are numeric in Python but not a range)."""
|
|
291
|
+
if isinstance(val, bool) or not isinstance(val, (int, float)):
|
|
292
|
+
return []
|
|
293
|
+
spec = catalog[gnode["type"]]
|
|
294
|
+
pj = next((p for p in spec["params"] if p["name"] == port_name), None)
|
|
295
|
+
if not pj:
|
|
296
|
+
return []
|
|
297
|
+
errs: list[str] = []
|
|
298
|
+
lo, hi = pj.get("min"), pj.get("max")
|
|
299
|
+
if lo is not None and val < lo:
|
|
300
|
+
errs.append(f"hyperparameter {port_name!r} = {val} is below its minimum {lo}")
|
|
301
|
+
if hi is not None and val > hi:
|
|
302
|
+
errs.append(f"hyperparameter {port_name!r} = {val} is above its maximum {hi}")
|
|
303
|
+
return errs
|
|
304
|
+
|
|
305
|
+
|
|
226
306
|
def _builtin_checks(c: CompiledGraph) -> list[str]:
|
|
227
307
|
"""Always-on rules derived from registry metadata: a dataset must be trainable,
|
|
228
308
|
and a model's ``requires_kind`` (if set) must match the dataset's ``kind``."""
|
|
@@ -60,14 +60,21 @@ def on_train(compiled, reporter):
|
|
|
60
60
|
epochs = int(hp.get("epochs", 60))
|
|
61
61
|
lr = float(hp.get("lr", 1e-3))
|
|
62
62
|
random.seed(int(hp.get("seed", 0)))
|
|
63
|
-
|
|
63
|
+
|
|
64
|
+
# The wired Loss node actually shapes the curve now: cross-entropy starts high
|
|
65
|
+
# and decays fast toward a higher floor; mse rides lower with a smaller
|
|
66
|
+
# generalization gap. A toy stand-in, but flipping the Loss node visibly
|
|
67
|
+
# changes what you train — the wire is no longer decorative.
|
|
68
|
+
loss_kind = compiled.loss_params.get("loss", "cross_entropy")
|
|
69
|
+
amp, floor_base, gap = (0.8, 0.05, 0.03) if loss_kind == "mse" else (1.4, 0.15, 0.05)
|
|
70
|
+
reporter.state(epochs=epochs, loss=loss_kind)
|
|
64
71
|
|
|
65
72
|
best = float("inf")
|
|
66
|
-
floor =
|
|
73
|
+
floor = floor_base + random.random() * 0.1
|
|
67
74
|
for e in range(epochs):
|
|
68
75
|
t = e / max(1, epochs - 1)
|
|
69
|
-
train = floor + (
|
|
70
|
-
val = train +
|
|
76
|
+
train = floor + (amp * math.exp(-4 * lr * 100 * t)) + random.uniform(-0.02, 0.02)
|
|
77
|
+
val = train + gap + random.uniform(0, 0.04)
|
|
71
78
|
best = min(best, val)
|
|
72
79
|
reporter.metric(epoch=e, train=round(train, 4), val=round(val, 4))
|
|
73
80
|
reporter.state(epoch=e, best_val=round(best, 4))
|
|
@@ -18,14 +18,19 @@ THE STATE CONTRACT (what the editor's live panel + chart read)
|
|
|
18
18
|
``reporter.state(...)`` merges keys into ``state.json``; ``reporter.metric(...)``
|
|
19
19
|
appends a row to ``metrics.jsonl``. The frontend understands these keys:
|
|
20
20
|
state: ``phase`` (running|done|error|paused), ``epoch``, ``epochs``,
|
|
21
|
-
``best_val``, ``test_score``, ``error``.
|
|
21
|
+
``best_val``, ``test_score``, ``error``, ``traceback``.
|
|
22
22
|
metric row: ``epoch``, ``train``, ``val`` — drives the live learning curve.
|
|
23
23
|
Anything else you write is preserved and returned by the API, just not charted.
|
|
24
|
+
|
|
25
|
+
If your ``on_train`` raises, the job runner catches it and records ``phase='error'``
|
|
26
|
+
with a one-line ``error`` summary plus the full ``traceback`` (see ``_Job._run``), so
|
|
27
|
+
a crashing loader or training loop shows up on the canvas instead of failing silently.
|
|
24
28
|
"""
|
|
25
29
|
from __future__ import annotations
|
|
26
30
|
|
|
27
31
|
import json
|
|
28
32
|
import threading
|
|
33
|
+
import traceback
|
|
29
34
|
import uuid
|
|
30
35
|
from dataclasses import dataclass, field
|
|
31
36
|
from pathlib import Path
|
|
@@ -82,7 +87,18 @@ class _Job:
|
|
|
82
87
|
if reporter._state.get("phase") == "running":
|
|
83
88
|
reporter.state(phase="done")
|
|
84
89
|
except Exception as e: # never let a job crash silently
|
|
85
|
-
|
|
90
|
+
# Record a clear, one-line summary for the editor panel, plus the full
|
|
91
|
+
# traceback for debugging. str(e) alone is often cryptic — e.g.
|
|
92
|
+
# str(KeyError('species')) is just "'species'" — so lead with the type.
|
|
93
|
+
reporter.state(
|
|
94
|
+
phase="error",
|
|
95
|
+
error=f"{type(e).__name__}: {e}",
|
|
96
|
+
traceback="".join(
|
|
97
|
+
traceback.format_exception(type(e), e, e.__traceback__)),
|
|
98
|
+
)
|
|
99
|
+
# Also surface it in the server log: a crashed run should never be
|
|
100
|
+
# invisible to whoever is running the process.
|
|
101
|
+
traceback.print_exc()
|
|
86
102
|
finally:
|
|
87
103
|
self._app.jobs.pop(self.compiled.run_id, None)
|
|
88
104
|
|
|
@@ -105,6 +105,11 @@
|
|
|
105
105
|
}
|
|
106
106
|
#out .ok { color:var(--ok); } #out .err { color:var(--err); }
|
|
107
107
|
#out .k { color:var(--muted); }
|
|
108
|
+
/* full on_train traceback: wrapped, scrollable, kept out of the way */
|
|
109
|
+
#out .trace { white-space:pre-wrap; margin:6px 0 0; padding:6px 8px;
|
|
110
|
+
background:#0f131a; border:1px solid var(--edge); border-radius:6px;
|
|
111
|
+
color:var(--muted); font-size:10px; line-height:1.45; max-height:170px;
|
|
112
|
+
overflow:auto; }
|
|
108
113
|
#out h4 { margin:0 0 6px; font:600 11px/1 ui-sans-serif; letter-spacing:.4px; }
|
|
109
114
|
/* clickable error lines: click to select + centre the offending node */
|
|
110
115
|
#out .errline { cursor:pointer; border-radius:3px; padding:0 2px;
|
|
@@ -863,13 +863,16 @@
|
|
|
863
863
|
if (dot) dot.classList.toggle("live", phase === "running");
|
|
864
864
|
setTrainState(phase === "running" ? "running" : (graphValid ? "ready" : "invalid"));
|
|
865
865
|
|
|
866
|
+
// st.error / st.traceback come straight from the host's exception, so they may
|
|
867
|
+
// contain HTML-special chars (Python errors are full of <, >, ') — escape both.
|
|
866
868
|
setPanel(errored ? "err" : (done ? "ok" : "k"),
|
|
867
|
-
`<b>${done ? "✓" : errored ? "✗" : "▶"} ${runId}</b> · ${phase}<br>` +
|
|
869
|
+
`<b>${done ? "✓" : errored ? "✗" : "▶"} ${escapeHtml(runId)}</b> · ${phase}<br>` +
|
|
868
870
|
`<span class="k">epoch</span> ${ep}/${eps}` +
|
|
869
871
|
(phase === "running" ? ` <span class="k">(${Math.round(prog * 100)}%)</span>` : "") + `<br>` +
|
|
870
872
|
`<span class="k">best val</span> ${fmt(st.best_val)}<br>` +
|
|
871
873
|
`<span class="k">test score</span> ${fmt(st.test_score)}` +
|
|
872
|
-
(errored && st.error ? `<br><span class="err">${st.error}</span>` : "")
|
|
874
|
+
(errored && st.error ? `<br><span class="err">${escapeHtml(st.error)}</span>` : "") +
|
|
875
|
+
(errored && st.traceback ? `<pre class="trace">${escapeHtml(st.traceback)}</pre>` : ""));
|
|
873
876
|
|
|
874
877
|
if (done || errored) { stopPolling(); refreshRuns(); return; }
|
|
875
878
|
pollTimer = setTimeout(() => pollRun(runId), 1500);
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
"""Robustness-section tests: the wired Loss node changes training (#2), the
|
|
2
|
+
CompiledGraph shape check catches a mismatched array (#3), and the compiler
|
|
3
|
+
enforces hyperparameter min/max at the boundary (#4).
|
|
4
|
+
|
|
5
|
+
Dependency-free — pure compiler + a fake reporter, no web server or ML deps. Runs
|
|
6
|
+
under pytest, or standalone with ``python tests/test_robustness.py``.
|
|
7
|
+
"""
|
|
8
|
+
from modulearn import compile_graph, GraphError
|
|
9
|
+
from modulearn.compiler import CompiledGraph
|
|
10
|
+
from modulearn.demo import build_registry, on_train
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
# -- shared graph builders ----------------------------------------------------
|
|
14
|
+
def _base_graph():
|
|
15
|
+
"""A minimal valid iris → mlp → train graph (no hyperparameters wired)."""
|
|
16
|
+
return {
|
|
17
|
+
"nodes": [
|
|
18
|
+
{"id": "d", "type": "dataset.iris", "params": {}},
|
|
19
|
+
{"id": "m", "type": "model.mlp", "params": {"hidden": [8]}},
|
|
20
|
+
{"id": "t", "type": "train.train", "params": {}},
|
|
21
|
+
],
|
|
22
|
+
"links": [
|
|
23
|
+
{"from": "d", "from_port": "dataset", "to": "m", "to_port": "dataset"},
|
|
24
|
+
{"from": "m", "from_port": "model", "to": "t", "to_port": "model"},
|
|
25
|
+
],
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _graph_with_epochs(value):
|
|
30
|
+
"""The base graph plus an epochs hyperparameter node wired to the Train sink."""
|
|
31
|
+
g = _base_graph()
|
|
32
|
+
g["nodes"].append({"id": "e", "type": "hyperparameter.epochs",
|
|
33
|
+
"params": {"epochs": value}})
|
|
34
|
+
g["links"].append({"from": "e", "from_port": "value", "to": "t", "to_port": "epochs"})
|
|
35
|
+
return g
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class _FakeReporter:
|
|
39
|
+
"""Captures what on_train publishes, without touching disk."""
|
|
40
|
+
def __init__(self):
|
|
41
|
+
self.state_kw = {}
|
|
42
|
+
self.metrics = []
|
|
43
|
+
|
|
44
|
+
def state(self, **kw):
|
|
45
|
+
self.state_kw.update(kw)
|
|
46
|
+
|
|
47
|
+
def metric(self, **row):
|
|
48
|
+
self.metrics.append(row)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
# -- #2: the Loss node actually shapes training -------------------------------
|
|
52
|
+
def _run_with_loss(loss_kind):
|
|
53
|
+
compiled = CompiledGraph(
|
|
54
|
+
run_id="r", dataset="iris", model="mlp",
|
|
55
|
+
loss_params={"loss": loss_kind},
|
|
56
|
+
hyperparameters={"epochs": 3, "lr": 0.1, "seed": 1})
|
|
57
|
+
rep = _FakeReporter()
|
|
58
|
+
on_train(compiled, rep)
|
|
59
|
+
return rep
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def test_loss_is_consumed_and_reported():
|
|
63
|
+
rep = _run_with_loss("mse")
|
|
64
|
+
assert rep.state_kw.get("loss") == "mse" # the wire is recorded...
|
|
65
|
+
assert rep.state_kw.get("phase") == "done"
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def test_loss_choice_changes_the_curve():
|
|
69
|
+
# Same seed, different objective → a different curve. Proves the Loss wire is
|
|
70
|
+
# no longer decorative: flipping it changes what you train.
|
|
71
|
+
ce = _run_with_loss("cross_entropy").metrics
|
|
72
|
+
mse = _run_with_loss("mse").metrics
|
|
73
|
+
assert ce[0]["train"] != mse[0]["train"]
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
# -- #3: shape validation against the declared dataset ------------------------
|
|
77
|
+
def test_declared_shape_is_exposed():
|
|
78
|
+
c = compile_graph(_base_graph(), build_registry(), run_id="r")
|
|
79
|
+
assert len(c.dataset_features) == 4 # iris: 4 features
|
|
80
|
+
assert c.dataset_targets == ["species"]
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def test_check_shape_accepts_matching_arrays():
|
|
84
|
+
c = compile_graph(_base_graph(), build_registry(), run_id="r")
|
|
85
|
+
c.check_shape([[0, 0, 0, 0], [1, 1, 1, 1]]) # 4 columns → ok
|
|
86
|
+
c.check_shape([[0, 0, 0, 0]], y=[0]) # 1 target column → ok
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def test_check_shape_rejects_wrong_feature_count():
|
|
90
|
+
c = compile_graph(_base_graph(), build_registry(), run_id="r")
|
|
91
|
+
try:
|
|
92
|
+
c.check_shape([[0, 0, 0]]) # 3 != 4
|
|
93
|
+
except ValueError as e:
|
|
94
|
+
assert "iris" in str(e) and "4 feature" in str(e)
|
|
95
|
+
return
|
|
96
|
+
raise AssertionError("expected ValueError for wrong feature count")
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def test_check_shape_rejects_wrong_target_count():
|
|
100
|
+
c = compile_graph(_base_graph(), build_registry(), run_id="r")
|
|
101
|
+
try:
|
|
102
|
+
c.check_shape([[0, 0, 0, 0]], y=[[0, 0]]) # declares 1 target, got 2
|
|
103
|
+
except ValueError as e:
|
|
104
|
+
assert "iris" in str(e) and "target" in str(e)
|
|
105
|
+
return
|
|
106
|
+
raise AssertionError("expected ValueError for wrong target count")
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
# -- #4: hyperparameter bounds enforced at compile ----------------------------
|
|
110
|
+
def test_in_range_hyperparameter_compiles():
|
|
111
|
+
c = compile_graph(_graph_with_epochs(50), build_registry(), run_id="r")
|
|
112
|
+
assert c.hyperparameters["epochs"] == 50
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def test_below_min_is_rejected():
|
|
116
|
+
try:
|
|
117
|
+
compile_graph(_graph_with_epochs(0), build_registry()) # min is 1
|
|
118
|
+
except GraphError as e:
|
|
119
|
+
assert any("epochs" in m and "minimum" in m for m in e.errors)
|
|
120
|
+
return
|
|
121
|
+
raise AssertionError("expected GraphError for epochs below minimum")
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def test_above_max_is_rejected():
|
|
125
|
+
try:
|
|
126
|
+
compile_graph(_graph_with_epochs(9999), build_registry()) # max is 2000
|
|
127
|
+
except GraphError as e:
|
|
128
|
+
assert any("epochs" in m and "maximum" in m for m in e.errors)
|
|
129
|
+
return
|
|
130
|
+
raise AssertionError("expected GraphError for epochs above maximum")
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
if __name__ == "__main__":
|
|
134
|
+
for name, fn in sorted(globals().items()):
|
|
135
|
+
if name.startswith("test_") and callable(fn):
|
|
136
|
+
fn()
|
|
137
|
+
print(f"ok {name}")
|
|
138
|
+
print("all robustness tests passed")
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
"""Run-lifecycle tests: a crashing ``on_train`` is recorded as a failed run — with
|
|
2
|
+
a clear, one-line error and a full traceback — rather than hanging or vanishing,
|
|
3
|
+
and a normal run still completes.
|
|
4
|
+
|
|
5
|
+
Dependency-free: it drives the job runner (``_Job._run``) synchronously, so there
|
|
6
|
+
is no daemon thread and no web server in the loop. Runs under pytest, or standalone
|
|
7
|
+
with ``python tests/test_run_errors.py``.
|
|
8
|
+
|
|
9
|
+
The crash tests intentionally exercise the server-side ``traceback.print_exc()``
|
|
10
|
+
log, so you will see tracebacks printed to stderr while they run — that is expected,
|
|
11
|
+
not a failure. The trailing "all run-error tests passed" line is the real verdict.
|
|
12
|
+
"""
|
|
13
|
+
import json
|
|
14
|
+
import tempfile
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
from modulearn.compiler import CompiledGraph
|
|
18
|
+
from modulearn.registry import Registry
|
|
19
|
+
from modulearn.server import _AppState, _Job
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _run_sync(on_train, run_id="r") -> dict:
|
|
23
|
+
"""Run one job to completion in-thread and return its parsed state.json."""
|
|
24
|
+
tmp = Path(tempfile.mkdtemp())
|
|
25
|
+
compiled = CompiledGraph(run_id=run_id, dataset="iris", model="mlp")
|
|
26
|
+
state = _AppState(registry=Registry(), on_train=on_train, runs=tmp)
|
|
27
|
+
_Job(state, compiled)._run() # synchronous: skip the daemon thread
|
|
28
|
+
return json.loads((tmp / run_id / "state.json").read_text())
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def test_crash_is_recorded_as_error():
|
|
32
|
+
def boom(compiled, reporter):
|
|
33
|
+
raise RuntimeError("bad shape: <42>")
|
|
34
|
+
st = _run_sync(boom)
|
|
35
|
+
assert st["phase"] == "error"
|
|
36
|
+
# The one-line summary leads with the exception type so it isn't cryptic...
|
|
37
|
+
assert st["error"] == "RuntimeError: bad shape: <42>"
|
|
38
|
+
# ...and the full traceback is kept for debugging.
|
|
39
|
+
assert "Traceback" in st["traceback"]
|
|
40
|
+
assert "boom" in st["traceback"]
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def test_crash_before_any_report_still_errors():
|
|
44
|
+
# A loader that dies immediately (e.g. a missing file) must still surface,
|
|
45
|
+
# even though on_train never called reporter.metric/state itself.
|
|
46
|
+
def boom(compiled, reporter):
|
|
47
|
+
raise FileNotFoundError("no such file: data.parquet")
|
|
48
|
+
st = _run_sync(boom)
|
|
49
|
+
assert st["phase"] == "error"
|
|
50
|
+
assert st["error"].startswith("FileNotFoundError:")
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def test_job_deregistered_after_crash():
|
|
54
|
+
# The finally-clause must drop the job so the run is no longer "live".
|
|
55
|
+
tmp = Path(tempfile.mkdtemp())
|
|
56
|
+
compiled = CompiledGraph(run_id="gone", dataset="iris", model="mlp")
|
|
57
|
+
|
|
58
|
+
def boom(compiled, reporter):
|
|
59
|
+
raise ValueError("x")
|
|
60
|
+
|
|
61
|
+
state = _AppState(registry=Registry(), on_train=boom, runs=tmp)
|
|
62
|
+
_Job(state, compiled)._run()
|
|
63
|
+
assert "gone" not in state.jobs
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def test_normal_run_completes():
|
|
67
|
+
# The success path is untouched: returning cleanly auto-finalizes to done.
|
|
68
|
+
def ok(compiled, reporter):
|
|
69
|
+
reporter.metric(epoch=0, train=1.0, val=1.1)
|
|
70
|
+
reporter.state(epoch=0)
|
|
71
|
+
st = _run_sync(ok)
|
|
72
|
+
assert st["phase"] == "done"
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
if __name__ == "__main__":
|
|
76
|
+
for name, fn in sorted(globals().items()):
|
|
77
|
+
if name.startswith("test_") and callable(fn):
|
|
78
|
+
fn()
|
|
79
|
+
print(f"ok {name}")
|
|
80
|
+
print("all run-error tests passed")
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|