modulearn 0.1.0__py3-none-any.whl

modulearn/__init__.py

@@ -0,0 +1,16 @@
+"""ModuLearn — a visual, node-graph training editor you wire to your own models.
+
+Describe your datasets/models/hyperparameters with a :class:`Registry`, pass it to
+:func:`create_app` with an ``on_train`` callback, and you get an Unreal-Blueprints
+style canvas: drop nodes, wire them, launch, and watch training live on the graph.
+"""
+from modulearn.registry import Registry, Param, Port, NodeSpec, PORT_TYPES, CATEGORIES
+from modulearn.compiler import compile_graph, CompiledGraph, GraphError
+from modulearn.server import create_app, RunReporter
+
+__all__ = [
+    "Registry", "Param", "Port", "NodeSpec", "PORT_TYPES", "CATEGORIES",
+    "compile_graph", "CompiledGraph", "GraphError",
+    "create_app", "RunReporter",
+]
+__version__ = "0.1.0"

modulearn/cli.py

@@ -0,0 +1,88 @@
+"""``modulearn`` command-line entry point.
+
+    modulearn demo                 # serve the bundled demo editor
+    modulearn run app.py           # serve YOUR app's FastAPI object (attr `app`)
+    modulearn run app.py:editor    # ...or a differently-named attribute
+    modulearn --version
+
+``run`` loads a Python file that builds an app with ``create_app`` and serves it.
+The target is a FastAPI instance (e.g. ``app = create_app(reg, on_train)``) or a
+zero-arg factory that returns one.
+"""
+import argparse
+import importlib.util
+import sys
+from pathlib import Path
+
+from modulearn import __version__
+
+
+def _load_app(target: str):
+    """Resolve ``path.py`` or ``path.py:attr`` to a FastAPI app object."""
+    path_str, _, attr = target.partition(":")
+    attr = attr or "app"
+    path = Path(path_str).resolve()
+    if not path.exists():
+        sys.exit(f"modulearn: no such file: {path_str}")
+
+    # Import the file as a module, with its own directory on sys.path so the
+    # user's app can import sibling modules.
+    sys.path.insert(0, str(path.parent))
+    spec = importlib.util.spec_from_file_location(path.stem, path)
+    module = importlib.util.module_from_spec(spec)
+    try:
+        spec.loader.exec_module(module)
+    except Exception as e:  # surface the user's import error cleanly
+        sys.exit(f"modulearn: failed to import {path_str}: {e}")
+
+    obj = getattr(module, attr, None)
+    if obj is None:
+        sys.exit(f"modulearn: {path_str} has no attribute '{attr}' "
+                 f"(expected a FastAPI app or a factory returning one)")
+    return obj() if callable(obj) and not _is_asgi_app(obj) else obj
+
+
+def _is_asgi_app(obj) -> bool:
+    """A FastAPI/Starlette instance is callable too — tell it apart from a factory."""
+    return obj.__class__.__module__.startswith(("fastapi", "starlette"))
+
+
+def _serve(app, host: str, port: int):
+    import uvicorn
+    url = f"http://{'localhost' if host in ('0.0.0.0', '') else host}:{port}"
+    print(f"ModuLearn → {url}  (Ctrl-C to stop)")
+    uvicorn.run(app, host=host, port=port)
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(
+        prog="modulearn",
+        description="Visual, node-graph editor for building and launching ML training runs.")
+    parser.add_argument("-V", "--version", action="version",
+                        version=f"modulearn {__version__}")
+    sub = parser.add_subparsers(dest="command")
+
+    p_demo = sub.add_parser("demo", help="serve the bundled demo editor")
+    p_demo.add_argument("--host", default="127.0.0.1")
+    p_demo.add_argument("--port", type=int, default=8000)
+
+    p_run = sub.add_parser("run", help="serve your own app.py (an app built with create_app)")
+    p_run.add_argument("target", metavar="APP",
+                       help="path to your app file, e.g. train.py or train.py:app")
+    p_run.add_argument("--host", default="127.0.0.1")
+    p_run.add_argument("--port", type=int, default=8000)
+
+    args = parser.parse_args(argv)
+
+    if args.command == "demo":
+        from modulearn.demo import build_app
+        _serve(build_app(), args.host, args.port)
+    elif args.command == "run":
+        _serve(_load_app(args.target), args.host, args.port)
+    else:
+        parser.print_help()
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

modulearn/compiler.py

@@ -0,0 +1,238 @@
+#!/usr/bin/env python3
+"""Compile a visual ('Blueprints') canvas into a validated :class:`CompiledGraph`.
+
+The frontend serializes the canvas to the compact **graph JSON** below and POSTs it
+here. This module walks that graph, type-checks every wire against the registry's
+catalog, runs any project-specific semantic checks, and — if valid — hands back a
+plain :class:`CompiledGraph`. Your ``on_train`` callback turns that into whatever
+config object your trainer wants. Compilation is pure and side-effect-free, so it is
+cheap to unit-test with no web server or ML framework in the loop.
+
+THE GRAPH JSON (input to :func:`compile_graph`)
+-----------------------------------------------
+    {
+      "nodes": [
+        {"id": "n1", "type": "dataset.iris",       "params": {}},
+        {"id": "n2", "type": "model.mlp",          "params": {"hidden": [64, 32]}},
+        {"id": "n3", "type": "hyperparameter.lr",  "params": {"lr": 5e-4}},
+        {"id": "n4", "type": "loss.loss",          "params": {"loss": "mse"}},
+        {"id": "n9", "type": "train.train",        "params": {}}
+      ],
+      "links": [
+        {"from": "n1", "from_port": "dataset", "to": "n2", "to_port": "dataset"},
+        {"from": "n2", "from_port": "model",   "to": "n9", "to_port": "model"},
+        {"from": "n4", "from_port": "loss",    "to": "n9", "to_port": "loss"},
+        {"from": "n3", "from_port": "value",   "to": "n9", "to_port": "lr"}
+      ]
+    }
+
+VALIDATION (all failures collected, then raised together)
+---------------------------------------------------------
+:func:`compile_graph` raises :class:`GraphError` (carrying the full list, so the UI
+can flag every problem at once) on: unknown/duplicate node, a link to a missing node
+or port, a **type mismatch** on any wire (scalars being field-specific catches
+lr→epochs), not exactly one Train node, an unconnected/mis-sourced ``model`` input,
+a Model→Transform→…→Dataset chain that doesn't terminate at a Dataset, a wired
+``experimental`` node, a feature-only dataset, or any host semantic check failing.
+"""
+from __future__ import annotations
+
+import uuid
+from dataclasses import dataclass, field
+
+from modulearn.registry import Registry
+
+
+class GraphError(Exception):
+    """Raised when a graph can't compile. ``.errors`` is the full problem list."""
+
+    def __init__(self, errors: list[str]):
+        self.errors = errors
+        super().__init__("; ".join(errors))
+
+
+@dataclass
+class CompiledGraph:
+    """The validated result of a canvas. Everything a trainer needs, no node-graph
+    or wire concepts left. ``model`` is the short model name (``model.mlp`` → ``mlp``);
+    ``hyperparameters`` maps each wired knob to its value."""
+    run_id: str
+    dataset: str
+    model: str
+    transforms: tuple[str, ...] = ()
+    model_params: dict = field(default_factory=dict)
+    loss_params: dict = field(default_factory=dict)
+    hyperparameters: dict = field(default_factory=dict)
+    meta: dict = field(default_factory=dict)   # {"dataset_meta":..., "model_meta":...}
+
+    def as_dict(self) -> dict:
+        return {
+            "run_id": self.run_id, "dataset": self.dataset, "model": self.model,
+            "transforms": list(self.transforms), "model_params": self.model_params,
+            "loss_params": self.loss_params, "hyperparameters": self.hyperparameters,
+        }
+
+
+def _index_catalog(reg: Registry) -> dict[str, dict]:
+    idx = {}
+    for n in reg.catalog()["nodes"]:
+        idx[n["id"]] = {
+            **n,
+            "_in": {p["name"]: p["type"] for p in n["inputs"]},
+            "_out": {p["name"]: p["type"] for p in n["outputs"]},
+        }
+    return idx
+
+
+def _params_with_defaults(catalog: dict, gnode: dict) -> dict:
+    spec = catalog[gnode["type"]]
+    out = {p["name"]: p.get("default") for p in spec["params"]}
+    out.update(gnode.get("params") or {})
+    return out
+
+
+def compile_graph(graph: dict, reg: Registry,
+                  run_id: str | None = None) -> CompiledGraph:
+    """Validate ``graph`` against ``reg`` and compile it to a :class:`CompiledGraph`.
+
+    Raises :class:`GraphError` with every problem found. ``run_id`` is used verbatim
+    when given, else generated as ``<dataset>_<hex8>``."""
+    catalog = _index_catalog(reg)
+    errors: list[str] = []
+
+    raw_nodes = graph.get("nodes", []) or []
+    raw_links = graph.get("links", []) or []
+
+    # -- index nodes, catching unknown types / duplicate ids -----------------
+    gnodes: dict[str, dict] = {}
+    for gn in raw_nodes:
+        nid, ntype = gn.get("id"), gn.get("type")
+        if nid is None or ntype is None:
+            errors.append(f"node missing id or type: {gn!r}")
+            continue
+        if nid in gnodes:
+            errors.append(f"duplicate node id {nid!r}")
+            continue
+        if ntype not in catalog:
+            errors.append(f"node {nid!r} has unknown type {ntype!r}")
+            continue
+        gnodes[nid] = gn
+
+    # -- validate every link: endpoints, ports, TYPES MATCH ------------------
+    incoming: dict[str, dict[str, tuple[str, str]]] = {nid: {} for nid in gnodes}
+    for lk in raw_links:
+        src, dst = lk.get("from"), lk.get("to")
+        sp, dp = lk.get("from_port"), lk.get("to_port")
+        if src not in gnodes or dst not in gnodes:
+            errors.append(f"link references unknown node(s): {lk!r}")
+            continue
+        src_out = catalog[gnodes[src]["type"]]["_out"]
+        dst_in = catalog[gnodes[dst]["type"]]["_in"]
+        if sp not in src_out:
+            errors.append(f"node {src!r} has no output port {sp!r}")
+            continue
+        if dp not in dst_in:
+            errors.append(f"node {dst!r} has no input port {dp!r}")
+            continue
+        if src_out[sp] != dst_in[dp]:
+            errors.append(f"type mismatch on wire {src}.{sp} ({src_out[sp]}) -> "
+                          f"{dst}.{dp} ({dst_in[dp]})")
+            continue
+        if dp in incoming[dst]:
+            errors.append(f"input {dst!r}.{dp!r} is wired more than once")
+            continue
+        incoming[dst][dp] = (src, sp)
+
+    # -- reject experimental nodes that are actually wired in ----------------
+    wired = {nid for lk in raw_links for nid in (lk.get("from"), lk.get("to"))
+             if nid in gnodes}
+    for nid in wired:
+        if catalog[gnodes[nid]["type"]].get("experimental"):
+            errors.append(f"node {nid!r} ({gnodes[nid]['type']}) is experimental "
+                          f"and cannot be compiled yet — remove it from the run")
+
+    if errors:
+        raise GraphError(errors)
+
+    # -- exactly one Train sink ----------------------------------------------
+    trains = [nid for nid, gn in gnodes.items() if gn["type"] == "train.train"]
+    if len(trains) != 1:
+        raise GraphError([f"graph needs exactly one Train node, found {len(trains)}"])
+    train_id = trains[0]
+
+    # -- resolve model <- (transforms) <- dataset ----------------------------
+    model_link = incoming[train_id].get("model")
+    if not model_link:
+        raise GraphError(["Train node's 'model' input is not connected"])
+    model_id = model_link[0]
+    model_type = gnodes[model_id]["type"]
+
+    transforms: list[str] = []
+    cur = model_id
+    while True:
+        link = incoming[cur].get("dataset")
+        if not link:
+            raise GraphError([f"node {cur!r} 'dataset' input is not connected"])
+        src_id = link[0]
+        src_type = gnodes[src_id]["type"]
+        if src_type.startswith("transform."):
+            transforms.append(src_type.split(".", 1)[1])
+            cur = src_id
+        elif src_type.startswith("dataset."):
+            ds_id = src_id
+            break
+        else:
+            raise GraphError([f"node {cur!r} 'dataset' input must come from a "
+                              f"Dataset or Transform, not {src_type!r}"])
+    transforms.reverse()
+
+    ds_key = gnodes[ds_id]["type"].split(".", 1)[1]
+    ds_meta = catalog[gnodes[ds_id]["type"]].get("meta", {})
+    model_meta = catalog[model_type].get("meta", {})
+
+    # -- gather params -------------------------------------------------------
+    model_params = _params_with_defaults(catalog, gnodes[model_id])
+    loss_params: dict = {}
+    loss_link = incoming[train_id].get("loss")
+    if loss_link:
+        loss_params = _params_with_defaults(catalog, gnodes[loss_link[0]])
+
+    hyper: dict = {}
+    for port_name, (src_id, _sp) in incoming[train_id].items():
+        if port_name in ("model", "loss"):
+            continue
+        hp = _params_with_defaults(catalog, gnodes[src_id])
+        if port_name in hp:
+            hyper[port_name] = hp[port_name]
+
+    compiled = CompiledGraph(
+        run_id=run_id or f"{ds_key}_{uuid.uuid4().hex[:8]}",
+        dataset=ds_key, model=model_type.split(".", 1)[1],
+        transforms=tuple(transforms), model_params=model_params,
+        loss_params=loss_params, hyperparameters=hyper,
+        meta={"dataset_meta": ds_meta, "model_meta": model_meta},
+    )
+
+    # -- semantic checks: built-in kind/trainable rules, then host rules -----
+    sem: list[str] = list(_builtin_checks(compiled))
+    for check in reg.semantic_checks:
+        sem.extend(check(compiled) or [])
+    if sem:
+        raise GraphError(sem)
+
+    return compiled
+
+
+def _builtin_checks(c: CompiledGraph) -> list[str]:
+    """Always-on rules derived from registry metadata: a dataset must be trainable,
+    and a model's ``requires_kind`` (if set) must match the dataset's ``kind``."""
+    errs: list[str] = []
+    ds = c.meta.get("dataset_meta", {})
+    if not ds.get("trainable", True) or not ds.get("targets"):
+        errs.append(f"dataset {c.dataset!r} is feature-only and cannot be trained")
+    need = c.meta.get("model_meta", {}).get("requires_kind")
+    have = ds.get("kind")
+    if need and have and need != have:
+        errs.append(f"model {c.model!r} needs a {need!r} dataset but "
+                    f"{c.dataset!r} is {have!r}")
+    return errs

modulearn/demo.py

@@ -0,0 +1,81 @@
+"""A complete, dependency-free demo app shipped inside the package.
+
+This is what ``modulearn demo`` serves. It mirrors ``examples/quickstart.py``
+(the teaching file), but lives in the package so it ships in the wheel and can be
+launched without the source checkout. Swap ``on_train`` for a real trainer and the
+same editor drives it.
+"""
+import math
+import random
+import time
+
+from modulearn import Registry, Param, create_app
+
+
+def build_registry() -> Registry:
+    """The declarative surface: what this demo can train."""
+    reg = Registry()
+
+    reg.add_dataset("iris", title="Iris (tabular)", kind="tabular",
+                    features=["sepal_len", "sepal_wid", "petal_len", "petal_wid"],
+                    targets=["species"], help="Classic 4-feature flower dataset.")
+    reg.add_dataset("digits", title="Digits (tabular)", kind="tabular",
+                    features=[f"px{i}" for i in range(64)], targets=["digit"],
+                    help="8x8 handwritten digits, flattened.")
+    reg.add_dataset("features_only", title="Unlabeled pool", kind="tabular",
+                    features=["a", "b", "c"], targets=[], trainable=False,
+                    help="No target column — the compiler will refuse to train it.")
+
+    reg.add_model("mlp", title="MLP (feed-forward net)", title_short="MLP",
+                  requires_kind="tabular",
+                  help="Dense ReLU network over tabular features.",
+                  params=[
+                      Param("hidden", "hidden layers", "int_list", [64, 32],
+                            help="Comma-separated widths, e.g. 64,32."),
+                      Param("dropout", "dropout", "float", 0.0, min=0.0, max=0.9),
+                  ])
+
+    reg.add_hyperparameter("lr", label="learning rate", kind="float", default=1e-3,
+                           min=1e-6, max=1.0, help="Step size.")
+    reg.add_hyperparameter("epochs", label="epochs", kind="int", default=60,
+                           min=1, max=2000, help="Total training epochs.")
+    reg.add_hyperparameter("seed", label="seed", kind="int", default=0, min=0,
+                           max=2**31 - 1, help="Reproducibility seed.")
+
+    reg.add_loss([
+        Param("loss", "kind", "enum", "cross_entropy",
+              choices=["cross_entropy", "mse"], help="Objective."),
+    ])
+
+    reg.add_transform("standardize", title="Standardize",
+                      help="Zero-mean, unit-variance each feature.")
+    reg.add_transform("pca", title="PCA (whiten)", live=False,
+                      help="Dimensionality reduction — not wired up in this demo.")
+    return reg
+
+
+def on_train(compiled, reporter):
+    """A toy loss curve so the demo runs anywhere — no ML deps."""
+    hp = compiled.hyperparameters
+    epochs = int(hp.get("epochs", 60))
+    lr = float(hp.get("lr", 1e-3))
+    random.seed(int(hp.get("seed", 0)))
+    reporter.state(epochs=epochs)
+
+    best = float("inf")
+    floor = 0.15 + random.random() * 0.1
+    for e in range(epochs):
+        t = e / max(1, epochs - 1)
+        train = floor + (1.4 * math.exp(-4 * lr * 100 * t)) + random.uniform(-0.02, 0.02)
+        val = train + 0.05 + random.uniform(0, 0.04)
+        best = min(best, val)
+        reporter.metric(epoch=e, train=round(train, 4), val=round(val, 4))
+        reporter.state(epoch=e, best_val=round(best, 4))
+        time.sleep(0.05)
+    reporter.state(phase="done", test_score=round(best + 0.02, 4))
+
+
+def build_app(runs_dir="runs"):
+    """The FastAPI app ``modulearn demo`` serves."""
+    return create_app(build_registry(), on_train, title="ModuLearn",
+                      subtitle="demo", runs_dir=runs_dir)