kanx 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kanx/__init__.py +49 -0
- kanx/__main__.py +121 -0
- kanx/config.py +136 -0
- kanx/inference.py +79 -0
- kanx/layers.py +258 -0
- kanx/model.py +112 -0
- kanx/onnx_export.py +71 -0
- kanx/torch/__init__.py +40 -0
- kanx/torch/layers.py +156 -0
- kanx/torch/model.py +115 -0
- kanx/torch/onnx_export.py +66 -0
- kanx/torch/trainer.py +154 -0
- kanx/train.py +146 -0
- kanx/utils.py +53 -0
- kanx-0.1.0.dist-info/METADATA +296 -0
- kanx-0.1.0.dist-info/RECORD +20 -0
- kanx-0.1.0.dist-info/WHEEL +5 -0
- kanx-0.1.0.dist-info/entry_points.txt +2 -0
- kanx-0.1.0.dist-info/licenses/LICENSE +201 -0
- kanx-0.1.0.dist-info/top_level.txt +1 -0
kanx/__init__.py
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""kanx — Production-grade Kolmogorov-Arnold Networks in TensorFlow.
|
|
2
|
+
|
|
3
|
+
Public API:
|
|
4
|
+
from kanx import KAN, KANLinear, load_config, train, predict
|
|
5
|
+
|
|
6
|
+
This package provides a clean, vectorized, fully-tested implementation of
|
|
7
|
+
Kolmogorov-Arnold Networks (Liu et al., 2024) on top of TensorFlow / Keras 3,
|
|
8
|
+
with first-class support for training, inference, checkpointing, configuration
|
|
9
|
+
files, REST serving (FastAPI), benchmarking and CI/CD.
|
|
10
|
+
|
|
11
|
+
Modules:
|
|
12
|
+
layers — `KANLinear` Keras layer and B-spline primitives.
|
|
13
|
+
model — `KAN` sequential model and high-level builders.
|
|
14
|
+
train — Training loop, callbacks, metrics, checkpointing.
|
|
15
|
+
inference — Inference helpers, batched predict, checkpoint loading.
|
|
16
|
+
config — YAML config schema and validators.
|
|
17
|
+
utils — Logging, seeding, IO helpers.
|
|
18
|
+
"""
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
from .layers import KANLinear, b_spline_basis, extend_grid
|
|
22
|
+
from .model import KAN, build_kan
|
|
23
|
+
from .config import load_config, validate_config, KanxConfig
|
|
24
|
+
from .train import train, build_optimizer, build_loss
|
|
25
|
+
from .inference import predict, load_model, save_model
|
|
26
|
+
from .utils import set_global_seed, get_logger
|
|
27
|
+
from .onnx_export import export_onnx_tf
|
|
28
|
+
|
|
29
|
+
__all__ = [
|
|
30
|
+
"KAN",
|
|
31
|
+
"KANLinear",
|
|
32
|
+
"build_kan",
|
|
33
|
+
"b_spline_basis",
|
|
34
|
+
"extend_grid",
|
|
35
|
+
"load_config",
|
|
36
|
+
"validate_config",
|
|
37
|
+
"KanxConfig",
|
|
38
|
+
"train",
|
|
39
|
+
"build_optimizer",
|
|
40
|
+
"build_loss",
|
|
41
|
+
"predict",
|
|
42
|
+
"load_model",
|
|
43
|
+
"save_model",
|
|
44
|
+
"set_global_seed",
|
|
45
|
+
"get_logger",
|
|
46
|
+
"export_onnx_tf",
|
|
47
|
+
]
|
|
48
|
+
|
|
49
|
+
__version__ = "0.1.0"
|
kanx/__main__.py
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
"""CLI entrypoint: ``python -m kanx --config configs/default.yaml``.
|
|
2
|
+
|
|
3
|
+
Subcommands:
|
|
4
|
+
train — train a model from a YAML config and a CSV / .npz dataset
|
|
5
|
+
predict — load a checkpoint and predict on a CSV / .npz / JSON input
|
|
6
|
+
info — print the package version and TF / device info
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import argparse
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
import sys
|
|
14
|
+
from typing import Tuple
|
|
15
|
+
|
|
16
|
+
import numpy as np
|
|
17
|
+
|
|
18
|
+
from . import __version__
|
|
19
|
+
from .config import load_config
|
|
20
|
+
from .inference import load_model, predict
|
|
21
|
+
from .train import train
|
|
22
|
+
from .utils import get_logger, resolve_checkpoint_path
|
|
23
|
+
|
|
24
|
+
_LOG = get_logger("kanx.cli")
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
# ---------------------------------------------------------------------------
|
|
28
|
+
def _load_xy(path: str) -> Tuple[np.ndarray, np.ndarray]:
|
|
29
|
+
"""Load (X, y) from either a .npz file with keys X/y, or a CSV where the
|
|
30
|
+
last column is the target. Used by `kanx train`."""
|
|
31
|
+
if path.endswith(".npz"):
|
|
32
|
+
data = np.load(path)
|
|
33
|
+
return data["X"].astype(np.float32), data["y"].astype(np.float32)
|
|
34
|
+
if path.endswith(".csv"):
|
|
35
|
+
arr = np.genfromtxt(path, delimiter=",", dtype=np.float32, skip_header=1)
|
|
36
|
+
if arr.ndim != 2 or arr.shape[1] < 2:
|
|
37
|
+
raise ValueError(f"CSV at {path} must have >= 2 columns")
|
|
38
|
+
return arr[:, :-1], arr[:, -1:]
|
|
39
|
+
raise ValueError(f"Unsupported dataset format: {path} (use .npz or .csv)")
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _load_x(path: str) -> np.ndarray:
|
|
43
|
+
if path.endswith(".json"):
|
|
44
|
+
with open(path, "r") as f:
|
|
45
|
+
return np.asarray(json.load(f), dtype=np.float32)
|
|
46
|
+
if path.endswith(".npz"):
|
|
47
|
+
return np.load(path)["X"].astype(np.float32)
|
|
48
|
+
if path.endswith(".csv"):
|
|
49
|
+
return np.genfromtxt(path, delimiter=",", dtype=np.float32, skip_header=1)
|
|
50
|
+
raise ValueError(f"Unsupported input format: {path}")
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
# ---------------------------------------------------------------------------
|
|
54
|
+
def main(argv=None) -> int:
|
|
55
|
+
parser = argparse.ArgumentParser(prog="kanx")
|
|
56
|
+
sub = parser.add_subparsers(dest="cmd")
|
|
57
|
+
|
|
58
|
+
p_train = sub.add_parser("train", help="train a KAN model")
|
|
59
|
+
p_train.add_argument("--config", default="configs/default.yaml")
|
|
60
|
+
p_train.add_argument(
|
|
61
|
+
"--data", required=False,
|
|
62
|
+
help="Path to dataset (.npz with X/y or .csv with y in last column). "
|
|
63
|
+
"If omitted a small synthetic regression dataset is used.",
|
|
64
|
+
)
|
|
65
|
+
p_train.add_argument("--verbose", type=int, default=1)
|
|
66
|
+
|
|
67
|
+
p_pred = sub.add_parser("predict", help="run inference on a checkpoint")
|
|
68
|
+
p_pred.add_argument("--checkpoint", required=True)
|
|
69
|
+
p_pred.add_argument("--input", required=True)
|
|
70
|
+
|
|
71
|
+
sub.add_parser("info", help="print version / runtime info")
|
|
72
|
+
|
|
73
|
+
# default subcommand → train
|
|
74
|
+
if argv is None:
|
|
75
|
+
argv = sys.argv[1:]
|
|
76
|
+
if not argv or argv[0].startswith("--"):
|
|
77
|
+
argv = ["train"] + argv
|
|
78
|
+
|
|
79
|
+
args = parser.parse_args(argv)
|
|
80
|
+
|
|
81
|
+
if args.cmd == "info":
|
|
82
|
+
import tensorflow as tf # noqa
|
|
83
|
+
print(f"kanx {__version__}")
|
|
84
|
+
print(f"TensorFlow {tf.__version__}")
|
|
85
|
+
print(f"GPUs available: {len(tf.config.list_physical_devices('GPU'))}")
|
|
86
|
+
return 0
|
|
87
|
+
|
|
88
|
+
if args.cmd == "train":
|
|
89
|
+
cfg = load_config(args.config)
|
|
90
|
+
if args.data:
|
|
91
|
+
X, y = _load_xy(args.data)
|
|
92
|
+
else:
|
|
93
|
+
# Synthetic regression: target = sin(pi*x1) + x2^2
|
|
94
|
+
rng = np.random.default_rng(cfg.training.seed)
|
|
95
|
+
X = rng.uniform(-1, 1, size=(512, cfg.model.layers[0])).astype(np.float32)
|
|
96
|
+
y = (np.sin(np.pi * X[:, :1]) + X[:, 1:2] ** 2).astype(np.float32)
|
|
97
|
+
if cfg.model.layers[-1] != 1:
|
|
98
|
+
# Broadcast to output dim by repeating the column.
|
|
99
|
+
y = np.tile(y, (1, cfg.model.layers[-1]))
|
|
100
|
+
_LOG.info("Loaded dataset: X=%s y=%s", X.shape, y.shape)
|
|
101
|
+
train(cfg, X, y, verbose=args.verbose)
|
|
102
|
+
ckpt = resolve_checkpoint_path(cfg.checkpoint.dir, cfg.checkpoint.filename)
|
|
103
|
+
_LOG.info("Done. Checkpoint: %s", ckpt)
|
|
104
|
+
return 0
|
|
105
|
+
|
|
106
|
+
if args.cmd == "predict":
|
|
107
|
+
if not os.path.exists(args.checkpoint):
|
|
108
|
+
_LOG.error("Checkpoint does not exist: %s", args.checkpoint)
|
|
109
|
+
return 2
|
|
110
|
+
model = load_model(args.checkpoint)
|
|
111
|
+
X = _load_x(args.input)
|
|
112
|
+
out = predict(model, X)
|
|
113
|
+
print(json.dumps({"shape": list(out.shape), "output": out.tolist()}))
|
|
114
|
+
return 0
|
|
115
|
+
|
|
116
|
+
parser.print_help()
|
|
117
|
+
return 1
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
if __name__ == "__main__":
|
|
121
|
+
raise SystemExit(main())
|
kanx/config.py
ADDED
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
"""Configuration loader and schema validator for kanx.
|
|
2
|
+
|
|
3
|
+
We deliberately avoid pulling in heavyweight schema libraries (pydantic) —
|
|
4
|
+
configs are simple nested dicts validated by a hand-written `validate_config`
|
|
5
|
+
function. This keeps the dependency footprint tiny and the failure modes
|
|
6
|
+
loud and obvious (a single ValueError with a precise message).
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import os
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
from typing import Any, Dict, List, Tuple
|
|
13
|
+
|
|
14
|
+
import yaml
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass
|
|
18
|
+
class ModelConfig:
|
|
19
|
+
layers: List[int]
|
|
20
|
+
grid_size: int = 5
|
|
21
|
+
spline_order: int = 3
|
|
22
|
+
base_activation: str = "silu"
|
|
23
|
+
grid_range: Tuple[float, float] = (-1.0, 1.0)
|
|
24
|
+
regularization_factor: float = 0.0
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@dataclass
|
|
28
|
+
class TrainingConfig:
|
|
29
|
+
lr: float = 1e-3
|
|
30
|
+
epochs: int = 50
|
|
31
|
+
batch_size: int = 32
|
|
32
|
+
loss: str = "mse"
|
|
33
|
+
optimizer: str = "adam"
|
|
34
|
+
val_split: float = 0.0
|
|
35
|
+
early_stopping_patience: int = 0 # 0 = disabled
|
|
36
|
+
seed: int = 42
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@dataclass
|
|
40
|
+
class CheckpointConfig:
|
|
41
|
+
dir: str = "checkpoints"
|
|
42
|
+
filename: str = "kanx_model.keras"
|
|
43
|
+
save_best_only: bool = True
|
|
44
|
+
monitor: str = "loss"
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@dataclass
|
|
48
|
+
class KanxConfig:
|
|
49
|
+
model: ModelConfig
|
|
50
|
+
training: TrainingConfig = field(default_factory=TrainingConfig)
|
|
51
|
+
checkpoint: CheckpointConfig = field(default_factory=CheckpointConfig)
|
|
52
|
+
raw: Dict[str, Any] = field(default_factory=dict)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
# ----------------------------------------------------------------------------
|
|
56
|
+
def load_config(path: str) -> KanxConfig:
|
|
57
|
+
"""Load a YAML config from disk and validate it.
|
|
58
|
+
|
|
59
|
+
Raises:
|
|
60
|
+
FileNotFoundError if `path` does not exist.
|
|
61
|
+
ValueError on invalid schema / values.
|
|
62
|
+
"""
|
|
63
|
+
if not os.path.exists(path):
|
|
64
|
+
raise FileNotFoundError(f"Config file not found: {path}")
|
|
65
|
+
with open(path, "r") as f:
|
|
66
|
+
raw = yaml.safe_load(f) or {}
|
|
67
|
+
return validate_config(raw)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def validate_config(raw: Dict[str, Any]) -> KanxConfig:
|
|
71
|
+
if not isinstance(raw, dict):
|
|
72
|
+
raise ValueError("Config root must be a mapping")
|
|
73
|
+
|
|
74
|
+
# ---- model ------------------------------------------------------------
|
|
75
|
+
if "model" not in raw or not isinstance(raw["model"], dict):
|
|
76
|
+
raise ValueError("Config must contain a 'model' mapping")
|
|
77
|
+
m = raw["model"]
|
|
78
|
+
if "layers" not in m or not isinstance(m["layers"], list) or len(m["layers"]) < 2:
|
|
79
|
+
raise ValueError("model.layers must be a list of >= 2 positive integers")
|
|
80
|
+
for w in m["layers"]:
|
|
81
|
+
if not isinstance(w, int) or w <= 0:
|
|
82
|
+
raise ValueError(f"model.layers entries must be positive ints, got {w!r}")
|
|
83
|
+
|
|
84
|
+
grid_range = tuple(m.get("grid_range", (-1.0, 1.0)))
|
|
85
|
+
if len(grid_range) != 2 or grid_range[0] >= grid_range[1]:
|
|
86
|
+
raise ValueError("model.grid_range must be [low, high] with low < high")
|
|
87
|
+
|
|
88
|
+
model_cfg = ModelConfig(
|
|
89
|
+
layers=list(m["layers"]),
|
|
90
|
+
grid_size=int(m.get("grid_size", 5)),
|
|
91
|
+
spline_order=int(m.get("spline_order", 3)),
|
|
92
|
+
base_activation=str(m.get("base_activation", "silu")),
|
|
93
|
+
grid_range=(float(grid_range[0]), float(grid_range[1])),
|
|
94
|
+
regularization_factor=float(m.get("regularization_factor", 0.0)),
|
|
95
|
+
)
|
|
96
|
+
if model_cfg.grid_size <= 0:
|
|
97
|
+
raise ValueError("model.grid_size must be > 0")
|
|
98
|
+
if model_cfg.spline_order < 1:
|
|
99
|
+
raise ValueError("model.spline_order must be >= 1")
|
|
100
|
+
|
|
101
|
+
# ---- training ---------------------------------------------------------
|
|
102
|
+
t = raw.get("training", {}) or {}
|
|
103
|
+
train_cfg = TrainingConfig(
|
|
104
|
+
lr=float(t.get("lr", 1e-3)),
|
|
105
|
+
epochs=int(t.get("epochs", 50)),
|
|
106
|
+
batch_size=int(t.get("batch_size", 32)),
|
|
107
|
+
loss=str(t.get("loss", "mse")),
|
|
108
|
+
optimizer=str(t.get("optimizer", "adam")),
|
|
109
|
+
val_split=float(t.get("val_split", 0.0)),
|
|
110
|
+
early_stopping_patience=int(t.get("early_stopping_patience", 0)),
|
|
111
|
+
seed=int(t.get("seed", 42)),
|
|
112
|
+
)
|
|
113
|
+
if train_cfg.lr <= 0:
|
|
114
|
+
raise ValueError("training.lr must be > 0")
|
|
115
|
+
if train_cfg.epochs <= 0:
|
|
116
|
+
raise ValueError("training.epochs must be > 0")
|
|
117
|
+
if train_cfg.batch_size <= 0:
|
|
118
|
+
raise ValueError("training.batch_size must be > 0")
|
|
119
|
+
if not (0.0 <= train_cfg.val_split < 1.0):
|
|
120
|
+
raise ValueError("training.val_split must be in [0, 1)")
|
|
121
|
+
|
|
122
|
+
# ---- checkpoint -------------------------------------------------------
|
|
123
|
+
c = raw.get("checkpoint", {}) or {}
|
|
124
|
+
ckpt_cfg = CheckpointConfig(
|
|
125
|
+
dir=str(c.get("dir", "checkpoints")),
|
|
126
|
+
filename=str(c.get("filename", "kanx_model.keras")),
|
|
127
|
+
save_best_only=bool(c.get("save_best_only", True)),
|
|
128
|
+
monitor=str(c.get("monitor", "loss")),
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
return KanxConfig(
|
|
132
|
+
model=model_cfg,
|
|
133
|
+
training=train_cfg,
|
|
134
|
+
checkpoint=ckpt_cfg,
|
|
135
|
+
raw=raw,
|
|
136
|
+
)
|
kanx/inference.py
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
"""Inference utilities: checkpoint IO + batched `predict`.
|
|
2
|
+
|
|
3
|
+
The API layer (`api/app.py`) and the CLI both go through this module so that
|
|
4
|
+
the load / save semantics are consistent across surfaces.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import os
|
|
9
|
+
from typing import Optional, Union
|
|
10
|
+
|
|
11
|
+
import numpy as np
|
|
12
|
+
import tensorflow as tf
|
|
13
|
+
|
|
14
|
+
from .layers import KANLinear # noqa: F401 (registers custom object on import)
|
|
15
|
+
from .model import KAN # noqa: F401 (registers custom object on import)
|
|
16
|
+
from .utils import get_logger
|
|
17
|
+
|
|
18
|
+
_LOG = get_logger("kanx.inference")
|
|
19
|
+
|
|
20
|
+
ArrayLike = Union[np.ndarray, tf.Tensor, list]
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
# ---------------------------------------------------------------------------
|
|
24
|
+
def load_model(path: str) -> tf.keras.Model:
|
|
25
|
+
"""Load a kanx checkpoint (`.keras` archive)."""
|
|
26
|
+
if not os.path.exists(path):
|
|
27
|
+
raise FileNotFoundError(f"Checkpoint not found: {path}")
|
|
28
|
+
_LOG.info("Loading model from %s", path)
|
|
29
|
+
# Custom objects are auto-discovered via @register_keras_serializable.
|
|
30
|
+
return tf.keras.models.load_model(path, compile=False)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def save_model(model: tf.keras.Model, path: str) -> str:
|
|
34
|
+
"""Persist a model to disk and return the absolute path."""
|
|
35
|
+
os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
|
|
36
|
+
model.save(path)
|
|
37
|
+
abs_path = os.path.abspath(path)
|
|
38
|
+
_LOG.info("Saved model to %s", abs_path)
|
|
39
|
+
return abs_path
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _coerce(x: ArrayLike) -> tf.Tensor:
|
|
43
|
+
if isinstance(x, tf.Tensor):
|
|
44
|
+
t = x
|
|
45
|
+
else:
|
|
46
|
+
t = tf.convert_to_tensor(x, dtype=tf.float32)
|
|
47
|
+
if t.shape.rank == 1:
|
|
48
|
+
# Allow single-sample lists like [1.0, 2.0].
|
|
49
|
+
t = tf.expand_dims(t, 0)
|
|
50
|
+
if t.shape.rank != 2:
|
|
51
|
+
raise ValueError(
|
|
52
|
+
f"Inputs must be rank-1 (single sample) or rank-2 (batch); "
|
|
53
|
+
f"got shape {t.shape}"
|
|
54
|
+
)
|
|
55
|
+
return tf.cast(t, tf.float32)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def predict(
|
|
59
|
+
model: tf.keras.Model,
|
|
60
|
+
x: ArrayLike,
|
|
61
|
+
*,
|
|
62
|
+
batch_size: Optional[int] = None,
|
|
63
|
+
) -> np.ndarray:
|
|
64
|
+
"""Run inference. Returns a NumPy array of shape ``(batch, out_features)``.
|
|
65
|
+
|
|
66
|
+
Args:
|
|
67
|
+
model: a `tf.keras.Model` (typically a `KAN`).
|
|
68
|
+
x: inputs as list / np.ndarray / tf.Tensor (rank 1 or 2).
|
|
69
|
+
batch_size: optional batching for very large inputs. ``None`` => one pass.
|
|
70
|
+
"""
|
|
71
|
+
x = _coerce(x)
|
|
72
|
+
if batch_size is None or x.shape[0] <= batch_size:
|
|
73
|
+
return model(x, training=False).numpy()
|
|
74
|
+
|
|
75
|
+
out_chunks = []
|
|
76
|
+
n = int(x.shape[0])
|
|
77
|
+
for i in range(0, n, batch_size):
|
|
78
|
+
out_chunks.append(model(x[i : i + batch_size], training=False).numpy())
|
|
79
|
+
return np.concatenate(out_chunks, axis=0)
|
kanx/layers.py
ADDED
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
"""Spline + custom layers for kanx.
|
|
2
|
+
|
|
3
|
+
This module implements:
|
|
4
|
+
|
|
5
|
+
* `extend_grid` — pads a uniform grid by `spline_order` knots on each side
|
|
6
|
+
following the standard pykan convention so that the
|
|
7
|
+
number of B-spline basis functions of degree `k`
|
|
8
|
+
equals `grid_size + k`.
|
|
9
|
+
* `b_spline_basis` — vectorized Cox-de Boor recursion that evaluates the
|
|
10
|
+
full basis matrix for a batch of inputs (no per-feature
|
|
11
|
+
Python loop, no per-sample loop).
|
|
12
|
+
* `KANLinear` — drop-in `tf.keras.layers.Layer` implementing one KAN
|
|
13
|
+
edge bank: a SiLU-residual base + B-spline edge
|
|
14
|
+
activations (Liu et al., 2024, eq. 2.10).
|
|
15
|
+
|
|
16
|
+
Design notes
|
|
17
|
+
------------
|
|
18
|
+
* The grid is **per-input-feature** (shape `(in_features, grid_size + 1)`).
|
|
19
|
+
Per-feature grids are required for adaptive grid updates and match the
|
|
20
|
+
reference pykan implementation.
|
|
21
|
+
* `spline_order` is the polynomial *degree* (k = 3 → cubic B-splines).
|
|
22
|
+
* The base path applies `silu` to the input *before* multiplying by
|
|
23
|
+
`base_weight`, matching pykan's `base_activation` design.
|
|
24
|
+
* The layer does **not** apply a non-linearity to its output: the B-spline
|
|
25
|
+
edge function is itself a learned non-linearity.
|
|
26
|
+
"""
|
|
27
|
+
from __future__ import annotations
|
|
28
|
+
|
|
29
|
+
from typing import Tuple
|
|
30
|
+
|
|
31
|
+
import tensorflow as tf
|
|
32
|
+
from tensorflow.keras import initializers, regularizers
|
|
33
|
+
from tensorflow.keras.layers import Layer
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
# ----------------------------------------------------------------------------
|
|
37
|
+
# B-spline primitives
|
|
38
|
+
# ----------------------------------------------------------------------------
|
|
39
|
+
def extend_grid(grid: tf.Tensor, k: int) -> tf.Tensor:
|
|
40
|
+
"""Symmetrically extend a uniform per-feature grid by `k` knots on each side.
|
|
41
|
+
|
|
42
|
+
Args:
|
|
43
|
+
grid: float tensor of shape ``(in_features, grid_size + 1)``.
|
|
44
|
+
k: spline polynomial degree (number of knots to add on each side).
|
|
45
|
+
|
|
46
|
+
Returns:
|
|
47
|
+
Tensor of shape ``(in_features, grid_size + 1 + 2k)``.
|
|
48
|
+
"""
|
|
49
|
+
if k < 0:
|
|
50
|
+
raise ValueError(f"k must be non-negative, got {k}")
|
|
51
|
+
if k == 0:
|
|
52
|
+
return grid
|
|
53
|
+
|
|
54
|
+
dtype = grid.dtype
|
|
55
|
+
n = tf.cast(tf.shape(grid)[1] - 1, dtype)
|
|
56
|
+
# Uniform step size per feature (works for non-uniform grids too as long
|
|
57
|
+
# as the boundary spacing is representative).
|
|
58
|
+
h = (grid[:, -1:] - grid[:, :1]) / n
|
|
59
|
+
|
|
60
|
+
left_offsets = tf.cast(tf.range(-k, 0), dtype)[None, :]
|
|
61
|
+
right_offsets = tf.cast(tf.range(1, k + 1), dtype)[None, :]
|
|
62
|
+
|
|
63
|
+
left = grid[:, :1] + left_offsets * h
|
|
64
|
+
right = grid[:, -1:] + right_offsets * h
|
|
65
|
+
return tf.concat([left, grid, right], axis=1)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def b_spline_basis(x: tf.Tensor, grid_ext: tf.Tensor, k: int) -> tf.Tensor:
|
|
69
|
+
"""Evaluate the B-spline basis of degree ``k`` for a batch of inputs.
|
|
70
|
+
|
|
71
|
+
Uses the Cox-de Boor recursion in a fully vectorized fashion (no Python
|
|
72
|
+
loop over features or batch).
|
|
73
|
+
|
|
74
|
+
Args:
|
|
75
|
+
x: ``(batch, in_features)`` input tensor.
|
|
76
|
+
grid_ext: ``(in_features, G_ext)`` already-extended grid.
|
|
77
|
+
k: spline degree (k=3 → cubic).
|
|
78
|
+
|
|
79
|
+
Returns:
|
|
80
|
+
``(batch, in_features, G_ext - k - 1)`` basis tensor. For the standard
|
|
81
|
+
case ``G_ext = grid_size + 1 + 2k`` this equals
|
|
82
|
+
``(batch, in_features, grid_size + k)`` — i.e. ``grid_size + k`` basis
|
|
83
|
+
functions per feature, matching the pykan convention.
|
|
84
|
+
"""
|
|
85
|
+
if k < 0:
|
|
86
|
+
raise ValueError(f"k must be non-negative, got {k}")
|
|
87
|
+
|
|
88
|
+
x_e = tf.expand_dims(x, -1) # (B, F, 1)
|
|
89
|
+
g_e = tf.expand_dims(grid_ext, 0) # (1, F, G_ext)
|
|
90
|
+
|
|
91
|
+
# Order-0 basis (piecewise constant).
|
|
92
|
+
basis = tf.cast(
|
|
93
|
+
(x_e >= g_e[..., :-1]) & (x_e < g_e[..., 1:]),
|
|
94
|
+
x.dtype,
|
|
95
|
+
) # (B, F, G_ext - 1)
|
|
96
|
+
|
|
97
|
+
for p in range(1, k + 1):
|
|
98
|
+
# After this iteration, basis length L_new = current_L - 1.
|
|
99
|
+
L = basis.shape[-1] - 1
|
|
100
|
+
g_l = g_e[..., :L] # g[i]
|
|
101
|
+
g_lp = g_e[..., p : p + L] # g[i+p]
|
|
102
|
+
g_r = g_e[..., 1 : L + 1] # g[i+1]
|
|
103
|
+
g_rp = g_e[..., p + 1 : p + 1 + L] # g[i+p+1]
|
|
104
|
+
|
|
105
|
+
denom_l = g_lp - g_l
|
|
106
|
+
denom_r = g_rp - g_r
|
|
107
|
+
# Stable division: replace 0 denominators with 1 (numerator is also 0).
|
|
108
|
+
denom_l = tf.where(tf.equal(denom_l, 0), tf.ones_like(denom_l), denom_l)
|
|
109
|
+
denom_r = tf.where(tf.equal(denom_r, 0), tf.ones_like(denom_r), denom_r)
|
|
110
|
+
|
|
111
|
+
left_w = (x_e - g_l) / denom_l
|
|
112
|
+
right_w = (g_rp - x_e) / denom_r
|
|
113
|
+
|
|
114
|
+
basis = left_w * basis[..., :L] + right_w * basis[..., 1 : L + 1]
|
|
115
|
+
|
|
116
|
+
return basis
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
# ----------------------------------------------------------------------------
|
|
120
|
+
# KAN edge bank
|
|
121
|
+
# ----------------------------------------------------------------------------
|
|
122
|
+
@tf.keras.utils.register_keras_serializable(package="kanx")
|
|
123
|
+
class KANLinear(Layer):
|
|
124
|
+
"""A single KAN layer (edge bank) with SiLU residual + B-spline edges.
|
|
125
|
+
|
|
126
|
+
Mathematically::
|
|
127
|
+
|
|
128
|
+
out(x) = silu(x) @ W_base + Σ_{i,j} spline_w[i, o, j] · B_j(x_i)
|
|
129
|
+
|
|
130
|
+
Args:
|
|
131
|
+
in_features: input dimension.
|
|
132
|
+
out_features: output dimension.
|
|
133
|
+
grid_size: number of grid intervals (default 5).
|
|
134
|
+
spline_order: spline polynomial degree (default 3).
|
|
135
|
+
base_activation: activation applied to the base path (default
|
|
136
|
+
``"silu"``).
|
|
137
|
+
grid_range: ``(low, high)`` of the initial uniform grid.
|
|
138
|
+
scale_noise: std of noise used to initialize spline weights.
|
|
139
|
+
scale_base: scale of the base-path Glorot init.
|
|
140
|
+
regularization_factor: L2 regularization strength (0 disables).
|
|
141
|
+
trainable_grid: whether the grid points are trainable.
|
|
142
|
+
"""
|
|
143
|
+
|
|
144
|
+
def __init__(
|
|
145
|
+
self,
|
|
146
|
+
in_features: int,
|
|
147
|
+
out_features: int,
|
|
148
|
+
grid_size: int = 5,
|
|
149
|
+
spline_order: int = 3,
|
|
150
|
+
base_activation: str = "silu",
|
|
151
|
+
grid_range: Tuple[float, float] = (-1.0, 1.0),
|
|
152
|
+
scale_noise: float = 0.1,
|
|
153
|
+
scale_base: float = 1.0,
|
|
154
|
+
regularization_factor: float = 0.0,
|
|
155
|
+
trainable_grid: bool = False,
|
|
156
|
+
**kwargs,
|
|
157
|
+
):
|
|
158
|
+
super().__init__(**kwargs)
|
|
159
|
+
if in_features <= 0 or out_features <= 0:
|
|
160
|
+
raise ValueError("in_features and out_features must be positive integers")
|
|
161
|
+
if grid_size <= 0:
|
|
162
|
+
raise ValueError("grid_size must be a positive integer")
|
|
163
|
+
if spline_order < 1:
|
|
164
|
+
raise ValueError("spline_order must be >= 1")
|
|
165
|
+
if grid_range[0] >= grid_range[1]:
|
|
166
|
+
raise ValueError("grid_range must be (low, high) with low < high")
|
|
167
|
+
|
|
168
|
+
self.in_features = int(in_features)
|
|
169
|
+
self.out_features = int(out_features)
|
|
170
|
+
self.grid_size = int(grid_size)
|
|
171
|
+
self.spline_order = int(spline_order)
|
|
172
|
+
self.base_activation_name = base_activation
|
|
173
|
+
self.base_activation = tf.keras.activations.get(base_activation)
|
|
174
|
+
self.grid_range = tuple(grid_range)
|
|
175
|
+
self.scale_noise = float(scale_noise)
|
|
176
|
+
self.scale_base = float(scale_base)
|
|
177
|
+
self.regularization_factor = float(regularization_factor)
|
|
178
|
+
self.trainable_grid = bool(trainable_grid)
|
|
179
|
+
|
|
180
|
+
# Number of B-spline basis functions of degree `spline_order`.
|
|
181
|
+
self.num_basis = self.grid_size + self.spline_order
|
|
182
|
+
|
|
183
|
+
# ---- Keras lifecycle ----------------------------------------------------
|
|
184
|
+
def build(self, input_shape):
|
|
185
|
+
if input_shape[-1] is None or int(input_shape[-1]) != self.in_features:
|
|
186
|
+
raise ValueError(
|
|
187
|
+
f"Last dim of input ({input_shape[-1]}) must equal "
|
|
188
|
+
f"in_features ({self.in_features})"
|
|
189
|
+
)
|
|
190
|
+
reg = (
|
|
191
|
+
regularizers.l2(self.regularization_factor)
|
|
192
|
+
if self.regularization_factor > 0
|
|
193
|
+
else None
|
|
194
|
+
)
|
|
195
|
+
|
|
196
|
+
self.base_weight = self.add_weight(
|
|
197
|
+
name="base_weight",
|
|
198
|
+
shape=(self.in_features, self.out_features),
|
|
199
|
+
initializer=initializers.GlorotUniform(),
|
|
200
|
+
regularizer=reg,
|
|
201
|
+
trainable=True,
|
|
202
|
+
)
|
|
203
|
+
# Small-noise init for spline weights so the layer starts close to
|
|
204
|
+
# the base path (this is critical for stable training of KANs).
|
|
205
|
+
self.spline_weight = self.add_weight(
|
|
206
|
+
name="spline_weight",
|
|
207
|
+
shape=(self.in_features, self.out_features, self.num_basis),
|
|
208
|
+
initializer=initializers.RandomNormal(stddev=self.scale_noise),
|
|
209
|
+
regularizer=reg,
|
|
210
|
+
trainable=True,
|
|
211
|
+
)
|
|
212
|
+
|
|
213
|
+
# Per-feature uniform grid of `grid_size + 1` knots.
|
|
214
|
+
import numpy as _np
|
|
215
|
+
low, high = self.grid_range
|
|
216
|
+
grid_init = _np.linspace(low, high, self.grid_size + 1, dtype=_np.float32)
|
|
217
|
+
grid_init = _np.tile(grid_init[None, :], (self.in_features, 1))
|
|
218
|
+
self.grid = self.add_weight(
|
|
219
|
+
name="grid",
|
|
220
|
+
shape=(self.in_features, self.grid_size + 1),
|
|
221
|
+
initializer=initializers.Constant(grid_init),
|
|
222
|
+
trainable=self.trainable_grid,
|
|
223
|
+
)
|
|
224
|
+
super().build(input_shape)
|
|
225
|
+
|
|
226
|
+
def call(self, inputs):
|
|
227
|
+
# Base residual path: silu(x) @ W_base → (B, out_features)
|
|
228
|
+
base_out = tf.matmul(self.base_activation(inputs), self.base_weight)
|
|
229
|
+
|
|
230
|
+
# Spline path
|
|
231
|
+
grid_ext = extend_grid(self.grid, self.spline_order)
|
|
232
|
+
basis = b_spline_basis(inputs, grid_ext, self.spline_order)
|
|
233
|
+
# basis: (B, F_in, num_basis)
|
|
234
|
+
# spline_weight:(F_in, F_out, num_basis)
|
|
235
|
+
# spline_out: (B, F_out)
|
|
236
|
+
spline_out = tf.einsum("bik,iok->bo", basis, self.spline_weight)
|
|
237
|
+
return base_out + spline_out
|
|
238
|
+
|
|
239
|
+
def compute_output_shape(self, input_shape):
|
|
240
|
+
return tuple(input_shape[:-1]) + (self.out_features,)
|
|
241
|
+
|
|
242
|
+
def get_config(self):
|
|
243
|
+
config = super().get_config()
|
|
244
|
+
config.update(
|
|
245
|
+
{
|
|
246
|
+
"in_features": self.in_features,
|
|
247
|
+
"out_features": self.out_features,
|
|
248
|
+
"grid_size": self.grid_size,
|
|
249
|
+
"spline_order": self.spline_order,
|
|
250
|
+
"base_activation": self.base_activation_name,
|
|
251
|
+
"grid_range": list(self.grid_range),
|
|
252
|
+
"scale_noise": self.scale_noise,
|
|
253
|
+
"scale_base": self.scale_base,
|
|
254
|
+
"regularization_factor": self.regularization_factor,
|
|
255
|
+
"trainable_grid": self.trainable_grid,
|
|
256
|
+
}
|
|
257
|
+
)
|
|
258
|
+
return config
|