oikan 0.0.1.1__tar.gz → 0.0.1.3__tar.gz

{oikan-0.0.1.1 → oikan-0.0.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: oikan
-Version: 0.0.1.1
+Version: 0.0.1.3
 Summary: OIKAN: Optimized Interpretable Kolmogorov-Arnold Networks
 Author: Arman Zhalgasbayev
 License: MIT

oikan-0.0.1.3/oikan/model.py

@@ -0,0 +1,65 @@
+import torch
+import torch.nn as nn
+from .utils import BSplineBasis, FourierBasis
+
+class AdaptiveBasisLayer(nn.Module):
+    def __init__(self, input_dim, hidden_dim):
+        super().__init__()
+        self.weights = nn.Parameter(torch.randn(input_dim, hidden_dim))
+        self.bias = nn.Parameter(torch.zeros(hidden_dim))
+    
+    def forward(self, x):
+        return torch.matmul(x, self.weights) + self.bias
+
+class EfficientKAN(nn.Module):
+    def __init__(self, input_dim, hidden_units=10, basis_type='bspline'):
+        super().__init__()
+        self.input_dim = input_dim
+        self.hidden_units = hidden_units
+        self.basis_type = basis_type
+        
+        if basis_type == 'bspline':
+            self.basis_functions = nn.ModuleList([BSplineBasis(hidden_units) for _ in range(input_dim)])
+            self.basis_output_dim = input_dim * (hidden_units - 4)  # Adjusted for BSpline output
+        elif basis_type == 'fourier':
+            self.basis_functions = nn.ModuleList([FourierBasis(hidden_units//2) for _ in range(input_dim)])
+            self.basis_output_dim = input_dim * hidden_units
+            
+        # Grid-based interaction layer
+        self.interaction_weights = nn.Parameter(torch.randn(input_dim, input_dim))
+    
+    def forward(self, x):
+        # Transform each feature using basis functions
+        transformed_features = [bf(x[:, i].unsqueeze(1)) for i, bf in enumerate(self.basis_functions)]
+        basis_output = torch.cat(transformed_features, dim=1)
+        
+        # Compute feature interactions - fixed matrix multiplication
+        batch_size = x.size(0)
+        x_reshaped = x.view(batch_size, self.input_dim, 1)  # [batch_size, input_dim, 1]
+        interaction_matrix = torch.sigmoid(self.interaction_weights)  # [input_dim, input_dim]
+        interaction_features = torch.bmm(x_reshaped.transpose(1, 2), 
+                                       x_reshaped * interaction_matrix.unsqueeze(0))  # [batch_size, 1, 1]
+        interaction_features = interaction_features.view(batch_size, -1)  # [batch_size, 1]
+        
+        return torch.cat([basis_output, interaction_features], dim=1)
+    
+    def get_output_dim(self):
+        return self.basis_output_dim + self.input_dim
+
+class OIKAN(nn.Module):
+    def __init__(self, input_dim, output_dim, hidden_units=10):
+        super().__init__()
+        self.efficientkan = EfficientKAN(input_dim, hidden_units)
+        
+        # Get actual feature dimension after transformation
+        feature_dim = self.efficientkan.get_output_dim()
+        
+        self.interpretable_layers = nn.Sequential(
+            AdaptiveBasisLayer(feature_dim, 32),
+            nn.ReLU(),
+            AdaptiveBasisLayer(32, output_dim)
+        )
+    
+    def forward(self, x):
+        transformed_x = self.efficientkan(x)
+        return self.interpretable_layers(transformed_x)

oikan-0.0.1.3/oikan/regularization.py

@@ -0,0 +1,30 @@
+import torch
+import torch.nn as nn
+
+class RegularizedLoss:
+    def __init__(self, base_criterion, model, l1_lambda=0.01, gradient_lambda=0.01):
+        self.base_criterion = base_criterion
+        self.model = model
+        self.l1_lambda = l1_lambda
+        self.gradient_lambda = gradient_lambda
+    
+    def __call__(self, pred, target, inputs):
+        base_loss = self.base_criterion(pred, target)
+        
+        # L1 regularization
+        l1_loss = 0
+        for param in self.model.parameters():
+            l1_loss += torch.norm(param, p=1)
+            
+        # Gradient penalty
+        grad_penalty = 0
+        inputs.requires_grad_(True)
+        outputs = self.model(inputs)
+        gradients = torch.autograd.grad(
+            outputs=outputs.sum(),
+            inputs=inputs,
+            create_graph=True
+        )[0]
+        grad_penalty = ((gradients.norm(2, dim=1) - 1) ** 2).mean()
+        
+        return base_loss + self.l1_lambda * l1_loss + self.gradient_lambda * grad_penalty

oikan-0.0.1.3/oikan/symbolic.py

@@ -0,0 +1,129 @@
+import torch
+import numpy as np
+import networkx as nx
+import matplotlib.pyplot as plt
+
+ADVANCED_LIB = {
+    'x':    lambda x: x,
+    'x^2':  lambda x: x**2,
+    'x^3':  lambda x: x**3,
+    'x^4':  lambda x: x**4,
+    'x^5':  lambda x: x**5,
+    'exp':  lambda x: np.exp(x),
+    'log':  lambda x: np.log(np.abs(x) + 1e-8),
+    'sqrt': lambda x: np.sqrt(np.abs(x)),
+    'tanh': lambda x: np.tanh(x),
+    'sin':  lambda x: np.sin(x),
+    'abs':  lambda x: np.abs(x)
+}
+
+# STEP-1: Helper functions
+def get_model_predictions(model, X, mode):
+    """Compute model predictions and return target values (and raw preds for classification)."""
+    X_tensor = torch.FloatTensor(X)
+    with torch.no_grad():
+        preds = model(X_tensor)
+    if mode == 'regression':
+        return preds.detach().cpu().numpy().flatten(), None
+    elif mode == 'classification':
+        out = preds.detach().cpu().numpy()
+        target = (out[:, 0] - out[:, 1]).flatten() if (out.ndim > 1 and out.shape[1] > 1) else out.flatten()
+        return target, out
+    else:
+        raise ValueError("Unknown mode")
+
+def build_design_matrix(X, return_names=False):
+    """Build the design matrix using the advanced nonlinear bases."""
+    X_np = np.array(X)
+    n_samples, d = X_np.shape
+    F_parts = [np.ones((n_samples, 1))]
+    names = ['1'] if return_names else None
+    for j in range(d):
+        xj = X_np[:, j:j+1]
+        for key, func in ADVANCED_LIB.items():
+            F_parts.append(func(xj))
+            if return_names:
+                names.append(f"{key}(x{j+1})")
+    return (np.hstack(F_parts), names) if return_names else np.hstack(F_parts)
+
+# STEP-2: Main functions using helpers
+def extract_symbolic_formula(model, X, mode='regression'):
+    """
+    Approximate a symbolic formula from the model using advanced nonlinear bases.
+    """
+    n_samples = np.array(X).shape[0]
+    y_target, _ = get_model_predictions(model, X, mode)
+    F, func_names = build_design_matrix(X, return_names=True)
+    beta, _, _, _ = np.linalg.lstsq(F, y_target, rcond=None)
+    terms = [f"({c:.2f}*{name})" for c, name in zip(beta, func_names) if abs(c) > 1e-4]
+    return " + ".join(terms)
+
+def test_symbolic_formula(model, X, mode='regression'):
+    """
+    Evaluate the extracted symbolic formula against model outputs.
+    """
+    n_samples = np.array(X).shape[0]
+    y_target, out = get_model_predictions(model, X, mode)
+    F = build_design_matrix(X, return_names=False)
+    beta, _, _, _ = np.linalg.lstsq(F, y_target, rcond=None)
+    symbolic_vals = F.dot(beta)
+    if mode == 'regression':
+        mse = np.mean((symbolic_vals - y_target) ** 2)
+        mae = np.mean(np.abs(symbolic_vals - y_target))
+        rmse = np.sqrt(mse)
+        print(f"(Advanced) MSE: {mse:.4f}, MAE: {mae:.4f}, RMSE: {rmse:.4f}")
+        return mse, mae, rmse
+    elif mode == 'classification':
+        sym_preds = np.where(symbolic_vals >= 0, 0, 1)
+        model_classes = np.argmax(out, axis=1) if (out.ndim > 1) else (out >= 0.5).astype(int)
+        if model_classes.shape[0] != sym_preds.shape[0]:
+            raise ValueError("Shape mismatch between symbolic and model predictions.")
+        accuracy = np.mean(sym_preds == model_classes)
+        print(f"(Advanced) Accuracy: {accuracy:.4f}")
+        return accuracy
+
+def plot_symbolic_formula(model, X, mode='regression'):
+    """
+    Plot a graph representation of the extracted symbolic formula.
+    """
+    formula = extract_symbolic_formula(model, X, mode)
+    G = nx.DiGraph()
+    G.add_node("Output")
+    terms = formula.split(" + ")
+    for term in terms:
+        expr = term.strip("()")
+        coeff_str, basis = expr.split("*", 1) if "*" in expr else (expr, "unknown")
+        node_label = f"{basis}\n({float(coeff_str):.2f})"
+        G.add_node(node_label)
+        G.add_edge(node_label, "Output", weight=float(coeff_str))
+    left_nodes = [n for n in G.nodes() if n != "Output"]
+    pos = {}
+    n_left = len(left_nodes)
+    for i, node in enumerate(sorted(left_nodes)):
+        pos[node] = (0, 1 - (i / max(n_left - 1, 1)))
+    pos["Output"] = (1, 0.5)
+    plt.figure(figsize=(12, 8))
+    nx.draw(G, pos, with_labels=True, node_color="skyblue", node_size=2500, font_size=10,
+            arrows=True, arrowstyle='->', arrowsize=20)
+    edge_labels = {(u, v): f"{d['weight']:.2f}" for u, v, d in G.edges(data=True)}
+    nx.draw_networkx_edge_labels(G, pos, edge_labels=edge_labels, font_color='red', font_size=10)
+    plt.title("OIKAN Symbolic Formula Graph")
+    plt.axis("off")
+    plt.show()
+
+def extract_latex_formula(model, X, mode='regression'):
+    """
+    Return the extracted symbolic formula as LaTeX code.
+    """
+    formula = extract_symbolic_formula(model, X, mode)
+    terms = formula.split(" + ")
+    latex_terms = []
+    for term in terms:
+        expr = term.strip("()")
+        coeff_str, basis = expr.split("*", 1) if "*" in expr else (expr, "")
+        coeff = float(coeff_str)
+        coeff_latex = f"{abs(coeff):.2f}".rstrip("0").rstrip(".")
+        term_latex = coeff_latex if basis.strip() == "1" else f"{coeff_latex} \\cdot {basis.strip()}"
+        latex_terms.append(f"- {term_latex}" if coeff < 0 else f"+ {term_latex}")
+    latex_formula = " ".join(latex_terms).lstrip("+ ").strip()
+    return f"$$ {latex_formula} $$"

oikan-0.0.1.3/oikan/trainer.py

@@ -0,0 +1,37 @@
+import torch
+import torch.nn as nn
+from .regularization import RegularizedLoss
+
+def train(model, train_data, epochs=100, lr=0.01):
+    X_train, y_train = train_data
+    optimizer = torch.optim.Adam(model.parameters(), lr=lr)
+    criterion = nn.MSELoss()
+    reg_loss = RegularizedLoss(criterion, model)
+    
+    model.train()
+    for epoch in range(epochs):
+        optimizer.zero_grad()
+        outputs = model(X_train)
+        loss = reg_loss(outputs, y_train, X_train)
+        loss.backward()
+        optimizer.step()
+        
+        if (epoch + 1) % 10 == 0:
+            print(f'Epoch [{epoch+1}/{epochs}], Loss: {loss.item():.4f}')
+
+def train_classification(model, train_data, epochs=100, lr=0.01):
+    X_train, y_train = train_data
+    optimizer = torch.optim.Adam(model.parameters(), lr=lr)
+    criterion = nn.CrossEntropyLoss()
+    reg_loss = RegularizedLoss(criterion, model)
+    
+    model.train()
+    for epoch in range(epochs):
+        optimizer.zero_grad()
+        outputs = model(X_train)
+        loss = reg_loss(outputs, y_train, X_train)
+        loss.backward()
+        optimizer.step()
+        
+        if (epoch + 1) % 10 == 0:
+            print(f'Epoch [{epoch+1}/{epochs}], Loss: {loss.item():.4f}')

oikan-0.0.1.3/oikan/utils.py

@@ -0,0 +1,43 @@
+import torch
+import torch.nn as nn
+import numpy as np
+from scipy.interpolate import BSpline
+
+class BSplineBasis(nn.Module):
+    def __init__(self, num_knots=10, degree=3):
+        super().__init__()
+        self.num_knots = max(num_knots, degree + 5)  # Ensure minimum number of knots
+        self.degree = degree
+        
+        # Create knot vector with proper padding
+        inner_knots = np.linspace(0, 1, self.num_knots - 2 * degree)
+        left_pad = np.zeros(degree)
+        right_pad = np.ones(degree)
+        knots = np.concatenate([left_pad, inner_knots, right_pad])
+        
+        self.register_buffer('knots', torch.FloatTensor(knots))
+        
+    def forward(self, x):
+        x_np = x.detach().cpu().numpy()
+        basis_values = np.zeros((x_np.shape[0], self.num_knots - self.degree - 1))
+        
+        # Normalize input to [0,1] range
+        x_normalized = (x_np - x_np.min()) / (x_np.max() - x_np.min() + 1e-8)
+        
+        for i in range(self.num_knots - self.degree - 1):
+            spl = BSpline.basis_element(self.knots[i:i+self.degree+2])
+            basis_values[:, i] = spl(x_normalized.squeeze())
+        
+        # Replace NaN values with 0
+        basis_values = np.nan_to_num(basis_values, 0)
+        return torch.FloatTensor(basis_values).to(x.device)
+
+class FourierBasis(nn.Module):
+    def __init__(self, num_frequencies=5):
+        super().__init__()
+        self.num_frequencies = num_frequencies
+        
+    def forward(self, x):
+        frequencies = torch.arange(1, self.num_frequencies + 1, device=x.device).float()
+        x_expanded = x * frequencies.view(1, -1) * 2 * np.pi
+        return torch.cat([torch.sin(x_expanded), torch.cos(x_expanded)], dim=1)

oikan-0.0.1.3/oikan/visualize.py

@@ -0,0 +1,37 @@
+import numpy as np
+import matplotlib.pyplot as plt
+import torch
+
+def visualize_regression(model, X, y):
+    model.eval()
+    with torch.no_grad():
+        X_tensor = torch.FloatTensor(X)
+        y_pred = model(X_tensor).numpy()
+    
+    plt.figure(figsize=(10, 6))
+    plt.scatter(X[:, 0], y, color='blue', label='True')
+    plt.scatter(X[:, 0], y_pred, color='red', label='Predicted')
+    plt.legend()
+    plt.show()
+
+def visualize_classification(model, X, y):
+    model.eval()
+    
+    # Create a mesh grid
+    x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
+    y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
+    xx, yy = np.meshgrid(np.linspace(x_min, x_max, 100),
+                        np.linspace(y_min, y_max, 100))
+    
+    # Make predictions
+    with torch.no_grad():
+        X_grid = torch.FloatTensor(np.c_[xx.ravel(), yy.ravel()])
+        Z = model(X_grid)
+        Z = torch.argmax(Z, dim=1).numpy()
+        Z = Z.reshape(xx.shape)
+    
+    # Plot
+    plt.figure(figsize=(10, 8))
+    plt.contourf(xx, yy, Z, alpha=0.4)
+    plt.scatter(X[:, 0], X[:, 1], c=y, alpha=0.8)
+    plt.show()

{oikan-0.0.1.1 → oikan-0.0.1.3}/oikan.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: oikan
-Version: 0.0.1.1
+Version: 0.0.1.3
 Summary: OIKAN: Optimized Interpretable Kolmogorov-Arnold Networks
 Author: Arman Zhalgasbayev
 License: MIT

{oikan-0.0.1.1 → oikan-0.0.1.3}/oikan.egg-info/SOURCES.txt

@@ -3,8 +3,10 @@ pyproject.toml
 setup.py
 oikan/__init__.py
 oikan/model.py
+oikan/regularization.py
 oikan/symbolic.py
 oikan/trainer.py
+oikan/utils.py
 oikan/visualize.py
 oikan.egg-info/PKG-INFO
 oikan.egg-info/SOURCES.txt

{oikan-0.0.1.1 → oikan-0.0.1.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "oikan"
-version = "0.0.1.1"
+version = "0.0.1.3"
 description = "OIKAN: Optimized Interpretable Kolmogorov-Arnold Networks"
 authors = [{name = "Arman Zhalgasbayev"}]
 dependencies = [

oikan-0.0.1.1/oikan/model.py

@@ -1,28 +0,0 @@
-import torch
-import torch.nn as nn
-
-# EfficientKAN Layer
-class EfficientKAN(nn.Module):
-    def __init__(self, input_dim, hidden_units=10):
-        super(EfficientKAN, self).__init__()
-        self.basis_functions = nn.ModuleList([nn.Linear(1, hidden_units) for _ in range(input_dim)])
-        self.activations = nn.ReLU()
-    
-    def forward(self, x):
-        transformed_features = [self.activations(bf(x[:, i].unsqueeze(1))) for i, bf in enumerate(self.basis_functions)]
-        return torch.cat(transformed_features, dim=1)
-
-# OIKAN Model
-class OIKAN(nn.Module):
-    def __init__(self, input_dim, output_dim, hidden_units=10):
-        super(OIKAN, self).__init__()
-        self.efficientkan = EfficientKAN(input_dim, hidden_units)
-        self.mlp = nn.Sequential(
-            nn.Linear(input_dim * hidden_units, 32),
-            nn.ReLU(),
-            nn.Linear(32, output_dim)
-        )
-    
-    def forward(self, x):
-        transformed_x = self.efficientkan(x)
-        return self.mlp(transformed_x)

oikan-0.0.1.1/oikan/symbolic.py

@@ -1,36 +0,0 @@
-import torch
-from sympy import symbols, simplify, Add
-
-# Regression symbolic extraction
-def extract_symbolic_formula_regression(model, input_data):
-    symbolic_vars = symbols([f'x{i}' for i in range(input_data.shape[1])])
-    
-    with torch.no_grad():
-        weights = model.mlp[0].weight.cpu().numpy()
-        if weights.size == 0:
-            print("Warning: Extracted weights are empty.")
-            return "NaN"
-
-    formula = sum(weights[0, i] * symbolic_vars[i] for i in range(len(symbolic_vars)))
-    return simplify(formula)
-
-# Classification symbolic extraction
-def extract_symbolic_formula_classification(model, input_data):
-    """
-    Extracts a symbolic decision boundary for a two-class classifier.
-    Approximates:
-      decision = (w[0] - w[1]) · x + (b[0] - b[1])
-    where w and b are from the model's final linear layer.
-    """
-    symbolic_vars = symbols([f'x{i}' for i in range(input_data.shape[1])])
-    with torch.no_grad():
-        final_layer = model.mlp[-1]
-        w = final_layer.weight.cpu().numpy()
-        b = final_layer.bias.cpu().numpy()
-        if w.shape[0] < 2:
-            print("Classification symbolic extraction requires at least 2 classes.")
-            return "NaN"
-        w_diff = w[0] - w[1]
-        b_diff = b[0] - b[1]
-    formula = sum(w_diff[i] * symbolic_vars[i] for i in range(len(symbolic_vars))) + b_diff
-    return simplify(formula)

oikan-0.0.1.1/oikan/trainer.py

@@ -1,32 +0,0 @@
-import torch.optim as optim
-import torch.nn as nn
-
-# Regression training
-def train(model, train_loader, epochs=100, lr=0.01):
-    criterion = nn.MSELoss()
-    optimizer = optim.LBFGS(model.parameters(), lr=lr)
-
-    def closure():
-        optimizer.zero_grad()
-        outputs = model(train_loader[0])
-        loss = criterion(outputs, train_loader[1])
-        loss.backward()
-        print(f"Loss: {loss.item()}")
-        return loss
-
-    for epoch in range(epochs):
-        optimizer.step(closure)
-        print(f"Epoch {epoch+1}/{epochs}")
-
-# Classification training
-def train_classification(model, train_loader, epochs=100, lr=0.01):
-    criterion = nn.CrossEntropyLoss()
-    optimizer = optim.Adam(model.parameters(), lr=lr)
-    
-    for epoch in range(epochs):
-        optimizer.zero_grad()
-        outputs = model(train_loader[0])
-        loss = criterion(outputs, train_loader[1])
-        loss.backward()
-        optimizer.step()
-        print(f"Epoch {epoch+1}/{epochs}, Loss: {loss.item()}")

oikan-0.0.1.1/oikan/visualize.py

@@ -1,20 +0,0 @@
-import matplotlib.pyplot as plt
-import torch
-
-# Regression Visualization Function
-def visualize_regression(model, X, y):
-    with torch.no_grad():
-        y_pred = model(torch.tensor(X, dtype=torch.float32)).numpy()
-    plt.scatter(X[:, 0], y, label='True Data')
-    plt.scatter(X[:, 0], y_pred, label='OIKAN Predictions', color='r')
-    plt.legend()
-    plt.show()
-
-# Classification visualization
-def visualize_classification(model, X, y):
-    with torch.no_grad():
-        outputs = model(torch.tensor(X, dtype=torch.float32))
-        preds = torch.argmax(outputs, dim=1).numpy()
-    plt.scatter(X[:, 0], X[:, 1], c=preds, cmap='viridis', edgecolor='k')
-    plt.title("Classification Results")
-    plt.show()