Enilnets 1.0.0__tar.gz

enilnets-1.0.0/Enilnets/__init__.py

@@ -0,0 +1,33 @@
+from .base import NeuralNet
+from .layers import add_dense, add_sparse, add_conv2d, add_flatten, add_maxpool2d, add_avgpool2d, add_batchnorm, add_dropout
+from .forward import Forward
+from .backward import Backward
+from .optimizer import update
+from .train import TrainBatch, Train, compute_accuracy
+from .reinforce import Reinforce
+from .loss import ComputeLoss
+from .io import Save, Load
+
+print("Loading Modules....")
+
+NeuralNet.add_dense = add_dense
+NeuralNet.add_sparse = add_sparse
+NeuralNet.add_conv2d = add_conv2d
+NeuralNet.add_flatten = add_flatten
+NeuralNet.add_maxpool2d = add_maxpool2d
+NeuralNet.add_avgpool2d = add_avgpool2d
+NeuralNet.add_batchnorm = add_batchnorm
+NeuralNet.add_dropout = add_dropout
+NeuralNet.Forward = Forward
+NeuralNet.predict = Forward
+NeuralNet.Backward = Backward
+NeuralNet.update = update
+NeuralNet.TrainBatch = TrainBatch
+NeuralNet.Train = Train
+NeuralNet.compute_accuracy = compute_accuracy
+NeuralNet.Reinforce = Reinforce
+NeuralNet.ComputeLoss = ComputeLoss
+NeuralNet.Save = Save
+NeuralNet.Load = Load
+
+print("Enilnets Library Ready!")

enilnets-1.0.0/Enilnets/activations.py

@@ -0,0 +1,39 @@
+import numpy as np
+
+def activate(name, x):
+    if name == "relu": return np.maximum(0, x)
+    if name == "leakyrelu": return np.where(x > 0, x, 0.01 * x)
+    if name == "elu": return np.where(x > 0, x, np.exp(x) - 1)
+    if name == "selu":
+        alpha = 1.6732632423543772848170429916717
+        scale = 1.0507009873554804934193349852946
+        return scale * np.where(x > 0, x, alpha * (np.exp(x) - 1))
+    if name == "gelu": return 0.5 * x * (1 + np.tanh(np.sqrt(2 / np.pi) * (x + 0.044715 * x**3)))
+    if name == "swish": return x * 1.0 / (1.0 + np.exp(-x))
+    if name == "sigmoid": return 1.0 / (1.0 + np.exp(-np.clip(x, -500, 500)))
+    if name == "tanh": return np.tanh(x)
+    if name == "softmax":
+        e_x = np.exp(x - np.max(x, axis=-1, keepdims=True))
+        return e_x / np.sum(e_x, axis=-1, keepdims=True)
+    return x
+
+def derivative(name, x):
+    if name == "relu": return (x > 0).astype(np.float64)
+    if name == "leakyrelu": return np.where(x > 0, 1.0, 0.01)
+    if name == "elu": return np.where(x > 0, 1.0, np.exp(x))
+    if name == "selu":
+        alpha = 1.6732632423543772848170429916717
+        scale = 1.0507009873554804934193349852946
+        return scale * np.where(x > 0, 1.0, alpha * np.exp(x))
+    if name == "gelu":
+        cdf = 0.5 * (1 + np.tanh(np.sqrt(2 / np.pi) * (x + 0.044715 * x**3)))
+        pdf = np.exp(-0.5 * x**2) / np.sqrt(2 * np.pi)
+        return cdf + x * pdf
+    if name == "swish":
+        s = 1.0 / (1.0 + np.exp(-x))
+        return s + x * s * (1 - s)
+    if name == "sigmoid":
+        s = 1.0 / (1.0 + np.exp(-np.clip(x, -500, 500)))
+        return s * (1 - s)
+    if name == "tanh": return 1 - np.tanh(x) ** 2
+    return np.ones_like(x)

enilnets-1.0.0/Enilnets/backward.py

@@ -0,0 +1,121 @@
+import numpy as np
+from .activations import derivative
+from .forward import im2col
+
+def maxpool2d_backward(delta, x, p):
+    B, C, H, W = x.shape
+    H_trim = (H // p) * p
+    W_trim = (W // p) * p
+    x_trim = x[:, :, :H_trim, :W_trim]
+
+    H_b, W_b = H_trim // p, W_trim // p
+    strides = x_trim.strides
+    new_shape = (B, C, H_b, p, W_b, p)
+    new_strides = (strides[0], strides[1], strides[2]*p, strides[2], strides[3]*p, strides[3])
+    x_blocks = np.lib.stride_tricks.as_strided(x_trim, shape=new_shape, strides=new_strides)
+
+    x_max = x_blocks.max(axis=(3, 5), keepdims=True)
+    mask = (x_blocks == x_max).astype(np.float64)
+    mask_sum = mask.sum(axis=(3, 5), keepdims=True)
+    mask = mask / np.maximum(mask_sum, 1e-12)
+
+    delta_expanded = delta[:, :, :H_b, :W_b][:, :, :, None, :, None]
+
+    dx = np.zeros_like(x)
+    dx_view = np.lib.stride_tricks.as_strided(dx[:, :, :H_trim, :W_trim], 
+                                                shape=new_shape, strides=new_strides)
+    dx_view[:] = mask * delta_expanded
+    return dx
+
+def avgpool2d_backward(delta, x, p):
+    B, C, H, W = x.shape
+    H_trim = (H // p) * p
+    W_trim = (W // p) * p
+    H_b, W_b = H_trim // p, W_trim // p
+
+    dx = np.zeros_like(x)
+    strides = dx[:, :, :H_trim, :W_trim].strides
+    new_shape = (B, C, H_b, p, W_b, p)
+    new_strides = (strides[0], strides[1], strides[2]*p, strides[2], strides[3]*p, strides[3])
+    dx_view = np.lib.stride_tricks.as_strided(dx[:, :, :H_trim, :W_trim], 
+                                               shape=new_shape, strides=new_strides)
+    dx_view[:] = delta[:, :, :H_b, :W_b][:, :, :, None, :, None] / (p * p)
+    return dx
+
+def batchnorm_backward(dout, cache):
+    x, x_norm, mean, var, gamma, epsilon = cache
+    N = x.shape[0]
+    dbeta = np.sum(dout, axis=0)
+    dgamma = np.sum(dout * x_norm, axis=0)
+    dx_norm = dout * gamma
+    dvar = np.sum(dx_norm * (x - mean) * -0.5 * (var + epsilon) ** (-1.5), axis=0)
+    dmean = np.sum(dx_norm * -1.0 / np.sqrt(var + epsilon), axis=0)
+    dx = dx_norm / np.sqrt(var + epsilon) + dvar * 2.0 * (x - mean) / N + dmean / N
+    return dx, dgamma, dbeta
+
+def conv2d_backward_input(delta, weights, input_shape):
+    B, F, out_h, out_w = delta.shape
+    F, C, K, _ = weights.shape
+    H, W = input_shape[2], input_shape[3]
+
+    padded_delta = np.pad(delta, [(0, 0), (0, 0), (K - 1, K - 1), (K - 1, K - 1)], mode="constant")
+    col = im2col(padded_delta, K, K)
+    weights_flat = weights[:, :, ::-1, ::-1].transpose(1, 0, 2, 3).reshape(C, -1)
+    grad = np.dot(col, weights_flat.T)
+    grad = grad.reshape(B, H, W, C).transpose(0, 3, 1, 2)
+    return grad
+
+def Backward(self, targets):
+    targets = np.asarray(targets, dtype=np.float64)
+    if targets.ndim == 1:
+        targets = targets.reshape(1, -1)
+    batch_size = targets.shape[0]
+    self.deltas = [None] * len(self.layers)
+    out = self.outputs[-1]
+    last = self.layers[-1]
+    if last.get("activation") == "softmax":
+        delta = (out - targets) / batch_size
+    else:
+        activation_input = self.pre_activations[-1] if self.pre_activations[-1] is not None else out
+        delta = (out - targets) * derivative(last.get("activation", "linear"), activation_input) / batch_size
+    self.deltas[-1] = delta
+
+    for l in reversed(range(len(self.layers) - 1)):
+        curr = self.layers[l]
+        nxt = self.layers[l + 1]
+        next_delta = self.deltas[l + 1]
+
+        if nxt["type"] in ("dense", "sparse"):
+            err = np.dot(next_delta, nxt["weights"])
+        elif nxt["type"] == "flatten":
+            err = next_delta.reshape(self.outputs[l + 1].shape)
+        elif nxt["type"] == "conv2d":
+            err = conv2d_backward_input(next_delta, nxt["weights"], self.outputs[l + 1].shape)
+        elif nxt["type"] == "maxpool2d":
+            err = maxpool2d_backward(next_delta, self.outputs[l + 1], nxt["p"])
+        elif nxt["type"] == "avgpool2d":
+            err = avgpool2d_backward(next_delta, self.outputs[l + 1], nxt["p"])
+        elif nxt["type"] == "dropout":
+            mask = nxt.get("mask")
+            rate = nxt.get("rate", 0.0)
+            if mask is None or rate == 0.0:
+                err = next_delta
+            else:
+                err = next_delta * mask / (1.0 - rate)
+        elif nxt["type"] == "batchnorm":
+            flat = next_delta.reshape(self.outputs[l + 1].shape[0], -1)
+            cache = self.batchnorm_cache[l + 1]
+            if cache is None:
+                raise ValueError("BatchNorm cache is None. Ensure Forward(training=True) was called before Backward.")
+            err_flat, dgamma, dbeta = batchnorm_backward(flat, cache)
+            nxt["d_gamma"] = dgamma
+            nxt["d_beta"] = dbeta
+            err = err_flat.reshape(self.outputs[l + 1].shape)
+        else:
+            err = np.zeros_like(self.outputs[l + 1])
+
+        if curr["type"] in ("dense", "sparse", "conv2d"):
+            activation_input = self.pre_activations[l+1] if self.pre_activations[l+1] is not None else self.outputs[l + 1]
+            self.deltas[l] = err * derivative(curr.get("activation", "linear"), activation_input)
+        else:
+            self.deltas[l] = err

enilnets-1.0.0/Enilnets/base.py

@@ -0,0 +1,41 @@
+import numpy as np
+
+class NeuralNet:
+    def __init__(self, learning_rate=0.001, optimizer="adam", l2_lambda=0.01, momentum=0.9):
+        self.layers = []
+        self.learning_rate = learning_rate
+        self.optimizer_type = optimizer.lower()
+        self.l2_lambda = l2_lambda
+        self.momentum = momentum
+
+        self.outputs = []
+        self.pre_activations = []
+        self.batchnorm_cache = []
+        self.deltas = []
+        self.opt_state = []
+        self.t = 0
+
+    def summary(self):
+        print("Model Summary")
+        print("=" * 60)
+        print(f"Optimizer: {self.optimizer_type.upper()} | LR: {self.learning_rate} | L2: {self.l2_lambda}")
+        print("=" * 60)
+        total_params = 0
+        for i, layer in enumerate(self.layers):
+            layer_type = layer["type"]
+            if layer_type in ("dense", "sparse"):
+                params = layer["weights"].size + layer["bias"].size
+                total_params += params
+                print(f"Layer {i}: {layer_type.upper()} - Input: {layer['weights'].shape[1]}, Output: {layer['weights'].shape[0]}, Params: {params}")
+            elif layer_type == "conv2d":
+                params = layer["weights"].size + layer["bias"].size
+                total_params += params
+                print(f"Layer {i}: {layer_type.upper()} - In_ch: {layer['in_ch']}, Out_ch: {layer['out_ch']}, Kernel: {layer['k']}x{layer['k']}, Params: {params}")
+            elif layer_type == "batchnorm":
+                params = layer["gamma"].size + layer["beta"].size
+                total_params += params
+                print(f"Layer {i}: {layer_type.upper()} - Features: {layer['num_features']}, Params: {params}")
+            else:
+                print(f"Layer {i}: {layer_type.upper()}")
+        print(f"Total Parameters: {total_params}")
+        print("=" * 60)

enilnets-1.0.0/Enilnets/forward.py

@@ -0,0 +1,101 @@
+import numpy as np
+from .activations import activate
+
+def im2col(input_data, filter_h, filter_w, stride=1, pad=0):
+    N, C, H, W = input_data.shape
+    out_h = (H + 2 * pad - filter_h) // stride + 1
+    out_w = (W + 2 * pad - filter_w) // stride + 1
+    img = np.pad(input_data, [(0, 0), (0, 0), (pad, pad), (pad, pad)], mode='constant')
+
+    N_stride, C_stride, H_stride, W_stride = img.strides
+    shape = (N, C, filter_h, filter_w, out_h, out_w)
+    strides = (N_stride, C_stride, H_stride, W_stride, H_stride * stride, W_stride * stride)
+
+    col = np.lib.stride_tricks.as_strided(img, shape=shape, strides=strides)
+    return col.transpose(0, 4, 5, 1, 2, 3).reshape(N * out_h * out_w, -1)
+
+def batchnorm_forward(x, layer, training):
+    epsilon = layer.get("epsilon", 1e-5)
+    momentum = layer.get("momentum", 0.1)
+    if training:
+        mean = np.mean(x, axis=0)
+        variance = np.var(x, axis=0)
+        x_norm = (x - mean) / np.sqrt(variance + epsilon)
+        out = layer["gamma"] * x_norm + layer["beta"]
+        layer["running_mean"] = (1 - momentum) * layer["running_mean"] + momentum * mean
+        layer["running_var"] = (1 - momentum) * layer["running_var"] + momentum * variance
+        cache = (x, x_norm, mean, variance, layer["gamma"], epsilon)
+    else:
+        x_norm = (x - layer["running_mean"]) / np.sqrt(layer["running_var"] + epsilon)
+        out = layer["gamma"] * x_norm + layer["beta"]
+        cache = None
+    return out, cache
+
+def Forward(self, inputs, training=False, dropout_rate=0.0):
+    x = np.asarray(inputs, dtype=np.float64)
+    if x.ndim == 1:
+        x = x.reshape(1, -1)
+    elif x.ndim == 3:
+        x = x.reshape(1, *x.shape)
+
+    self.outputs = [x]
+    self.pre_activations = [None]
+    self.batchnorm_cache = []
+
+    for layer in self.layers:
+        x = self.outputs[-1]
+        if layer["type"] in ("dense", "sparse"):
+            z = np.dot(x, layer["weights"].T) + layer["bias"]
+            x = activate(layer["activation"], z)
+            self.pre_activations.append(z)
+            self.batchnorm_cache.append(None)
+        elif layer["type"] == "conv2d":
+            B, C, H, W = x.shape
+            F, _, K, _ = layer["weights"].shape
+            out_h, out_w = H - K + 1, W - K + 1
+            col = im2col(x, K, K)
+            weights_flat = layer["weights"].reshape(F, -1)
+            out = np.dot(col, weights_flat.T).reshape(B, out_h, out_w, F).transpose(0, 3, 1, 2)
+            z = out + layer["bias"][None, :, None, None]
+            x = activate(layer["activation"], z)
+            self.pre_activations.append(z)
+            self.batchnorm_cache.append(None)
+        elif layer["type"] == "flatten":
+            x = x.reshape(x.shape[0], -1)
+            self.pre_activations.append(None)
+            self.batchnorm_cache.append(None)
+        elif layer["type"] == "maxpool2d":
+            B, C, H, W, p = *x.shape, layer["p"]
+            x = x[:, :, : H // p * p, : W // p * p].reshape(B, C, H // p, p, W // p, p).max(axis=(3, 5))
+            self.pre_activations.append(None)
+            self.batchnorm_cache.append(None)
+        elif layer["type"] == "avgpool2d":
+            B, C, H, W, p = *x.shape, layer["p"]
+            x = x[:, :, : H // p * p, : W // p * p].reshape(B, C, H // p, p, W // p, p).mean(axis=(3, 5))
+            self.pre_activations.append(None)
+            self.batchnorm_cache.append(None)
+        elif layer["type"] == "batchnorm":
+            flat = x.reshape(x.shape[0], -1)
+            normalized, cache = batchnorm_forward(flat, layer, training)
+            x = normalized.reshape(x.shape)
+            self.pre_activations.append(None)
+            self.batchnorm_cache.append(cache)
+        elif layer["type"] == "dropout":
+            rate = layer.get("rate", dropout_rate)
+            if training and rate > 0:
+                if rate >= 1.0:
+                    mask = np.zeros_like(x, dtype=np.float64)
+                    x = np.zeros_like(x)
+                else:
+                    mask = (np.random.rand(*x.shape) > rate).astype(np.float64)
+                    x = x * mask / (1.0 - rate)
+                layer["mask"] = mask
+            else:
+                layer["mask"] = None
+                x = x
+            self.pre_activations.append(None)
+            self.batchnorm_cache.append(None)
+        else:
+            raise ValueError(f"Unknown layer type: {layer['type']}")
+        self.outputs.append(x)
+    return self.outputs[-1]

enilnets-1.0.0/Enilnets/io.py

@@ -0,0 +1,48 @@
+import json
+import pickle
+import os
+import numpy as np
+
+def _numpy_encoder(obj):
+    if isinstance(obj, np.ndarray):
+        return obj.tolist()
+    raise TypeError(f"Object of type {type(obj)} is not JSON serializable")
+
+def Save(self, file):
+    payload = {
+        "version": 2,
+        "layers": self.layers,
+        "optimizer": self.optimizer_type,
+        "learning_rate": self.learning_rate,
+        "l2_lambda": self.l2_lambda,
+        "momentum": self.momentum,
+        "t": self.t,
+    }
+    ext = os.path.splitext(file)[1].lower()
+    if ext == ".pkl":
+        with open(file, "wb") as f:
+            pickle.dump(payload, f)
+    else:
+        with open(file, "w") as f:
+            json.dump(payload, f, default=_numpy_encoder)
+
+def Load(self, file):
+    ext = os.path.splitext(file)[1].lower()
+    if ext == ".pkl":
+        with open(file, "rb") as f:
+            raw = pickle.load(f)
+    else:
+        with open(file, "r") as f:
+            raw = json.load(f)
+    self.layers = []
+    for l in raw.get("layers", []):
+        for k in ["weights", "bias", "mask", "gamma", "beta", "running_mean", "running_var"]:
+            if k in l:
+                l[k] = np.array(l[k], dtype=np.float64)
+        self.layers.append(l)
+    self.opt_state = []
+    self.t = raw.get("t", 0)
+    self.learning_rate = raw.get("learning_rate", self.learning_rate)
+    self.optimizer_type = raw.get("optimizer", self.optimizer_type)
+    self.l2_lambda = raw.get("l2_lambda", self.l2_lambda)
+    self.momentum = raw.get("momentum", self.momentum)

enilnets-1.0.0/Enilnets/layers.py

@@ -0,0 +1,34 @@
+import numpy as np
+from .weight_init import init_weights, init_conv_weights
+
+def add_dense(self, n_in, n_out, activation="relu", init_method="xavier_uniform"):
+    w, b = init_weights(n_in, n_out, method=init_method)
+    self.layers.append({"type": "dense", "weights": w, "bias": b, "activation": activation})
+
+def add_sparse(self, n_in, n_out, connectivity=0.5, activation="relu", init_method="xavier_uniform"):
+    w, b = init_weights(n_in, n_out, method=init_method)
+    mask = (np.random.rand(n_out, n_in) < connectivity).astype(np.float64)
+    self.layers.append({"type": "sparse", "weights": w * mask, "bias": b, "mask": mask, "activation": activation})
+
+def add_conv2d(self, in_ch, out_ch, k, activation="relu", init_method="he_normal"):
+    w, b = init_conv_weights(in_ch, out_ch, k, method=init_method)
+    self.layers.append({"type": "conv2d", "weights": w, "bias": b, "in_ch": in_ch, "out_ch": out_ch, "k": k, "activation": activation})
+
+def add_flatten(self):
+    self.layers.append({"type": "flatten"})
+
+def add_maxpool2d(self, pool_size=2):
+    self.layers.append({"type": "maxpool2d", "p": pool_size})
+
+def add_avgpool2d(self, pool_size=2):
+    self.layers.append({"type": "avgpool2d", "p": pool_size})
+
+def add_batchnorm(self, num_features, epsilon=1e-5, momentum=0.1):
+    self.layers.append({"type": "batchnorm", "num_features": num_features, "epsilon": epsilon, "momentum": momentum,
+                        "running_mean": np.zeros(num_features, dtype=np.float64),
+                        "running_var": np.ones(num_features, dtype=np.float64),
+                        "gamma": np.ones(num_features, dtype=np.float64),
+                        "beta": np.zeros(num_features, dtype=np.float64)})
+
+def add_dropout(self, rate=0.5):
+    self.layers.append({"type": "dropout", "rate": rate})

enilnets-1.0.0/Enilnets/loss.py

@@ -0,0 +1,43 @@
+import numpy as np
+
+def ComputeLoss(self, output, target, function="mse", reduction="mean", **kwargs):
+    o = np.asarray(output, dtype=np.float64)
+    t = np.asarray(target, dtype=np.float64)
+    if function == "mse":
+        loss = (o - t) ** 2
+    elif function == "mae":
+        loss = np.abs(o - t)
+    elif function == "huber":
+        delta = kwargs.get("delta", 1.0)
+        diff = np.abs(o - t)
+        loss = np.where(diff < delta, 0.5 * diff**2, delta * (diff - 0.5 * delta))
+    elif function == "smooth_l1":
+        diff = np.abs(o - t)
+        loss = np.where(diff < 1, 0.5 * diff**2, diff - 0.5)
+    elif function == "binary_cross_entropy":
+        o = np.clip(o, 1e-12, 1 - 1e-12)
+        loss = -(t * np.log(o) + (1 - t) * np.log(1 - o))
+    elif function in ("cross_entropy", "categorical_cross_entropy"):
+        o = np.clip(o, 1e-12, 1.0)
+        loss = -t * np.log(o)
+        if reduction == "mean":
+            return float(np.sum(loss) / o.shape[0])
+        if reduction == "sum":
+            return float(np.sum(loss))
+        return loss
+    elif function == "focal":
+        alpha = kwargs.get("alpha", 0.25)
+        gamma = kwargs.get("gamma", 2.0)
+        o = np.clip(o, 1e-12, 1.0)
+        pt = o * t + (1 - o) * (1 - t)
+        loss = - (alpha * t * (1 - pt) ** gamma * np.log(o) + (1 - alpha) * (1 - t) * pt ** gamma * np.log(1 - o))
+    elif function == "hinge":
+        loss = np.maximum(0, 1 - t * o)
+    else:
+        raise ValueError(f"Unknown loss function: {function}")
+
+    if reduction == "mean":
+        return float(np.mean(loss))
+    if reduction == "sum":
+        return float(np.sum(loss))
+    return loss

enilnets-1.0.0/Enilnets/optimizer.py

@@ -0,0 +1,96 @@
+import numpy as np
+from .forward import im2col
+
+def update(self):
+    self.t += 1
+    b1, b2, eps = 0.9, 0.999, 1e-8
+
+    if not self.opt_state:
+        for layer in self.layers:
+            if layer["type"] in ("dense", "sparse", "conv2d"):
+                self.opt_state.append({
+                    "mw": np.zeros_like(layer["weights"]),
+                    "vw": np.zeros_like(layer["weights"]),
+                    "mb": np.zeros_like(layer["bias"]),
+                    "vb": np.zeros_like(layer["bias"]),
+                    "vgw": np.zeros_like(layer["weights"]),
+                    "vgb": np.zeros_like(layer["bias"]),
+                })
+            elif layer["type"] == "batchnorm":
+                self.opt_state.append({
+                    "mg": np.zeros_like(layer["gamma"]),
+                    "vg": np.zeros_like(layer["gamma"]),
+                    "mb": np.zeros_like(layer["beta"]),
+                    "vb": np.zeros_like(layer["beta"]),
+                })
+            else:
+                self.opt_state.append(None)
+
+    for l, layer in enumerate(self.layers):
+        state = self.opt_state[l]
+        if layer["type"] in ("dense", "sparse"):
+            grad_w = np.dot(self.deltas[l].T, self.outputs[l])
+            grad_b = np.sum(self.deltas[l], axis=0)
+            if layer["type"] == "sparse":
+                grad_w *= layer["mask"]
+        elif layer["type"] == "conv2d":
+            K = layer["k"]
+            col = im2col(self.outputs[l], K, K)
+            delta_flat = self.deltas[l].transpose(0, 2, 3, 1).reshape(-1, layer["weights"].shape[0])
+            grad_w_flat = np.dot(delta_flat.T, col)
+            grad_w = grad_w_flat.reshape(layer["weights"].shape)
+            grad_b = np.sum(self.deltas[l], axis=(0, 2, 3))
+        elif layer["type"] == "batchnorm":
+            grad_gamma = layer.get("d_gamma", np.zeros_like(layer["gamma"]))
+            grad_beta = layer.get("d_beta", np.zeros_like(layer["beta"]))
+
+            if self.optimizer_type == "sgd":
+                state["mg"] = self.momentum * state["mg"] - self.learning_rate * grad_gamma
+                state["mb"] = self.momentum * state["mb"] - self.learning_rate * grad_beta
+                layer["gamma"] += state["mg"]
+                layer["beta"] += state["mb"]
+            elif self.optimizer_type == "rmsprop":
+                state["vg"] = b2 * state["vg"] + (1 - b2) * (grad_gamma ** 2)
+                state["vb"] = b2 * state["vb"] + (1 - b2) * (grad_beta ** 2)
+                layer["gamma"] -= self.learning_rate * grad_gamma / (np.sqrt(state["vg"]) + eps)
+                layer["beta"] -= self.learning_rate * grad_beta / (np.sqrt(state["vb"]) + eps)
+            elif self.optimizer_type == "adagrad":
+                state["vg"] += grad_gamma ** 2
+                state["vb"] += grad_beta ** 2
+                layer["gamma"] -= self.learning_rate * grad_gamma / (np.sqrt(state["vg"]) + eps)
+                layer["beta"] -= self.learning_rate * grad_beta / (np.sqrt(state["vb"]) + eps)
+            else:  # adam
+                state["mg"] = b1 * state["mg"] + (1 - b1) * grad_gamma
+                state["vg"] = b2 * state["vg"] + (1 - b2) * (grad_gamma ** 2)
+                layer["gamma"] -= self.learning_rate * (state["mg"] / (1 - b1 ** self.t)) / (np.sqrt(state["vg"] / (1 - b2 ** self.t)) + eps)
+                state["mb"] = b1 * state["mb"] + (1 - b1) * grad_beta
+                state["vb"] = b2 * state["vb"] + (1 - b2) * (grad_beta ** 2)
+                layer["beta"] -= self.learning_rate * (state["mb"] / (1 - b1 ** self.t)) / (np.sqrt(state["vb"] / (1 - b2 ** self.t)) + eps)
+            continue
+        else:
+            continue
+
+        grad_w = grad_w + self.l2_lambda * layer["weights"] * layer.get("mask", 1.0)
+
+        if self.optimizer_type == "sgd":
+            state["vgw"] = self.momentum * state["vgw"] - self.learning_rate * grad_w
+            state["vgb"] = self.momentum * state["vgb"] - self.learning_rate * grad_b
+            layer["weights"] += state["vgw"]
+            layer["bias"] += state["vgb"]
+        elif self.optimizer_type == "rmsprop":
+            state["vw"] = b2 * state["vw"] + (1 - b2) * (grad_w ** 2)
+            state["vb"] = b2 * state["vb"] + (1 - b2) * (grad_b ** 2)
+            layer["weights"] -= self.learning_rate * grad_w / (np.sqrt(state["vw"]) + eps)
+            layer["bias"] -= self.learning_rate * grad_b / (np.sqrt(state["vb"]) + eps)
+        elif self.optimizer_type == "adagrad":
+            state["vw"] += grad_w ** 2
+            state["vb"] += grad_b ** 2
+            layer["weights"] -= self.learning_rate * grad_w / (np.sqrt(state["vw"]) + eps)
+            layer["bias"] -= self.learning_rate * grad_b / (np.sqrt(state["vb"]) + eps)
+        else:
+            state["mw"] = b1 * state["mw"] + (1 - b1) * grad_w
+            state["vw"] = b2 * state["vw"] + (1 - b2) * (grad_w ** 2)
+            layer["weights"] -= self.learning_rate * (state["mw"] / (1 - b1 ** self.t)) / (np.sqrt(state["vw"] / (1 - b2 ** self.t)) + eps)
+            state["mb"] = b1 * state["mb"] + (1 - b1) * grad_b
+            state["vb"] = b2 * state["vb"] + (1 - b2) * (grad_b ** 2)
+            layer["bias"] -= self.learning_rate * (state["mb"] / (1 - b1 ** self.t)) / (np.sqrt(state["vb"] / (1 - b2 ** self.t)) + eps)

enilnets-1.0.0/Enilnets/reinforce.py

@@ -0,0 +1,25 @@
+import copy
+import numpy as np
+
+def Reinforce(self, inputs, score_fn, noise=0.05, tries=10, sigma=1.0):
+    inputs = np.asarray(inputs, dtype=np.float64)
+    best_score = score_fn(self.Forward(inputs))
+    best_layers = copy.deepcopy(self.layers)
+    base_layers = copy.deepcopy(self.layers)
+
+    for _ in range(max(1, tries)):
+        candidate = copy.deepcopy(base_layers)
+        for layer in candidate:
+            if "weights" in layer:
+                layer["weights"] += np.random.normal(0, sigma * noise, layer["weights"].shape)
+                if layer["type"] == "sparse":
+                    layer["weights"] *= layer["mask"]
+                layer["bias"] += np.random.normal(0, sigma * noise, layer["bias"].shape)
+        self.layers = candidate
+        score = score_fn(self.Forward(inputs))
+        if score > best_score:
+            best_score = score
+            best_layers = copy.deepcopy(candidate)
+
+    self.layers = best_layers
+    return best_score

enilnets-1.0.0/Enilnets/train.py

@@ -0,0 +1,54 @@
+import numpy as np
+
+def TrainBatch(self, xs, ys, loss_function=None, **loss_kwargs):
+    out = self.Forward(xs, training=True)
+    if loss_function is None:
+        loss_function = "cross_entropy" if self.layers[-1].get("activation") == "softmax" else "mse"
+    loss = self.ComputeLoss(out, ys, loss_function, **loss_kwargs)
+    self.Backward(ys)
+    self.update()
+    return loss, out
+
+def compute_accuracy(self, predictions, targets):
+    if predictions.shape[-1] > 1:  # Multi-class
+        pred_classes = np.argmax(predictions, axis=1)
+        true_classes = np.argmax(targets, axis=1)
+    else:  # Binary
+        pred_classes = (predictions > 0.5).astype(int).flatten()
+        true_classes = targets.flatten()
+    return np.mean(pred_classes == true_classes)
+
+def Train(self, X_train, Y_train, epochs=10, batch_size=32, X_val=None, Y_val=None, loss_function=None, verbose=True, **loss_kwargs):
+    history = {"loss": [], "val_loss": [], "accuracy": [], "val_accuracy": []}
+    n_samples = X_train.shape[0]
+    for epoch in range(epochs):
+        indices = np.random.permutation(n_samples)
+        X_shuffled = X_train[indices]
+        Y_shuffled = Y_train[indices]
+        epoch_loss = 0.0
+        epoch_acc = 0.0
+        total_samples = 0
+        for i in range(0, n_samples, batch_size):
+            X_batch = X_shuffled[i:i+batch_size]
+            Y_batch = Y_shuffled[i:i+batch_size]
+            loss, preds = self.TrainBatch(X_batch, Y_batch, loss_function=loss_function, **loss_kwargs)
+            batch_size_actual = X_batch.shape[0]
+            epoch_loss += loss * batch_size_actual
+            epoch_acc += self.compute_accuracy(preds, Y_batch) * batch_size_actual
+            total_samples += batch_size_actual
+        avg_loss = epoch_loss / total_samples
+        avg_acc = epoch_acc / total_samples
+        history["loss"].append(avg_loss)
+        history["accuracy"].append(avg_acc)
+        if X_val is not None and Y_val is not None:
+            val_pred = self.Forward(X_val)
+            val_loss = self.ComputeLoss(val_pred, Y_val, loss_function if loss_function is not None else ("cross_entropy" if self.layers[-1].get("activation") == "softmax" else "mse"), **loss_kwargs)
+            val_acc = self.compute_accuracy(val_pred, Y_val)
+            history["val_loss"].append(val_loss)
+            history["val_accuracy"].append(val_acc)
+            if verbose:
+                print(f"Epoch {epoch+1}/{epochs} - loss: {avg_loss:.4f} - acc: {avg_acc:.4f} - val_loss: {val_loss:.4f} - val_acc: {val_acc:.4f}")
+        else:
+            if verbose:
+                print(f"Epoch {epoch+1}/{epochs} - loss: {avg_loss:.4f} - acc: {avg_acc:.4f}")
+    return history