neuralnetworknumpy 0.1.1__py3-none-any.whl

NeuralNetworkFromScratch/__init__.py

@@ -0,0 +1,56 @@
+"""
+NeuralNetworkFromScratch
+A minimal deep learning framework built using NumPy.
+"""
+
+__version__ = "0.1.0"
+
+# Layers
+from .layers import (
+    Layer,
+    Dense,
+    Activation,
+    ReLu,
+    Sigmoid,
+    Softmax,
+    Linear,
+    Tanh,
+    BatchNorm,
+    Dropout,
+)
+
+# Model
+from .model import NeuralNetwork
+
+# Utils
+from .utils import (
+    History,
+    Scaler,
+    split_train_test,
+    split_train_validation,
+)
+
+__all__ = [
+    # Core
+    "NeuralNetwork",
+
+    # Base
+    "Layer",
+
+    # Layers
+    "Dense",
+    "Activation",
+    "ReLu",
+    "Sigmoid",
+    "Softmax",
+    "Linear",
+    "Tanh",
+    "BatchNorm",
+    "Dropout",
+
+    # Utilities
+    "History",
+    "Scaler",
+    "split_train_test",
+    "split_train_validation",
+]

NeuralNetworkFromScratch/layers.py

@@ -0,0 +1,342 @@
+import numpy as np
+
+class Layer:
+  def __init__(self):
+    # Trainable parameters (some layers won't use them)
+    self.W = None
+    self.b = None
+
+    # Gradients
+    self.dW = None
+    self.db = None
+
+    # Optimizer states
+    self.vW = None
+    self.vb = None
+    self.mW = None
+    self.mb = None
+
+    # Forward pass values
+    self.A = None
+    self.A_prev = None  # Input to this layer
+
+    # Backprop pass values
+    self.Z = None
+
+  # Forward and backward are abstract methods — override in subclasses
+  def _forward(self, A_prev, training=None):
+    raise NotImplementedError
+
+  def _backward(self, dA):
+    raise NotImplementedError
+
+  def _update(self, lambda_, lr, beta1, beta2, _eps, optimizer, t):
+    raise NotImplementedError
+
+
+class Dense(Layer):
+  def __init__(self, units: int, inputs:int=0, kernel_initializer: str=None):
+    super().__init__()
+
+    self.units = units
+    self.in_size = inputs
+    self.out_size = units
+    self.kernel_initializer = kernel_initializer
+
+    if not kernel_initializer:
+      self._set_default_initializers()
+
+
+  def _set_default_initializers(self):
+      self.initializer = "he"
+      """if self.activation == "relu":
+          self.initializer = "he"
+      elif self.activation in ["sigmoid", "tanh", "softmax", "linear"]:
+          self.initializer = "xavier"
+      else:
+          raise Exception("Invalid activation function")"""
+
+  def build(self, input_size):
+      self.in_size = input_size
+      self._initialize_weights()
+
+      self.b = np.zeros((self.out_size, 1), dtype=np.float32)
+
+      # Optimizer state
+      self.vW = np.zeros_like(self.W).astype(np.float32)
+      self.vb = np.zeros_like(self.b).astype(np.float32)
+      self.mW = np.zeros_like(self.W).astype(np.float32)
+      self.mb = np.zeros_like(self.b).astype(np.float32)
+
+  def _initialize_weights(self):
+
+      if self.initializer == "he":
+          # W ~ N(0, √(2/in))
+          std = np.sqrt(2.0 / self.in_size)
+          self.W = np.random.randn(self.out_size, self.in_size).astype(np.float32) * std
+
+      elif self.initializer == "xavier":
+          # U(-√(6/(in+out)),√(6/(in+out)))
+          limit = np.sqrt(6.0 / (self.in_size + self.out_size))
+          self.W = np.random.uniform(-limit, limit, (self.out_size, self.in_size)).astype(np.float32)
+
+      else:
+          self.W = np.random.randn(self.out_size, self.in_size).astype(np.float32) * 0.01
+
+  def _forward(self, A_prev, training=None):
+      if self.W is None:
+          self.build(A_prev.shape[0])
+
+      self.A_prev = A_prev
+      self.Z = np.dot(self.W, A_prev) + self.b
+
+      return self.Z
+
+  def _backward(self, dA, skip_activation=False):
+
+      # dW_i = dZ_i · A_{i}^T
+      # Gradient of the loss w.r.t. weights of layer i
+      self.dW = np.dot(dA, self.A_prev.T)
+
+      # dB_i = sum(dZ_i) over the batch
+      # Gradient of the loss w.r.t. biases of layer i
+      self.db = np.sum(dA, axis=1, keepdims=True)
+
+      # Gradient to pass backward
+      # dA_prev_i = W_i · dA_i
+      return np.dot(self.W.T, dA)
+
+
+  def _update(self, lambda_, lr, beta1, beta2, _eps, optimizer, t):
+
+      if optimizer == "adamW":
+          dw = self.dW # pure gradient
+      else:
+          # Get batch size from the last dZ calculation to scale regularization
+          m = self.A.shape[1]
+          dw = self.dW + (lambda_ / m) * self.W # L2 regularization
+
+      if optimizer == "momentum":
+        self.vW = beta1 * self.vW + dw
+        self.vb = beta1 * self.vb + self.db
+
+        update_w = self.vW
+        update_b = self.vb
+
+      elif optimizer == "adam" or optimizer == "adamW":
+
+        self.mW = beta1 * self.mW + (1 - beta1) * dw
+        self.mb = beta1 * self.mb + (1 - beta1) * self.db
+
+        self.vW = beta2 * self.vW + (1 - beta2) * (dw ** 2)
+        self.vb = beta2 * self.vb + (1 - beta2) * (self.db ** 2)
+
+        m_w_hat = self.mW / (1 - beta1 ** t)
+        m_b_hat = self.mb / (1 - beta1 ** t)
+
+        v_w_hat = self.vW / (1 - beta2 ** t)
+        v_b_hat = self.vb / (1 - beta2 ** t)
+
+
+        update_w = m_w_hat / (np.sqrt(v_w_hat) + _eps)
+        update_b = m_b_hat / (np.sqrt(v_b_hat) + _eps)
+
+      elif optimizer == "rmsprop":
+        self.vW = beta2 * self.vW + (1 - beta2) * (dw ** 2)
+        self.vb = beta2 * self.vb + (1 - beta2) * (self.db ** 2)
+
+        update_w = dw / (np.sqrt(self.vW) + _eps)
+        update_b = self.db / (np.sqrt(self.vb) + _eps)
+
+      else:
+        # Classic SGD
+        update_w = dw
+        update_b = self.db
+
+
+      # W = W - lr * (dW + λ * W)
+      self.W -= lr * update_w
+      self.b -= lr * update_b
+
+      # Decouple (adamW) Weight Decay
+      if optimizer == "adamW":
+        self.W *= (1 - lr * lambda_)  # Decoupled weight decay
+
+
+class Activation(Layer):
+  def __init__(self):
+    super().__init__()
+
+  def _update(self, *args, **kwargs):
+    pass  # no parameters to update
+
+class ReLu(Activation):
+
+  def _forward(self, Z, training=None):
+    self.Z = Z
+    self.A = np.maximum(0, Z)
+    return self.A
+
+  def _backward(self, dA):
+    return dA * (self.Z > 0)
+
+class Sigmoid(Activation):
+
+  def _forward(self, Z, training=None):
+      self.Z = Z
+      self.A = 1 / (1 + np.exp(-Z))
+      return self.A
+
+  def _backward(self, dA):
+      return dA * self.A * (1 - self.A)
+
+class Softmax(Activation):
+
+  def _forward(self, Z, training=None):
+      self.Z = Z
+      shifted = Z - np.max(Z, axis=0, keepdims=True)
+      exp_vals = np.exp(shifted)
+      self.A = exp_vals / np.sum(exp_vals, axis=0, keepdims=True)
+      return self.A
+
+  def _backward(self, dA):
+      s = np.sum(dA * self.A, axis=0, keepdims=True)
+      return self.A * (dA - s)
+
+class Linear(Activation):
+
+  def _forward(self, Z, training=None):
+      self.Z = Z
+      self.A = Z
+      return self.A
+
+  def _backward(self, dA):
+      return dA  # derivative is 1
+
+class Tanh(Activation):
+
+  def _forward(self, Z, training=None):
+      self.Z = Z
+      self.A = np.tanh(Z)
+      return self.A
+
+  def _backward(self, dA):
+      return dA * (1 - self.A ** 2)
+
+
+
+class BatchNorm(Layer):
+  def __init__(self, momentum=0.9):
+    super().__init__()
+    self.momentum = momentum
+    self._eps = 1e-08
+
+    self.gamma = None
+    self.beta = None
+
+    self.running_mean = None
+    self.running_var = None
+
+  def build(self, input_size):
+    self.gamma = np.ones((input_size, 1), dtype=np.float32)
+    self.beta = np.zeros((input_size, 1), dtype=np.float32)
+
+    self.running_mean = np.zeros((input_size, 1), dtype=np.float32)
+    self.running_var = np.ones((input_size, 1), dtype=np.float32)
+
+  def _forward(self, A_prev, training=True):
+    if self.gamma is None:
+      self.build(A_prev.shape[0])
+
+    self.A_prev = A_prev
+
+    if training:
+      self.mean = np.mean(A_prev, axis=1, keepdims=True)
+      self.var = np.var(A_prev, axis=1, keepdims=True)
+
+      # X̂ = (X - μB) / √(σB^2 + ε)
+      self.X_hat = (A_prev - self.mean) / np.sqrt(self.var + self._eps)
+      # A = γ * X̂ + β
+      self.A = self.gamma * self.X_hat + self.beta
+
+      # Update running stats
+      self.running_mean = (
+          self.momentum * self.running_mean
+          + (1 - self.momentum) * self.mean
+      )
+
+      self.running_var = (
+          self.momentum * self.running_var
+          + (1 - self.momentum) * self.var
+      )
+
+    else:
+      self.X_hat = (A_prev - self.running_mean) / np.sqrt(self.running_var + self._eps)
+      self.A = self.gamma * self.X_hat + self.beta
+
+    return self.A
+
+  def _backward(self, dA, skip_activation=False):
+
+    m = dA.shape[1]
+
+    dgamma = np.sum(dA * self.X_hat, axis=1, keepdims=True)
+    dbeta  = np.sum(dA, axis=1, keepdims=True)
+
+    dX_hat = dA * self.gamma
+
+    var_inv = 1. / np.sqrt(self.var + self._eps)
+
+    dvar = np.sum(dX_hat * (self.A_prev - self.mean) * -0.5 * var_inv**3,
+                  axis=1, keepdims=True)
+
+    dmean = (
+        np.sum(dX_hat * -var_inv, axis=1, keepdims=True)
+        + dvar * np.mean(-2. * (self.A_prev - self.mean), axis=1, keepdims=True)
+    )
+
+    dX = (
+        dX_hat * var_inv
+        + dvar * 2 * (self.A_prev - self.mean) / m
+        + dmean / m
+    )
+
+    self.dgamma = dgamma / m
+    self.dbeta  = dbeta / m
+
+    return dX
+
+
+
+  def _update(self, lambda_, lr, beta1, beta2, eps, optimizer, t):
+    self.gamma -= lr * self.dgamma
+    self.beta -= lr * self.dbeta
+
+class Dropout(Layer):
+  def __init__(self, rate):
+    super().__init__()
+    self.rate = rate  # probability of dropping a unit
+    self.mask = None
+
+  def _forward(self, A_prev, training=True):
+    if not training:
+      # No dropout during inference
+      self.mask = np.ones_like(A_prev)
+      self.A = A_prev
+      return self.A
+
+    # Create dropout mask
+    self.mask = np.random.rand(A_prev.shape[0], A_prev.shape[1]) > self.rate
+    # Apply mask AND scale (inverted dropout)
+    self.A = (A_prev * self.mask) / (1 - self.rate)
+
+    return self.A
+
+  def _backward(self, dA, skip_activation=False):
+    # Backprop only through active neurons
+    dA_prev = (dA * self.mask) / (1 - self.rate)
+    return dA_prev
+
+  def _update(self, lambda_, lr, beta1, beta2, _eps, optimizer, t):
+    # Dropout layer has no trainable parameters
+    pass

NeuralNetworkFromScratch/model.py

@@ -0,0 +1,506 @@
+import numpy as np
+from tqdm.auto import tqdm
+
+from .layers import Dropout, Activation, BatchNorm, Dense
+from .utils import History
+
+class NeuralNetwork:
+    def __init__(self, layers:list):
+
+        self.num_classes = 0
+        self.layers = layers
+
+        self._eps = 1e-08 # Avoid dividing by zero
+        self.lr = 0.001
+        self.lambda_ = 0.0
+        self.beta1 = 0.9
+        self.beta2 = 0.999
+        self.loss_type = "cross_entropy"
+        self.optimizer = "adam"
+
+    def save(self, path):
+        layer_data = []
+
+        for layer in self.layers:
+            entry = {"type": type(layer).__name__}
+
+            if isinstance(layer, Dense):
+                entry.update({
+                    "W": layer.W,
+                    "b": layer.b,
+                    "in_size": layer.in_size,
+                    "out_size": layer.out_size,
+                    "initializer": layer.initializer,
+                })
+            elif isinstance(layer, BatchNorm):
+                entry.update({
+                    "gamma": layer.gamma,
+                    "beta": layer.beta,
+                    "running_mean": layer.running_mean,
+                    "running_var": layer.running_var,
+                    "momentum": layer.momentum,
+                })
+            elif isinstance(layer, Dropout):
+                entry["rate"] = layer.rate
+
+            # Activation layers (ReLu, Sigmoid, etc.) need no extra data
+
+            layer_data.append(entry)
+
+        np.savez(
+            path,
+            layers=np.array(layer_data, dtype=object),
+            lr=self.lr,
+            lambda_=self.lambda_,
+            beta1=self.beta1,
+            beta2=self.beta2,
+            loss_type=self.loss_type,
+            optimizer=self.optimizer,
+            num_classes=self.num_classes,
+        )
+
+
+    @staticmethod
+    def load(path):
+        from .layers import Dense, BatchNorm, Dropout, ReLu, Sigmoid, Softmax, Tanh, Linear
+
+        ACTIVATION_MAP = {
+            "ReLu": ReLu,
+            "Sigmoid": Sigmoid,
+            "Softmax": Softmax,
+            "Tanh": Tanh,
+            "Linear": Linear,
+        }
+
+        data = np.load(path, allow_pickle=True)
+        layers = []
+
+        for entry in data["layers"]:
+            layer_type = entry["type"]
+
+            if layer_type == "Dense":
+                layer = Dense(units=entry["out_size"])
+                layer.W = entry["W"]
+                layer.b = entry["b"]
+                layer.in_size = entry["in_size"]
+                layer.out_size = entry["out_size"]
+                layer.initializer = entry["initializer"]
+                # Restore optimizer states as zeros (not serialized)
+                layer.vW = np.zeros_like(layer.W)
+                layer.vb = np.zeros_like(layer.b)
+                layer.mW = np.zeros_like(layer.W)
+                layer.mb = np.zeros_like(layer.b)
+
+            elif layer_type == "BatchNorm":
+                layer = BatchNorm(momentum=entry["momentum"])
+                layer.gamma = entry["gamma"]
+                layer.beta = entry["beta"]
+                layer.running_mean = entry["running_mean"]
+                layer.running_var = entry["running_var"]
+
+            elif layer_type == "Dropout":
+                layer = Dropout(rate=entry["rate"])
+
+            elif layer_type in ACTIVATION_MAP:
+                layer = ACTIVATION_MAP[layer_type]()
+
+            else:
+                raise ValueError(f"Unknown layer type: {layer_type}")
+
+            layers.append(layer)
+
+        model = NeuralNetwork(layers)
+        model.lr = data["lr"].item()
+        model.lambda_ = data["lambda_"].item()
+        model.beta1 = data["beta1"].item()
+        model.beta2 = data["beta2"].item()
+        model.loss_type = data["loss_type"].item()
+        model.optimizer = data["optimizer"].item()
+        model.num_classes = data["num_classes"].item()
+
+        return model
+
+
+    def summary(self):
+        print("=" * 55)
+        print("Model Summary")
+        print("=" * 55)
+
+        total_params = 0
+
+        for i, layer in enumerate(self.layers):
+            layer_type = type(layer).__name__
+
+            if isinstance(layer, Dense):
+                params = layer.W.size + layer.b.size if layer.W is not None else 0
+                total_params += params
+                built = f"{layer.in_size} → {layer.out_size}"
+                print(f"[{i+1}] Dense          {built:<20} params: {params}")
+
+            elif isinstance(layer, BatchNorm):
+                params = layer.gamma.size + layer.beta.size if layer.gamma is not None else 0
+                total_params += params
+                print(f"[{i+1}] BatchNorm      momentum={layer.momentum:<13} params: {params}")
+
+            elif isinstance(layer, Dropout):
+                print(f"[{i+1}] Dropout        rate={layer.rate}")
+
+            elif isinstance(layer, Activation):
+                print(f"[{i+1}] {layer_type:<15}")
+
+            print("-" * 55)
+
+        print(f"Total trainable parameters: {total_params}")
+        print("=" * 55)
+
+
+    # One hot encoding for y_true - convert format into a matrix for calculations
+    def _one_hot_encoding(self, y):
+        one_hot_y = np.zeros((y.size, self.num_classes))
+        one_hot_y[np.arange(y.size), y] = 1
+        return one_hot_y.T
+
+    """ **********************************************************
+    Model Algorithms
+    ********************************************************** """
+
+    # Forward function - feed input and get prediction
+    def _forward(self, X, training=True):
+        # Z = W * A + B
+        # A - output (after activation function)
+        # Also next layer input
+        for layer in self.layers:
+            X = layer._forward(X, training=training)
+        return X
+
+
+    # Backward function - locates the origin of the loss and tweaks it
+    def _backward(self, y_true):
+      m = y_true.size
+
+
+      dA = self._loss_derivative(self.layers[-1].A, y_true) / m
+      dA = self.layers[-1]._backward(dA)
+
+      # Remaining layers
+      # Backpropagation: iterate layers in reverse order (from last to first)
+      # Compute gradients dW and dB for each layer
+      for layer in reversed(self.layers[:-1]):
+          # dA = ∂J/∂A_L
+          # This is the derivative of the loss w.r.t. the network output
+          dA = layer._backward(dA)
+
+
+    # Loss derivative - for the last layer based on the loss-type
+    def _loss_derivative(self, y_pred, y_true):
+        one_hot = self._one_hot_encoding(y_true) # y_true formatting
+        # Division by m happens in backward function
+        if self.loss_type == "cross_entropy":
+            # - y_true / y_pred
+            return -(one_hot / (y_pred + self._eps))
+
+        elif self.loss_type == "mse":
+            # 2 * (y_pred - y_true)
+            return 2 * (y_pred - one_hot)
+
+        else:
+            raise Exception("Invalid loss function")
+
+
+    # Compute loss for logging
+    def _compute_loss(self, y_pred, y_true):
+        one_hot = self._one_hot_encoding(y_true)  # shape: (num_classes, N)
+        m = y_true.size
+
+        if self.loss_type == "cross_entropy":
+          # -1/m * ∑ (y_true * log(y_pred))
+          data_loss = -1 * np.sum(one_hot * np.log(y_pred + self._eps)) / m
+        elif self.loss_type == "mse":
+          # 1/m * ∑ ((y_pred - y_true)^2)
+          data_loss = np.mean((y_pred - one_hot) ** 2)
+        else:
+          raise Exception("Invalid loss function")
+
+        # L2 regularization term: (λ / 2m) * sum(||W||^2)
+        reg_loss = 0.0
+        for layer in self.layers:
+          if not isinstance(layer, (Dropout, Activation, BatchNorm)):
+            reg_loss += np.sum(layer.W ** 2)
+
+        reg_loss = (self.lambda_ / (2 * m)) * reg_loss
+
+        return data_loss + reg_loss
+
+
+    # Update weights and biases
+    # lr = learning rate
+    # high lr - impact the model fast, can overshoot
+    # low lr - learns slower, wont overshoot
+    # Lambda λ = Counter overfitting by punishing big weights
+    # Force weights to be small but not zero (w = 0 -> no impact on the model)
+    # Beta1 = momentum factor
+    # Beta2 = RSMprop factor
+    def _update(self, optimizer_t):
+
+      for layer in self.layers:
+          layer._update(self.lambda_, self.lr, self.beta1, self.beta2, self._eps, self.optimizer, optimizer_t)
+
+    """ **********************************************************
+    Metrics
+    ********************************************************** """
+
+    # Calculate model accuracy
+    @staticmethod
+    def accuracy(predictions, y):
+      return np.sum(predictions==y) / y.size
+
+
+    # Calculate model precision
+    @staticmethod
+    def precision(predictions, y, num_classes):
+      precisions = []
+
+      for c in range(num_classes):
+        tp = np.sum((predictions == c) & (y == c))
+        fp = np.sum((predictions == c) & (y != c))
+
+        precisions.append(tp / (tp + fp + 1e-8))
+
+      return np.mean(precisions)
+
+
+    # Calculate model recall
+    @staticmethod
+    def recall(predictions, y, num_classes):
+      recalls = []
+
+      for c in range(num_classes):
+        tp = np.sum((predictions == c) & (y == c))
+        fn = np.sum((predictions != c) & (y == c))
+
+        recalls.append(tp / (tp + fn + 1e-8))
+
+      return np.mean(recalls)
+
+
+    # Calculate model f1
+    @staticmethod
+    def f1(predictions, y, num_classes):
+      precision = NeuralNetwork.precision(predictions, y, num_classes)
+      recall = NeuralNetwork.recall(predictions, y, num_classes)
+      return 2 * (precision * recall) / (precision + recall + 1e-8)
+
+
+    def calc_metrics(self, history:History, y_pred, y_true, metrics=None):
+      if metrics is None:
+          metrics = []
+      for metric in metrics:
+        if metric == "accuracy":
+          accuracy = NeuralNetwork.accuracy(y_pred, y_true)
+          history.add("accuracy", accuracy)
+        if metric == "precision":
+          precision = NeuralNetwork.precision(y_pred, y_true, self.num_classes)
+          history.add("precision", precision)
+        if metric == "recall":
+          recall = NeuralNetwork.recall(y_pred, y_true, self.num_classes)
+          history.add("recall", recall)
+        if metric == "f1":
+          f1 = NeuralNetwork.f1(y_pred, y_true, self.num_classes)
+          history.add("f1", f1)
+
+      return history
+
+    """ **********************************************************
+    Runtime functions
+    ********************************************************** """
+
+    # Converts final layer activation to predicted class labels
+    @staticmethod
+    def _decode_output(output):
+      if output.shape[0] == 1:  # Binary classification
+          return (output > 0.5).astype(int).flatten()
+      else:  # Multi-class (softmax)
+          return np.argmax(output, axis=0)
+
+
+    @staticmethod
+    def shuffle_data(x, y):
+      perm = np.random.permutation(y.size)
+      x = x[:, perm]
+      y = y[perm]
+      return x, y
+
+    @staticmethod
+    def set_seed(seed):
+      np.random.seed(seed)
+
+
+    def check_gradient(self, X, y):
+      assert X.shape[1] == y.size, f"X has {X.shape[1]} samples but y has {y.size}"
+
+      # Use a small batch to avoid numerical issues
+      X = X[:, :8].astype(np.float64)  # <-- float64 is critical for numerical grad
+      y = y[:8]
+
+      rel_diff = []
+      original_lambda = self.lambda_
+      self.lambda_ = 0.0
+      epsilon = 1e-5  # smaller epsilon can help
+
+      self._forward(X, training=False)
+      self._backward(y)
+
+      # Snapshot ALL analytical gradients before any weight perturbation
+      analytical_grads = {}
+      for idx, layer in enumerate(self.layers):
+          if hasattr(layer, 'dW') and layer.dW is not None:
+              analytical_grads[idx] = layer.dW.copy()  # <-- copy before perturbation
+
+      for idx, layer in enumerate(self.layers):
+          if hasattr(layer, 'W') and layer.W is not None:
+              i = np.random.randint(0, layer.W.shape[0])
+              j = np.random.randint(0, layer.W.shape[1])
+
+              W_orig = layer.W[i, j]
+              grad_analytical = analytical_grads[idx][i, j]
+
+              layer.W[i, j] = W_orig + epsilon
+              y_pred = self._forward(X, training=False)
+              loss_plus = self._compute_loss(y_pred, y)
+
+              layer.W[i, j] = W_orig - epsilon
+              y_pred = self._forward(X, training=False)
+              loss_minus = self._compute_loss(y_pred, y)
+
+              grad_numerical = (loss_plus - loss_minus) / (2 * epsilon)
+              layer.W[i, j] = W_orig
+
+              numerator = abs(grad_numerical - grad_analytical)
+              denominator = abs(grad_numerical) + abs(grad_analytical) + 1e-10
+              rel_diff.append(numerator / denominator)
+
+              print(f"Layer {idx} W[{i},{j}]  Numerical: {grad_numerical:.10f}  Analytical: {grad_analytical:.10f}  Rel diff: {rel_diff[-1]:.2e}")
+
+      self.lambda_ = original_lambda
+      return rel_diff
+
+
+    # Compile the model and trains it
+    # x - input features
+    # y - labels
+    # lr - learning rate
+    # epochs - num of iterations over the data
+    # batch_size - data size before model updating
+    # loss_type - loss functions used
+    # lambda_ - lambda used for preventing weights overfitting
+    def gradient_descent(self, X, y, X_val=None, y_val=None, epochs=10, batch_size=1):
+      X = X.astype(np.float32)
+      history = History()
+
+      optimizer_t = 0
+
+      stopping_patience = 5
+
+      for ep in range(epochs):
+        predictions = []
+        epoch_loss = 0
+
+        x_shuffled, y_shuffled = NeuralNetwork.shuffle_data(X, y)
+
+        # Batches
+        # tqdm - loading animation
+        for i in tqdm(range(0, x_shuffled.shape[1], batch_size)):
+
+          optimizer_t += 1
+
+          # get batch
+          x_batch = x_shuffled[:, i:i+batch_size]
+          y_batch = y_shuffled[i:i+batch_size]
+          # feed model
+          self._forward(x_batch)
+          self._backward(y_batch)
+          self._update(optimizer_t)
+
+          y_pred = self.layers[-1].A
+          # Monitor loss - epoch_loss = Avg(batches_loss)
+          predictions.append(self._decode_output(y_pred))
+          batch_loss = self._compute_loss(y_pred, y_batch)
+          epoch_loss += batch_loss * x_batch.shape[1] / X.shape[1]
+
+          # Check Gradient - make sure backpropagation works well
+          #self.check_gradient(x_batch, y_batch)
+
+        predictions = np.concatenate(predictions)
+
+        history.add("epoch", ep)
+        history.add("loss", epoch_loss)
+        history = self.calc_metrics(history, predictions, y_shuffled, metrics=["accuracy", "precision", "recall"])
+
+        # Validation
+        if X_val is not None and y_val is not None:
+          val_pred = self.predict_proba(X_val)
+          val_loss = self._compute_loss(val_pred, y_val)
+          val_acc = NeuralNetwork.accuracy(self._decode_output(val_pred), y_val)
+          history.add("val_loss", val_loss)
+          history.add("val_accuracy", val_acc)
+
+          history.progress()
+
+          if len(history.history["val_loss"]) > 1:
+            if history.history["val_loss"][-1] > history.history["val_loss"][-2]:
+                stopping_patience -= 1
+            if stopping_patience == 0:
+                print("Early stopping")
+                break
+        else:
+          history.progress()
+
+      return history
+
+
+    """ **********************************************************
+    Model API functions
+    ********************************************************** """
+
+    # Configure training hyperparameters and optimization settings
+    def compile(self, loss_type="cross_entropy", optimizer="adam", lr=0.001, lambda_=0.0, beta1=0.9, beta2=0.999):
+        self.lr = lr
+        self.lambda_ = lambda_
+        self.beta1 = beta1
+        self.beta2 = beta2
+        self.loss_type = loss_type
+        self.optimizer = optimizer
+
+
+    # Train the model
+    def fit(self, X, y, X_val=None, y_val=None, epochs=10, batch_size=1):
+        if X.shape[1] != y.size:
+            raise ValueError("Mismatch between samples and labels")
+
+        # Find last layer - last outsize is num classes
+        for layer in reversed(self.layers):
+            if hasattr(layer, "out_size"):
+                self.num_classes = layer.out_size
+                break
+        return self.gradient_descent(X, y, X_val=X_val, y_val=y_val, epochs=epochs, batch_size=batch_size)
+
+
+    # Add layers to model
+    def add(self, layer):
+        self.layers.append(layer)
+
+
+    # Return predicted class labels for input data
+    def predict(self, X):
+        return self._decode_output(self.predict_proba(X))
+
+
+    # Return raw output activations (probabilities or scores)
+    def predict_proba(self, X):
+        return self._forward(X, training=False)
+
+
+    # Evaluate model performance (accuracy) on given dataset
+    def evaluate(self, X, y):
+        predictions = self.predict(X)
+        return NeuralNetwork.accuracy(predictions, y)

NeuralNetworkFromScratch/utils.py

@@ -0,0 +1,87 @@
+import numpy as np
+
+class History:
+  def __init__(self):
+    self.history = {}
+
+  def add(self, key, value):
+    if key not in self.history:
+      self.history[key] = []
+    self.history[key].append(value)
+
+  def progress(self):
+    for key, value in self.history.items():
+      print(f"{key}: {value[-1]}")
+
+
+
+class Scaler:
+    def __init__(self, mode="standard"):
+        """
+        mode: "standard" for Z-score (Mean=0, Std=1)
+              "minmax" for range scaling (0 to 1)
+        """
+        self.mode = mode
+        self.mean = None
+        self.std = None
+        self.min = None
+        self.max = None
+
+    def fit(self, X):
+        """Calculates parameters from training data. X shape: (features, samples)"""
+        if self.mode == "standard":
+            self.mean = np.mean(X, axis=1, keepdims=True)
+            self.std = np.std(X, axis=1, keepdims=True)
+            self.std[self.std == 0] = 1e-8  # Avoid division by zero
+
+        elif self.mode == "minmax":
+            self.min = np.min(X, axis=1, keepdims=True)
+            self.max = np.max(X, axis=1, keepdims=True)
+            # Avoid division by zero if all values in a feature are the same
+            self.diff = self.max - self.min
+            self.diff[self.diff == 0] = 1e-8
+
+    def transform(self, X):
+        """Applies scaling to data using fitted parameters."""
+        if self.mode == "standard":
+            return (X - self.mean) / self.std
+        elif self.mode == "minmax":
+            return (X - self.min) / self.diff
+        else:
+            raise NotImplementedError
+
+    def fit_transform(self, X):
+        self.fit(X)
+        return self.transform(X)
+
+def split_train_test(X, y, test_ratio=0.2):
+  m = X.shape[1]
+  perm = np.random.permutation(m)
+
+  X = X[:, perm]
+  y = y[perm]
+
+  test_size = int(m * test_ratio)
+
+  X_test = X[:, :test_size]
+  y_test = y[:test_size]
+
+  X_train = X[:, test_size:]
+  y_train = y[test_size:]
+
+  return X_train, y_train, X_test, y_test
+
+
+def split_train_validation(X, y, val_ratio=0.2):
+    m = X.shape[0]
+    perm = np.random.permutation(m)
+    X = X[perm]
+    y = y[perm]
+
+    val_size = int(m * val_ratio)
+    X_val = X[:val_size]
+    y_val = y[:val_size]
+    X_train = X[val_size:]
+    y_train = y[val_size:]
+
+    return X_train, y_train, X_val, y_val

neuralnetworknumpy-0.1.1.dist-info/METADATA

@@ -0,0 +1,150 @@
+Metadata-Version: 2.4
+Name: neuralnetworknumpy
+Version: 0.1.1
+Summary: A neural network framework built completely from scratch using NumPy
+Author: Itamar Senderovitz
+License: MIT
+Project-URL: Homepage, https://github.com/Sendy45/NeuralNetworkFromScratch
+Keywords: neural network,deep learning,machine learning,numpy
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: numpy>=1.21
+Requires-Dist: tqdm>=4.60
+
+# NeuralNetworkFromScratch
+
+A lightweight Python library implementing a fully functional neural network **from scratch using NumPy**, without relying on machine learning frameworks such as TensorFlow or PyTorch.
+
+The goal of this project is to provide a clear and educational implementation of neural networks, including forward propagation, backpropagation, normalization, and regularization techniques.
+
+---
+
+## Features
+
+* Fully connected neural network implementation
+* Modular layer system
+* Forward and backward propagation
+* Batch normalization
+* Dropout regularization
+* ReLU and Softmax activation functions
+* Dataset scaling utilities
+* Train / validation split helpers
+
+---
+
+## Installation
+
+Install from PyPI:
+
+```bash
+pip install neuralnetwork-from-scratch
+```
+
+Or install from source:
+
+```bash
+git clone https://github.com/Sendy45/NeuralNetworkFromScratch.git
+cd NeuralNetworkFromScratch
+pip install .
+```
+
+---
+
+## Example Usage
+
+```python
+import numpy as np
+from keras.datasets import mnist
+
+from NeuralNetworkFromScratch import (
+    NeuralNetwork,
+    Dense,
+    ReLu,
+    BatchNorm,
+    Dropout,
+    Softmax
+)
+
+# load dataset
+(X_train, y_train), _ = mnist.load_data()
+
+# flatten images
+X_train = X_train.reshape(-1, 784) / 255.0
+
+model = NeuralNetwork([
+    Dense(64, inputs=784),
+    ReLu(),
+    BatchNorm(),
+    Dropout(0.1),
+    Dense(10),
+    Softmax()
+])
+
+model.compile(
+    optimizer="adam",
+    loss="categorical_crossentropy"
+)
+
+model.fit(X_train, y_train, epochs=10, batch_size=32)
+```
+
+---
+
+## Project Structure
+
+```
+NeuralNetworkFromScratch
+│
+├── neuralnet
+│   ├── __init__.py
+│   ├── network.py
+│   ├── layers.py
+│   ├── activations.py
+│   └── utils.py
+│
+├── tests
+├── README.md
+└── pyproject.toml
+```
+
+---
+
+## Goals of the Project
+
+This project was designed to:
+
+* Demonstrate **how neural networks work internally**
+* Provide a **clean NumPy-based implementation**
+* Serve as an **educational resource for learning deep learning fundamentals**
+
+Unlike production ML frameworks, this project prioritizes **clarity and learning over performance**.
+
+---
+
+## Dependencies
+
+* numpy
+* tqdm
+
+Optional dependencies used in examples:
+
+* matplotlib
+* keras (for datasets such as MNIST)
+
+---
+
+## License
+
+This project is licensed under the MIT License.
+
+---
+
+## Author
+
+Created by **Itamar Senderovitz**.

neuralnetworknumpy-0.1.1.dist-info/RECORD

@@ -0,0 +1,8 @@
+NeuralNetworkFromScratch/__init__.py,sha256=fDI9TXts-_WKxQLUfT_S99sU2bP3OBJOHssFq-jODTU,787
+NeuralNetworkFromScratch/layers.py,sha256=QUJACrS62u_oyAX1xfo_92OGWnAxemXthJ-4vRFQBFU,9462
+NeuralNetworkFromScratch/model.py,sha256=uTASLPdXtIFAaXbWGLWul9vHdX0Q8XpwJsxI_S5d0T0,17343
+NeuralNetworkFromScratch/utils.py,sha256=ck9yNw3dymXqGEvXyaexMHaK2lqLF5M34_EkqIrR4n0,2372
+neuralnetworknumpy-0.1.1.dist-info/METADATA,sha256=UGeqACpBhBVE41_KavxzisVy_6hRtmcoobPy_ESO5vc,3275
+neuralnetworknumpy-0.1.1.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+neuralnetworknumpy-0.1.1.dist-info/top_level.txt,sha256=lUKcwPPsuJ-pqHQfY-ukbA3K1X9wY1vzgjptuBVw8PI,25
+neuralnetworknumpy-0.1.1.dist-info/RECORD,,

neuralnetworknumpy-0.1.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

neuralnetworknumpy-0.1.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+NeuralNetworkFromScratch