nn-simulator 0.1.0__py2.py3-none-any.whl

nn_simulator/__init__.py

@@ -0,0 +1,3 @@
+"""nn_simulator imports."""
+from .nn_sim import NeuralNetwork
+from .classes import Architecture, TrainingParameters, UpdateParameters, TrainingLog

nn_simulator/activation_functions.py

@@ -0,0 +1,122 @@
+
+"""Module containing the activation functions and their derivatives."""
+import numpy as np
+
+
+# Linear
+def linear(x):
+    """Linear activation function."""
+    return x
+
+
+def linear_derivative(x):
+    """Derivative of the linear activation function."""
+    return np.ones_like(x)
+
+
+# Sigmoid
+def sigmoid(x):
+    """Sigmoid activation function."""
+    return 1 / (1 + np.exp(-x))
+
+
+def sigmoid_derivative(x):
+    """Derivative of the sigmoid activation function."""
+    s = sigmoid(x)
+    return s * (1 - s)
+
+
+# Hyperbolic tangent
+def tanh(x):
+    """Hyperbolic tangent activation function."""
+    return np.tanh(x)
+
+
+def tanh_derivative(x):
+    """Derivative of the hyperbolic tangent activation function."""
+    return 1 - np.tanh(x)**2
+
+
+# ReLU
+def relu(x):
+    """Rectified Linear Unit activation function."""
+    return np.maximum(0, x)
+
+
+def relu_derivative(x):
+    """Derivative of the ReLU activation function."""
+    return np.where(x > 0, 1, 0)
+
+
+# Leaky ReLU
+def leaky_relu(x, alpha=0.01):
+    """Leaky ReLU activation function."""
+    return np.where(x > 0, x, alpha * x)
+
+
+def leaky_relu_derivative(x, alpha=0.01):
+    """Derivative of the Leaky ReLU activation function."""
+    return np.where(x > 0, 1, alpha)
+
+
+# ELU
+def elu(x, alpha=1.0):
+    """Exponential Linear Unit activation function."""
+    return np.where(x > 0, x, alpha * np.expm1(x))
+
+
+def elu_derivative(x, alpha=1.0):
+    """Derivative of the ELU activation function."""
+    return np.where(x > 0, 1, alpha * np.exp(x))
+
+
+# Softplus
+def softplus(x):
+    """Softplus activation function."""
+    return np.logaddexp(0, x)
+
+
+def softplus_derivative(x):
+    """Derivative of the Softplus activation function."""
+    return sigmoid(x)
+
+
+# Softmax
+def softmax(x):
+    """Softmax activation function."""
+    exp_x = np.exp(x - np.max(x, axis=0, keepdims=True))
+    return exp_x / np.sum(exp_x, axis=0, keepdims=True)
+
+
+def softmax_derivative(x):
+    """Jacobian matrix of the Softmax activation function."""
+    s = softmax(x)
+    return np.diag(s) - np.outer(s, s)
+
+
+# Activation functions dictionary
+af_list = {
+    "LINEAR": linear,
+    "LINEAR_DERIVATIVE": linear_derivative,
+
+    "SIGMOID": sigmoid,
+    "SIGMOID_DERIVATIVE": sigmoid_derivative,
+
+    "TANH": tanh,
+    "TANH_DERIVATIVE": tanh_derivative,
+
+    "RELU": relu,
+    "RELU_DERIVATIVE": relu_derivative,
+
+    "LEAKY_RELU": leaky_relu,
+    "LEAKY_RELU_DERIVATIVE": leaky_relu_derivative,
+
+    "ELU": elu,
+    "ELU_DERIVATIVE": elu_derivative,
+
+    "SOFTPLUS": softplus,
+    "SOFTPLUS_DERIVATIVE": softplus_derivative,
+
+    "SOFTMAX": softmax,
+    "SOFTMAX_DERIVATIVE": softmax_derivative,
+}

nn_simulator/classes.py

@@ -0,0 +1,223 @@
+"""Module containing the dataclasses used in the neural network simulator."""
+from datetime import datetime
+from dataclasses import dataclass, field
+from . import activation_functions as af
+
+@dataclass
+class Architecture:
+    """
+    Store the architecture of a neural network.
+
+    Parameters
+    ----------
+    layers : sequence of tuple[int, str]
+        Network layers, each specified by a tuple containing the
+        number of neurons and the activation function name.
+    input_vector_size : int
+        Number of features in the input vector.
+
+    Attributes
+    ----------
+    input_vector_size : int
+        Number of features in the input vector.
+    layers : list of int
+        Number of neurons in each network layer.
+    act_funcs : list of str
+        Activation function name associated with each layer.
+
+    Raises
+    ------
+    ValueError
+        If an activation function is not supported.
+
+    Examples
+    --------
+    >>> architecture = Architecture(
+    ...     layers=[
+    ...         (3, "Linear"),
+    ...         (2, "Sigmoid"),
+    ...         (1, "SoftMax"),
+    ...     ],
+    ...     input_vector_size=2,
+    ... )
+    >>> architecture.layers
+    [3, 2, 1]
+    >>> architecture.act_funcs
+    ['Linear', 'Sigmoid', 'SoftMax']
+    """
+    input_vector_size: int
+    layers: list[tuple]
+    act_funcs: list = field(init=False, default_factory=list)
+
+    def __post_init__(self):
+        self.act_funcs = []
+        layer_specs = self.layers
+        self.layers = []
+
+        for layer in layer_specs:
+            act_func_name = layer[1].upper()
+            if act_func_name not in af.af_list:
+                raise ValueError(
+                    f"Activation function {layer[1]} is not supported. "
+                    f"Supported functions are: {af.af_list}"
+                )
+
+            self.act_funcs.append(act_func_name)
+            self.layers.append(layer[0])
+
+@dataclass
+class TrainingParameters:
+    """
+    Store the parameters used to train a neural network.
+
+    Parameters
+    ----------
+    bounds : tuple[float, float], default=(-1, 1)
+        Lower and upper bounds of the uniform distribution used
+        to initialize the network weights.
+    seed : int or None, default=None
+        Seed used to initialize the random number generator.
+        If None, a random seed is generated automatically.
+        The generated seed can be saved and reused to reproduce
+        the same initial weights, provided that the network
+        architecture and bounds remain unchanged.
+    epochs : int or None, default=None
+        Maximum number of training epochs. If None, no maximum
+        number of epochs is imposed.
+    batch_size : int or None, default=None
+        Number of training examples processed in each mini-batch.
+        If None, the entire training dataset is processed at once
+        (batch training). A value of 1 corresponds to online
+        training.
+    min_delta_loss : tuple[float, int] or None, default=None
+        Convergence criterion based on the absolute change in the
+        loss function between consecutive epochs. The tuple
+        contains (threshold, patience), where threshold is the
+        minimum required loss variation and patience is the number
+        of consecutive epochs for which the absolute loss variation
+        must remain below this threshold before training stops.
+        If None, this criterion is disabled.
+    min_gradient_normt : tuple[float, int] or None, default=None
+        Convergence criterion based on the norm of the weights
+        gradient. The tuple contains (threshold, patience), where
+        threshold is the minimum gradient norm and patience is the
+        number of consecutive epochs for which the gradient norm
+        must remain below this threshold before training stops.
+        If None, this criterion is disabled.
+    target_loss : float or None, default=None
+        Target value for the loss function. Training stops when
+        the loss reaches or falls below this threshold.
+        If None, this criterion is disabled.
+    
+    Raises
+    ------
+    ValueError
+        If no stopping criterion is specified.
+    """
+    bounds: tuple[float, float] = (-1, 1)
+    seed: int | None = None
+    batch_size: int | None = None
+    epochs: int | None = None
+    min_delta_loss: tuple[float, int] | None = None
+    min_gradient_norm: tuple[float, int] | None = None
+    target_loss: float | None = None
+
+    def __post_init__(self):
+        if (
+            self.epochs is None
+            and self.min_delta_loss is None
+            and self.min_gradient_norm is None
+            and self.target_loss is None
+        ):
+            raise ValueError(
+                "At least one stopping criterion must be specified."
+            )
+
+@dataclass
+class UpdateParameters:
+    """
+    Store the parameters used to update network weights.
+
+    Parameters
+    ----------
+    eta : float
+        Learning rate controlling the magnitude of weight updates.
+    alpha : float, default=0
+        Momentum coefficient used to incorporate previous updates.
+    lambda_ : float, default=0
+        Regularization coefficient controlling the strength
+        of the regularization term.
+    full_independence : bool, default=False
+        If True removes the contribution of the regularizatione term
+        in the total weights delta, making the three coefficient
+        totally independent.
+
+    Notes
+    -----
+    The weights update will be performed following:
+        delta_w = - eta * w_grad - lambda * w + alpha * delta_w_old
+        w_new = w + delta_w
+    if full_independence=False, or:
+        delta_w = - eta * w_grad + alpha * delta_w_old
+        w_new = w + delta_w - lambda * w
+    if full_independence=True.
+    """
+    eta: float
+    alpha: float = 0.
+    lambda_: float = 0.
+    full_independence: bool = False
+
+@dataclass
+class TrainingLog:
+    """
+    Store the results and metadata of a neural network training session.
+
+    Attributes
+    ----------
+    seed : int or None, default=None
+        Random seed used to initialize the network weights.
+    loss : list[float] or None, default=None
+        Loss values recorded during training, typically one per epoch.
+    elapsed_epoch : int or None, default=None
+        Number of epochs completed before training stopped.
+    stopping_cause : list[str] or None, default=None
+        Stopping criteria satisfied at the end of training. Multiple
+        criteria may be satisfied simultaneously.
+    training_start_stop : list[datetime, datetime] or None, default=None
+        Start and stop timestamps of the training session.
+    last_delta_loss : list[float] or None, default=None
+        Most recent changes in loss values, used to evaluate the
+        loss convergence criterion.
+    last_delta_weights_gradient : list[float] or None, default=None
+        Most recent changes in weight-gradient norms, used to evaluate
+        the gradient convergence criterion.
+
+    Notes
+    -----
+    The initial loss, final loss and elapsed training time can be
+    derived from the stored attributes.
+    """
+    seed: int | None = None
+    loss: list[float] | None = None
+    elapsed_epoch: int | None = None
+    stopping_cause: list[str] | None = None
+    training_start_stop: list[datetime, datetime] | None = None
+    delta_loss: list[float] | None = None
+    gradient_norms: list[float] | None = None
+
+    def __repr__(self):
+        return (
+            "TrainingLog(\n"
+            f"    seed={self.seed},\n"
+            f"    elapsed_epoch={self.elapsed_epoch},\n"
+            f"    elapsed_time={
+                (
+                self.training_start_stop[1] - self.training_start_stop[0]
+                ).total_seconds():.3f}s,\n"
+            f"    training_start={self.training_start_stop[0]},\n"
+            f"    training_stop={self.training_start_stop[1]},\n"
+            f"    stopping_cause={self.stopping_cause},\n"
+            f"    initial_loss={self.loss[0]},\n"
+            f"    final_loss={self.loss[-1]},\n"
+            ")"
+        )

nn_simulator/loss.py

@@ -0,0 +1,67 @@
+"""Module containing functions to compute errors and losses."""
+import numpy as np
+
+def compute_error(
+        output_vectors: list[np.ndarray],
+        target_vector: np.ndarray
+        ) -> np.ndarray:
+    """
+    Computes the residual error of a neural network prediction.
+
+    Parameters
+    ----------
+    output_vectors : list of np.ndarray
+        Output vector of each layer in the neural network
+        (output_vector[-1] represents the final prediction).
+    target_vector : np.ndarray
+        Target vector for the training example.
+
+    Returns
+    -------
+    error : np.ndarray
+        Difference between prediction and target.
+    """
+
+    # Automatically select the output layer's output vector from the list
+    if isinstance(output_vectors, (list, tuple)):
+        prediction = output_vectors[-1]
+    else:
+        prediction = output_vectors
+
+    return target_vector - prediction
+
+def compute_loss(
+        output_vectors: list[np.ndarray],
+        target_vector: np.ndarray,
+        error_vector: np.ndarray | None = None
+        ) -> float:
+    """
+    Computes the total empirical loss (HMSE) 
+    over a batch of examples.
+
+    Parameters
+    ----------
+    output_vectors : list of np.ndarray
+        Output vector of each layer in the neural network
+        (output_vector[-1] represents the final prediction).
+    target_vector : np.ndarray
+        Target vector for the training example.
+    error_vector : np.ndarray or None, default=None
+        Precomputed residual error vector.
+
+    Returns
+    -------
+    loss : float
+        Total loss computed as the Half Mean Squared Error (HMSE).
+    """
+
+    # Compute error_vector if it wasn't previously comptued
+    if error_vector is None:
+        error_vector = compute_error(
+            output_vectors=output_vectors,
+            target_vector=target_vector
+            )
+
+    loss = 0.5*(np.mean(np.sum(error_vector**2, axis=0)))
+
+    return loss

nn_simulator/nn_sim.py

@@ -0,0 +1,283 @@
+"""NN simulation main module. Contains the NeuralNetwork class."""
+import os
+from datetime import datetime
+import numpy as np
+from .classes import TrainingParameters, UpdateParameters, Architecture, TrainingLog
+from .propagation import forward_propagation
+from .utils import check_input
+from .training import Trainer
+
+class NeuralNetwork:
+    """
+    Class that implements a simulated neural network.
+    IN DEPT DESCRIPTION TO BE WRITTEN
+    IN THE FINAL IMPLEMENTATION SHOULD HAVE THE FOLLOWING PUBLIC METHODS:
+        - save_model()
+        - load_model()
+        - train()
+        - predict()
+    DECIDE HOW TO COMMENT CORRECTLY
+    """
+
+    def __init__(
+            self,
+            architecture: Architecture,
+            training_parameters: TrainingParameters = None,
+            update_parameters: UpdateParameters = None
+            ):
+        """
+        Constructor for the NeuralNetwork class.
+        Initialize the required attributes for the training and prediction phases.
+        DECIDE HOW TO COMMENT CORRECTLY
+        """
+        # Initialize nn architecture attribute
+        self.architecture = architecture
+
+        # Initialize layer weights matrices list
+        self.weights = [None] * len(self.architecture.layers)
+
+        # Initialize nn training/update parameters attributes
+        self.update_params = update_parameters
+        self.training_params = training_parameters
+
+        # Initialize training log attribute to None
+        self.training_log: TrainingLog = None
+
+    def __repr__(self) -> str:
+        weights_initialized = all(
+            w is not None for w in self.weights
+        )
+        weights_status = (
+            "initialized"
+            if weights_initialized
+            else "uninitialized"
+        )
+        lines = [
+            "NeuralNetwork(",
+            f"    architecture={self.architecture},",
+            f"    weights={weights_status},",
+        ]
+
+        if self.training_params is not None:
+            lines.append(
+                f"    training_params={self.training_params},"
+            )
+
+        if self.update_params is not None:
+            lines.append(
+                f"    update_params={self.update_params},"
+            )
+
+        if self.training_log is not None:
+            lines.append(
+                f"    training_log={self.training_log},"
+            )
+
+        lines.append(")")
+
+        return "\n".join(lines)
+
+    def save_model(
+            self,
+            model_name: str = None,
+            models_dir: str = "models"
+            ) -> str:
+        """
+        Saves the neural network model to a dedicated directory.
+
+        Parameters
+        ----------
+        model_name : str, default=None
+            Name of the directory where the model will be saved.
+            If None, a timestamp-based name is generated.
+
+        models_dir : str, default="models"
+            Parent directory where the model directory will be created.
+
+        Returns
+        -------
+        model_path : str
+            Path to the directory containing the saved model.
+        """
+
+        # Generate model name if not provided
+        if model_name is None:
+            model_name = (
+                f"model_{datetime.now().strftime('%Y%m%d_%H%M')}"
+            )
+
+        # Create model directory
+        model_path = os.path.join(models_dir, model_name)
+        os.makedirs(model_path, exist_ok=True)
+
+        # Save weights
+        np.save(
+            os.path.join(model_path, "weights.npy"),
+            np.array(self.weights, dtype=object),
+            allow_pickle=True
+        )
+
+        # Save architecture
+        np.save(
+            os.path.join(model_path, "architecture.npy"),
+            self.architecture,
+            allow_pickle=True
+        )
+
+        # Save training parameters
+        if self.training_params is not None:
+            np.save(
+                os.path.join(model_path, "training_parameters.npy"),
+                self.training_params,
+                allow_pickle=True
+            )
+
+        # Save update parameters
+        if self.update_params is not None:
+            np.save(
+                os.path.join(model_path, "update_parameters.npy"),
+                self.update_params,
+                allow_pickle=True
+            )
+
+        # Save training log
+        if self.training_log is not None:
+            np.save(
+                os.path.join(model_path, "training_log.npy"),
+                self.training_log,
+                allow_pickle=True
+            )
+
+        return model_path
+
+    @classmethod
+    def load_model(cls, model_path: str):
+        """
+        Loads a neural network model from a directory.
+
+        Parameters
+        ----------
+        model_path : str
+            Path to the directory containing the saved model.
+
+        Returns
+        -------
+        nn : NeuralNetwork
+            Loaded neural network instance.
+        """
+
+        # Load architecture
+        architecture = np.load(
+            os.path.join(model_path, "architecture.npy"),
+            allow_pickle=True
+        ).item()
+
+        # Initialize neural network
+        nn = cls(architecture=architecture)
+
+        # Load weights
+        nn.weights = np.load(
+            os.path.join(model_path, "weights.npy"),
+            allow_pickle=True
+        ).tolist()
+
+        # Load training parameters if available
+        training_params_path = os.path.join(
+            model_path, "training_parameters.npy"
+        )
+        if os.path.isfile(training_params_path):
+            nn.training_params = np.load(
+                training_params_path,
+                allow_pickle=True
+            ).item()
+
+        # Load update parameters if available
+        update_params_path = os.path.join(
+            model_path, "update_parameters.npy"
+        )
+        if os.path.isfile(update_params_path):
+            nn.update_params = np.load(
+                update_params_path,
+                allow_pickle=True
+            ).item()
+
+        return nn
+
+    def predict(
+            self,
+            input_vector: np.ndarray
+            ) -> np.ndarray:
+        """
+        Computes neural network prediction given one or more
+        input patterns.
+
+        Parameters
+        ----------
+        input_vector : np.ndarray
+            Input vector to be forwarded into the neural network.
+
+        Returns
+        -------
+        output_vector : np.ndarray
+            Contains the final neural network prediction.
+        """
+        # Should this also return the loss?
+
+        # Checks if the model has been trained or loaded from
+        # memory before attempting a prediction
+        if all(w is None for w in self.weights):
+            raise RuntimeError(
+                "Cannot perform prediction: the model has not been trained or " \
+                "loaded from memory."
+            )
+
+        # Check if the input vector is suitable
+        check_input(
+            input_vector=input_vector,
+            input_vector_size=self.architecture.input_vector_size
+        )
+
+        # Performs forward propagation in the neural network
+        output_vector = forward_propagation(
+            weights=self.weights,
+            act_funcs=self.architecture.act_funcs,
+            input_vector=input_vector
+        )[0][-1]
+
+        # Returns only the output layer output vector
+        # a.k.a. final prediction
+        return output_vector
+
+    def train(
+            self,
+            update_params: UpdateParameters,
+            training_params: TrainingParameters,
+            input_vector: np.ndarray,
+            target_vector: np.ndarray
+            ):
+
+        # Save update/training parameters to their realtive attributes
+        self.update_params = update_params
+        self.training_params = training_params
+
+        # Check if the input vector is suitable
+        check_input(
+            input_vector=input_vector,
+            input_vector_size=self.architecture.input_vector_size
+        )
+
+        tr = Trainer(
+            architecture=self.architecture,
+            update_params=update_params,
+            training_params=training_params,
+            weights=self.weights
+        )
+        self.training_log = tr.training_loop(
+            input_vector=input_vector,
+            target_vector=target_vector
+        )
+        self.log()
+
+    def log(self):
+        if self.training_log is not None:
+            print(self.training_log)