PyDLLib 1.0__py3-none-any.whl

DLL/DeepLearning/Model.py

@@ -0,0 +1,265 @@
+import torch
+from copy import deepcopy
+import pickle
+
+
+class Model:
+    """
+    The base model for a sequantial deep learning model. Uses a linear stack of layers to do forward- and backpropagation.
+
+    Args:
+        input_shape (tuple[int] | int): A tuple or an int containing the input shape of the model. The batch size should not be given as the first member of the tuple. For instance, if the input is of shape (n_sample, n_features), the input_shape should be n_features or if the input is of shape (n_samples, n_channels, width, heigth), the input_shape should be (n_channels, width, heigth).
+        data_type (torch.dtype, optional): The data type used by the model. Defaults to torch.float32.
+        device (torch.device, optional): The device of the model. Determines if the computation is made using the gpu or the cpu. Defaults to torch.device("cpu").
+    """
+    def __init__(self, input_shape, data_type=torch.float32, device=torch.device("cpu")):
+        if not isinstance(input_shape, tuple | int):
+            raise TypeError("input_shape must be a tuple of ints or an int. See documentation for examples.")
+        if not isinstance(data_type, torch.dtype):
+            raise TypeError("data_type must be an instance of torch.dtype.")
+        if not isinstance(device, torch.device):
+            raise TypeError('device must be either torch.device("cpu") or torch.device("cuda").')
+
+        from .Layers import Input
+        input_shape = (input_shape,) if isinstance(input_shape, int) else input_shape
+        self.layers = [Input(input_shape, device=device, data_type=data_type)]
+        self.optimiser = None
+        self.loss = None
+        self.data_type = data_type
+        self.device = device
+        self.training = False
+
+    def compile(self, optimiser=None, loss=None, metrics=("loss",), callbacks=tuple()):
+        """
+        Configures the model for training. Sets the optimiser and the loss function.
+
+        Args:
+            optimiser (:ref:`optimisers_section_label` | None, optional): The optimiser used for training the model. If None, the ADAM optimiser is used.
+            loss (:ref:`losses_section_label` | None, optional): The loss function used for training the model. If None, the MSE loss is used.
+            metrics (tuple[str], optional): The metrics that will be tracked during training. Defaults to ("loss").
+            callbacks (tuple[:ref:`callbacks_section_label`], optional): The callbacks used by the model. Defaults to ().
+        Raises:
+            TypeError: If the optimiser is not from DLL.DeepLearning.Optimisers, the loss is not from DLL.DeepLearning.Losses or the metrics is not a tuple or a list of strings.
+        """
+
+        from .Losses._BaseLoss import BaseLoss
+        from .Optimisers._BaseOptimiser import BaseOptimiser
+        from .Callbacks import Callback
+        from .Losses import MSE
+        from .Optimisers import ADAM
+
+        if not isinstance(optimiser, BaseOptimiser) and optimiser is not None:
+            raise TypeError("optimiser must be from DLL.DeepLearning.Optimisers")
+        if not isinstance(loss, BaseLoss) and loss is not None:
+            raise TypeError("loss must be from DLL.DeepLearning.Losses")
+        if not isinstance(metrics, list | tuple):
+            raise TypeError("metrics must be a list or a tuple containing the strings of wanted metrics.")
+        if any([not isinstance(callback, Callback) for callback in callbacks]):
+            raise TypeError("callbacks must be a tuple of callback objects.")
+
+        self.optimiser = optimiser if optimiser is not None else ADAM()
+        parameters = [parameter for layer in self.layers for parameter in layer.get_parameters()]
+        self.optimiser.initialise_parameters(parameters)
+        self.loss = loss if loss is not None else MSE()
+        self.metrics = metrics
+        self.callbacks = callbacks
+
+    def clone(self):
+        """
+        Returns a copy of the same model.
+        """
+        return deepcopy(self)
+
+    def add(self, layer):
+        """
+        Adds and initializes a layer to the model.
+
+        Args:
+            layer (:ref:`layers_section_label`): The layer that is added to the model.
+        """
+        from .Layers._BaseLayer import BaseLayer
+        if not isinstance(layer, BaseLayer):
+            raise TypeError("layer must be from DLL.Deeplearning.Layers")
+
+        layer.initialise_layer(input_shape=self.layers[-1].output_shape, data_type=self.data_type, device=self.device)
+        self.layers.append(layer)
+    
+    def train(self):
+        """
+        Change the state of the model to training state. Effects some layers, but other layers remain the same.
+        """
+        self.training = True
+
+    def eval(self):
+        """
+        Change the state of the model to evaluation state. Effects some layers, but other layers remain the same.
+        """
+        self.training = False
+    
+    def summary(self):
+        """
+        Prints the summary of the model containing its architecture and the number of parameters of the model.
+        """
+        print("Model summary:")
+        total_params = 0
+        for layer in self.layers:
+            print(layer.summary())
+            total_params += layer.get_nparams()
+        print(f"Total number of parameters: {total_params}")
+    
+    def __str__(self):
+        message = "Model summary:"
+        total_params = 0
+        for layer in self.layers:
+            message += layer.summary() + "\n"
+            total_params += layer.get_nparams()
+        message += f"Total number of parameters: {total_params}"
+        return message
+
+    def predict(self, X):
+        """
+        Applies the fitted Model to the input data, predicting wanted values by forward propagation.
+
+        Args:
+            X (torch.Tensor of shape (n_samples, *input_shape)): The input data that goes through the model by forward propagation.
+        
+        Raises:
+            NotCompiledError: If the Model has not been compiled before predicting.
+
+        Returns:
+            torch.Tensor of shape (n_samples, *last_layer.output_shape)): The predictions made by the model.
+        """
+        from ..Exceptions import NotCompiledError
+        if self.optimiser is None:
+            raise NotCompiledError("Model.compile() must be called before predicting.")
+
+        current = X
+        for layer in self.layers:
+            current = layer.forward(current, training=self.training)
+        return current
+    
+    def backward(self, initial_gradient):
+        """
+        :meta private:
+        """
+        reversedLayers = reversed(self.layers)
+        gradient = initial_gradient
+        for layer in reversedLayers:
+            gradient = layer.backward(gradient, training=self.training)
+        return gradient
+
+    def fit(self, X, Y, val_data=None, epochs=10, callback_frequency=1, batch_size=None, shuffle_every_epoch=True, shuffle_data=True, verbose=False):
+        """
+        Fits the LogisticRegression model to the input data by minimizing the cross entropy loss (logistic loss).
+
+        Args:
+            X (torch.Tensor of shape (n_samples, *first_layer.input_shape)): The input data, of correct shape determined by the input_shape of the model.
+            y (torch.Tensor): The targets corresponding to each sample.
+            val_data (tuple[X_val, y_val] | None, optional): Optional validation samples. Must have the same remaining dimensions than X and y apart from n_samples. If None, no validation data is used. Defaults to None.
+            epochs (int, optional): The number of training iterations. Must be a positive integer. Defaults to 10.
+            callback_frequency (int, optional): The number of iterations between printing info from training. Must be a positive integer. Defaults to 1, which means that every iteration, info is printed assuming verbose=True.
+            batch_size (int | None, optional): The batch size used in training. Must be a positive integer. If None, every sample is used for every gradient calculation. Defaults to None.
+            shuffle_every_epoch (bool, optional): If True, shuffles the order of the samples every epoch. Defaults to True.
+            shuffle_data (bool, optional): If True, shuffles data before the training.
+            verbose (bool, optional): If True, prints info of the chosen metrics during training. Defaults to False.
+        Returns:
+            history (dict[str, list]): A dictionary tracking the evolution of selected metrics at intervals defined by callback_frequency. If training was not stopped early, each metric is floor(epochs / callback_frequency) long.
+        Raises:
+            TypeError: If the input matrix or the target matrix is not a PyTorch tensor or if other parameters are of wrong type.
+            ValueError: If the input matrix or the target matrix is not the correct shape or if other parameters have incorrect values.
+        """
+        if not isinstance(X, torch.Tensor) or not isinstance(Y, torch.Tensor):
+            raise TypeError("The input matrix and the target matrix must be a PyTorch tensor.")
+        if X.shape[1:] != self.layers[0].input_shape:
+            raise ValueError("The input matrix must have the same shape as input_shape.")
+        if len(Y) != len(X) or Y.shape[1:] != self.layers[-1].output_shape:
+            raise ValueError(f"The targets must have the same shape as the output_shape of the last layer with the same number of samples as the input data {Y.shape[1:], self.layers[-1].output_shape}.")
+        if not isinstance(val_data, list | tuple) and val_data is not None:
+            raise TypeError("val_data must either be a tuple containing validation samples or None.")
+        if isinstance(val_data, list | tuple) and len(val_data) != 2:
+            raise ValueError("val_data must contain both X_val and y_val.")
+        if isinstance(val_data, list | tuple) and len(val_data) == 2 and (val_data[0].shape[1:] != X.shape[1:] or val_data[1].shape[1:] != Y.shape[1:] or len(val_data[0]) != len(val_data[1])):
+            raise ValueError("X_val and y_val must be of correct shape.")
+        if not isinstance(epochs, int) or epochs <= 0:
+            raise ValueError("epochs must be a positive integer.")
+        if not isinstance(callback_frequency, int) or callback_frequency <= 0:
+            raise ValueError("callback_frequency must be a positive integer.")
+        if (not isinstance(batch_size, int) or batch_size <= 0) and batch_size is not None:
+            raise ValueError("batch_size must be a positive integer.")
+        if not isinstance(shuffle_every_epoch, bool):
+            raise TypeError("shuffle_every_epoch must be a boolean.")
+        if not isinstance(shuffle_data, bool):
+            raise TypeError("shuffle_data must be a boolean.")
+        if not isinstance(verbose, bool):
+            raise TypeError("verbose must be a boolean.")
+        
+        from ..Data import DataReader
+        from ..Data.Metrics import calculate_metrics, _round_dictionary
+
+        history = {metric: [] for metric in self.metrics}
+        batch_size = len(X) if batch_size is None else batch_size
+        data_reader = DataReader(X, Y, batch_size=batch_size, shuffle=shuffle_data, shuffle_every_epoch=shuffle_every_epoch)
+
+        train_metrics = [metric for metric in self.metrics if metric[:4] != "val_"]
+        val_metrics = [metric for metric in self.metrics if metric[:4] == "val_"]
+
+        self.stop_training = False  # boolean for callbacks to stop the training
+        for callback in self.callbacks:
+            callback.set_model(self)
+            callback.on_train_start()
+
+        for epoch in range(epochs):
+            if self.stop_training:
+                break
+            self.train()
+            for x, y in data_reader.get_data():
+                self.optimiser.zero_grad()
+                predictions = self.predict(x)
+                initial_gradient = self.loss.gradient(predictions, y)
+                self.backward(initial_gradient)
+                self.optimiser.update_parameters()
+                for callback in self.callbacks:
+                    callback.on_batch_end(epoch)
+
+            self.eval()
+            if epoch % callback_frequency == 0:
+                values = calculate_metrics(data=(self.predict(X), Y), metrics=train_metrics, loss=self.loss.loss)
+                if val_data is not None:
+                    val_values = calculate_metrics(data=(self.predict(val_data[0]), val_data[1]), metrics=val_metrics, loss=self.loss.loss, validation=True)
+                    values |= val_values
+                for metric, value in values.items():
+                    history[metric].append(value)
+                if verbose: print(f"Epoch: {epoch + 1} - Metrics: {_round_dictionary(values)}")
+            
+            for callback in self.callbacks:
+                callback.on_epoch_end(epoch, values)
+        
+        for callback in self.callbacks:
+            callback.on_train_end()
+
+        return history
+
+
+def save_model(model, filepath="./model.pkl"):
+    """Saves a model using pickle serialization."""
+    try:
+        with open(filepath, "wb") as f:
+            pickle.dump(model, f)
+    except TypeError as e:
+        print(f"Error: The model is not serializable. Saving failed.")
+        raise
+    except OSError as e:
+        print(f"File error while saving the model.")
+        raise
+
+def load_model(filepath="./model.pkl"):
+    """Loads a model from a pickle file."""
+    try:
+        with open(filepath, "rb") as f:
+            return pickle.load(f)
+    except (pickle.UnpicklingError, EOFError, AttributeError, ModuleNotFoundError, TypeError) as e:
+        print(f"Error loading the model file.")
+        raise
+    except OSError as e:
+        print(f"File error while loading the model.")
+        raise

DLL/DeepLearning/Optimisers/_ADADELTA.py

@@ -0,0 +1,53 @@
+import torch
+
+from ._BaseOptimiser import BaseOptimiser
+
+
+class ADADELTA(BaseOptimiser):
+    """
+    The adadelta optimiser. A first order method and therefore does not use information on second gradients, i.e. the hessian matrix. Hence, does not require a lot of memory.
+
+    Args:
+        learning_rate (float, optional): The learning rate of the optimiser. Must be positive. Defaults to 0.001.
+        rho (float, optional): Determines how long the previous gradients affect the current step direction. Must be in range [0, 1). Defaults to 0.9.
+        weight_decay (float, optional): Determines if regularisation should be applied to the weights. Must be in range [0, 1). Defaults to 0.
+    """
+    def __init__(self, learning_rate=0.001, rho=0.9, weight_decay=0):
+        if not isinstance(learning_rate, int | float) or learning_rate <= 0:
+            raise ValueError("learning_rate must be a positive real number.")
+        if not isinstance(rho, int | float) or rho < 0 or rho >= 1:
+            raise ValueError("rho must be in range [0, 1).")
+        if not isinstance(weight_decay, int | float) or weight_decay < 0 or weight_decay >= 1:
+            raise ValueError("weight_decay must be in range [0, 1).")
+
+        self.learning_rate = learning_rate
+        self.rho = rho
+        self.weight_decay = weight_decay
+    
+    def initialise_parameters(self, model_parameters):
+        """
+        Initialises the optimiser with the parameters that need to be optimised.
+
+        Args:
+            model_parameters (list[torch.Tensor]): The parameters that will be optimised. Must be a list or a tuple of torch tensors.
+        """
+
+        if not isinstance(model_parameters, list | tuple):
+            raise TypeError("model_parameters must be a list or a tuple of torch tensors.")
+
+        self.model_parameters = model_parameters
+        self.square_avg = [torch.zeros_like(param) for param in model_parameters]
+        self.accumulate_variables = [torch.zeros_like(param) for param in model_parameters]
+        self.t = 0
+    
+    def update_parameters(self):
+        """
+        Takes a step towards the optimum for each parameter.
+        """
+        self.t += 1
+        for i, parameter in enumerate(self.model_parameters):
+            if self.weight_decay > 0: parameter.grad += self.weight_decay * parameter
+            self.square_avg[i] = self.square_avg[i] * self.rho + parameter.grad ** 2 * (1 - self.rho)
+            delta_x = parameter.grad * torch.sqrt(self.accumulate_variables[i] + 1e-10) / torch.sqrt(self.square_avg[i] + 1e-10)
+            self.accumulate_variables[i] = self.accumulate_variables[i] * self.rho + delta_x ** 2 * (1 - self.rho)
+            parameter -= self.learning_rate * delta_x

DLL/DeepLearning/Optimisers/_ADAGRAD.py

@@ -0,0 +1,51 @@
+import torch
+
+from ._BaseOptimiser import BaseOptimiser
+
+
+class ADAGRAD(BaseOptimiser):
+    """
+    The adaptive gradient optimiser. A first order method and therefore does not use information on second gradients, i.e. the hessian matrix. Hence, does not require a lot of memory.
+
+    Args:
+        learning_rate (float, optional): The learning rate of the optimiser. Must be positive. Defaults to 0.001.
+        lr_decay (float, optional): Determines how fast the learning rate decreases. Must be positive. Defaults to 0.
+        weight_decay (float, optional): Determines if regularisation should be applied to the weights. Must be in range [0, 1). Defaults to 0.
+    """
+    def __init__(self, learning_rate=0.001, lr_decay=0, weight_decay=0):
+        if not isinstance(learning_rate, int | float) or learning_rate <= 0:
+            raise ValueError("learning_rate must be a positive real number.")
+        if not isinstance(lr_decay, int | float) or lr_decay < 0:
+            raise ValueError("lr_decay must be positive.")
+        if not isinstance(weight_decay, int | float) or weight_decay < 0 or weight_decay >= 1:
+            raise ValueError("weight_decay must be in range [0, 1).")
+        
+        self.learning_rate = learning_rate
+        self.lr_decay = lr_decay
+        self.weight_decay = weight_decay
+    
+    def initialise_parameters(self, model_parameters):
+        """
+        Initialises the optimiser with the parameters that need to be optimised.
+
+        Args:
+            model_parameters (list[torch.Tensor]): The parameters that will be optimised. Must be a list or a tuple of torch tensors.
+        """
+
+        if not isinstance(model_parameters, list | tuple):
+            raise TypeError("model_parameters must be a list or a tuple of torch tensors.")
+
+        self.model_parameters = model_parameters
+        self.state_sum = [torch.zeros_like(param) for param in model_parameters]
+        self.t = 0
+    
+    def update_parameters(self):
+        """
+        Takes a step towards the optimum for each parameter.
+        """
+        self.t += 1
+        for i, parameter in enumerate(self.model_parameters):
+            learning_rate = self.learning_rate / (1 + (self.t - 1) * self.lr_decay)
+            if self.weight_decay > 0: parameter.grad += self.weight_decay * parameter
+            self.state_sum[i] += parameter.grad ** 2
+            parameter -= learning_rate * parameter.grad / (torch.sqrt(self.state_sum[i]) + 1e-10)

DLL/DeepLearning/Optimisers/_ADAM.py

@@ -0,0 +1,65 @@
+import torch
+
+from ._BaseOptimiser import BaseOptimiser
+
+
+class ADAM(BaseOptimiser):
+    """
+    The adaptive moment estimation optimiser. Is very robust and does not require a lot of tuning it's hyperparameters. A first order method and therefore does not use information on second gradients, i.e. the hessian matrix. Hence, does not require a lot of memory. Is based on algorithm 1 on `this paper <https://arxiv.org/pdf/1412.6980>`_.
+
+    Args:
+        learning_rate (float, optional): The learning rate of the optimiser. Must be positive. Defaults to 0.001.
+        beta1 (float, optional): Determines how long the previous gradients affect the current step direction. Must be in range [0, 1). Defaults to 0.9.
+        beta2 (float, optional): Determines how long the previous squared gradients affect the current step direction. Must be in range [0, 1). Defaults to 0.999.
+        weight_decay (float, optional): Determines if regularisation should be applied to the weights. Must be in range [0, 1). Defaults to 0.
+        amsgrad (bool, optional): Determines if a modified version of the algorithm is used. Defaults to False.
+    """
+    def __init__(self, learning_rate=0.001, beta1=0.9, beta2=0.999, weight_decay=0, amsgrad=False):
+        if not isinstance(learning_rate, int | float) or learning_rate <= 0:
+            raise ValueError("learning_rate must be a positive real number.")
+        if not isinstance(beta1, int | float) or beta1 < 0 or beta1 >= 1:
+            raise ValueError("momentum must be in range [0, 1).")
+        if not isinstance(beta2, int | float) or beta2 < 0 or beta2 >= 1:
+            raise ValueError("momentum must be in range [0, 1).")
+        if not isinstance(weight_decay, int | float) or weight_decay < 0 or weight_decay >= 1:
+            raise ValueError("weight_decay must be in range [0, 1).")
+        
+        self.learning_rate = learning_rate
+        self.beta1 = beta1
+        self.beta2 = beta2
+        self.weight_decay = weight_decay
+        self.amsgrad = amsgrad
+    
+    def initialise_parameters(self, model_parameters):
+        """
+        Initialises the optimiser with the parameters that need to be optimised.
+
+        Args:
+            model_parameters (list[torch.Tensor]): The parameters that will be optimised. Must be a list or a tuple of torch tensors.
+        """
+
+        if not isinstance(model_parameters, list | tuple):
+            raise TypeError("model_parameters must be a list or a tuple of torch tensors.")
+
+        self.model_parameters = model_parameters
+        self.m = [torch.zeros_like(parameter) for parameter in self.model_parameters]
+        self.v = [torch.zeros_like(parameter) for parameter in self.model_parameters]
+        self.v_max = [torch.zeros_like(parameter) for parameter in self.model_parameters]
+        self.t = 0
+    
+    def update_parameters(self):
+        """
+        Takes a step towards the optimum for each parameter.
+        """
+        self.t += 1
+        for i, parameter in enumerate(self.model_parameters):
+            if self.weight_decay > 0: parameter.grad += self.weight_decay * parameter
+            self.m[i] = self.beta1 * self.m[i] + (1 - self.beta1) * parameter.grad
+            self.v[i] = self.beta2 * self.v[i] + (1 - self.beta2) * parameter.grad ** 2
+            m_hat = self.m[i] / (1 - self.beta1 ** self.t)
+            v_hat = self.v[i] / (1 - self.beta2 ** self.t)
+            if self.amsgrad:
+                self.v_max[i] = torch.maximum(v_hat, self.v_max[i])
+                parameter -= self.learning_rate * m_hat / (torch.sqrt(self.v_max[i]) + 1e-10)
+            else:
+                parameter -= self.learning_rate * m_hat / (torch.sqrt(v_hat) + 1e-10)

DLL/DeepLearning/Optimisers/_BaseOptimiser.py

@@ -0,0 +1,20 @@
+import torch
+
+from abc import ABC, abstractmethod
+
+
+class BaseOptimiser(ABC):
+    """
+    :meta private:
+    """
+    @abstractmethod
+    def initialise_parameters(self, model_parameters):
+        pass
+    
+    @abstractmethod
+    def update_parameters(self):
+        pass
+
+    def zero_grad(self):
+        for param in self.model_parameters:
+            param.grad = torch.zeros_like(param)

DLL/DeepLearning/Optimisers/_LBFGS.py

@@ -0,0 +1,120 @@
+import torch
+from collections import deque
+
+from ._BaseOptimiser import BaseOptimiser
+
+
+class LBFGS(BaseOptimiser):
+    """
+    Limited-memory Broyden-Fletcher-Goldfarb-Shanno optimiser. A second order method and approximates the hessian matrix using changes in position and gradient. Hence, requires more memory than first order methods.
+
+    Args:
+        loss (Callable[[], float]): The target function. For a deep learning model, one could use eg. lambda: model.loss.loss(model.predict(x_train), y_train).
+        learning_rate (float, optional): The learning rate of the optimiser. Must be positive. Defaults to 0.001.
+        history_size (int, optional): The number of old changes in position and gradient stored. Must be a non-negative integer. Defaults to 10.
+        maxiterls (int, optional): The maximum number of iterations in the line search. Must be a non-negative integer. Defaults to 20.
+    """
+    def __init__(self, loss, learning_rate=0.001, history_size=10, maxiterls=20):
+        if not isinstance(learning_rate, int | float) or learning_rate <= 0:
+            raise ValueError("learning_rate must be a positive real number.")
+        if not isinstance(history_size, int) or history_size < 0:
+            raise ValueError("history_size must be a non-negative integer.")
+        if not isinstance(maxiterls, int) or maxiterls < 0:
+            raise ValueError("maxiterls must be a non-negative integer.")
+        
+        self.loss = loss
+        self.learning_rate = learning_rate
+        self.history_size = history_size
+        self.maxiterls = maxiterls
+    
+    def initialise_parameters(self, model_parameters):
+        """
+        Initialises the optimiser with the parameters that need to be optimised.
+
+        Args:
+            model_parameters (list[torch.Tensor]): The parameters that will be optimised. Must be a list or a tuple of torch tensors.
+        """
+
+        if not isinstance(model_parameters, list | tuple):
+            raise TypeError("model_parameters must be a list or a tuple of torch tensors.")
+
+        self.model_parameters = model_parameters
+        self.s_history = [deque([], maxlen=self.history_size) for _ in model_parameters]
+        self.y_history = [deque([], maxlen=self.history_size) for _ in model_parameters]
+        self.prevs = [param.clone() for param in model_parameters]
+        self.prev_Bs = [torch.ones_like(param.flatten()) for param in model_parameters]
+    
+    def update_parameters(self):
+        """
+        Takes a step towards the optimum for each parameter.
+        """
+
+        for i, param in enumerate(self.model_parameters):
+            if param.grad is None:
+                continue
+
+            current = param.clone()
+            current.grad = param.grad.clone()
+            s = current - self.prevs[i]
+            y = current.grad - self.prevs[i].grad if self.prevs[i].grad is not None else torch.zeros_like(current.grad)
+            
+            if s.flatten() @ y.flatten() > 1e-8:
+                self.s_history[i].append(s)  # automatically pops the oldest values as maxlen is spesified for the deque
+                self.y_history[i].append(y)  # automatically pops the oldest values as maxlen is spesified for the deque
+
+            Bs = self.learning_rate * self._recursion_two_loops(i)
+
+            learning_rate = self._line_search(param, Bs)
+            param -= learning_rate * Bs.view_as(param)
+            
+            self.prevs[i] = current
+            self.prev_Bs[i] = Bs
+    
+    def _recursion_two_loops(self, i):
+        q = self.model_parameters[i].grad.clone().flatten()
+        alphas = torch.zeros(self.history_size)
+        rhos = torch.tensor([1.0 / (self.y_history[i][j].flatten() @ self.s_history[i][j].flatten()) for j in range(self.history_size) if len(self.s_history[i]) > j])
+
+        for j, (s, y) in enumerate(zip(reversed(self.s_history[i]), reversed(self.y_history[i]))):
+            j = len(self.s_history[i]) - j - 1
+            s, y = s.flatten(), y.flatten()
+            alphas[j] = rhos[j] * (s @ q)
+            q -= alphas[j] * y
+        
+        if len(self.s_history[i]) > 0:
+            last_s, last_y = self.s_history[i][-1].flatten(), self.y_history[i][-1].flatten()
+            Hk_0 = (last_s @ last_y) / last_y @ last_y
+        else:
+            Hk_0 = 1
+        r = Hk_0 * q
+
+        for j, (s, y) in enumerate(zip(self.s_history[i], self.y_history[i])):
+            s, y = s.flatten(), y.flatten()
+            beta = rhos[j] * (y @ r)
+            r += (alphas[j] - beta) * s
+        return r
+    
+    def _line_search(self, param, direction):
+        step = 1.0
+        initial_func_value = self.loss()
+        grad_dot_direction = torch.sum(param.grad.flatten() * direction)
+        orig_param = param.clone()
+        c = 1e-4
+        
+        iter = 0
+        while iter < self.maxiterls:
+            new_param_value = orig_param - step * direction.view_as(orig_param)
+            param.data = new_param_value
+            new_func_value = self.loss()
+            
+            # Does not include the other wolfe condition as it requires us to calculate the gradient of the parameter
+            # at the new point, which is only possible to calculate using finite differences with current architecture.
+            if new_func_value <= initial_func_value + c * step * grad_dot_direction:
+                break
+            else:
+                step *= 0.5
+            
+            iter += 1
+        
+        param.data = orig_param.data
+        return step

DLL/DeepLearning/Optimisers/_RMSPROP.py

@@ -0,0 +1,62 @@
+import torch
+
+from ._BaseOptimiser import BaseOptimiser
+
+
+class RMSPROP(BaseOptimiser):
+    """
+    The adaptive moment estimation optimiser. Is very robust and does not require a lot of tuning it's hyperparameters. A first order method and therefore does not use information on second gradients, i.e. the hessian matrix. Hence, does not require a lot of memory.
+
+    Args:
+        learning_rate (float, optional): The learning rate of the optimiser. Must be positive. Defaults to 0.001.
+        alpha (float, optional): A smoothing constant. Defaults to 0.99.
+        momentum (float, optional): Determines how long the previous gradients affect the current direction. Must be in range [0, 1). Defaults to 0.
+        weight_decay (float, optional): Determines if regularisation should be applied to the weights. Must be in range [0, 1). Defaults to 0.
+        centered (bool, optional): Determines if a centered version of the algorithm is used. Defaults to False.
+    """
+    def __init__(self, learning_rate=0.001, alpha=0.99, momentum=0, weight_decay=0, centered=False):
+        if not isinstance(learning_rate, int | float) or learning_rate <= 0:
+            raise ValueError("learning_rate must be a positive real number.")
+        if not isinstance(momentum, int | float) or momentum < 0 or momentum >= 1:
+            raise ValueError("momentum must be in range [0, 1).")
+        if not isinstance(weight_decay, int | float) or weight_decay < 0 or weight_decay >= 1:
+            raise ValueError("weight_decay must be in range [0, 1).")
+        
+        self.learning_rate = learning_rate
+        self.alpha = alpha
+        self.momentum = momentum
+        self.weight_decay = weight_decay
+        self.centered = centered
+    
+    def initialise_parameters(self, model_parameters):
+        """
+        Initialises the optimiser with the parameters that need to be optimised.
+
+        Args:
+            model_parameters (list[torch.Tensor]): The parameters that will be optimised. Must be a list or a tuple of torch tensors.
+        """
+
+        if not isinstance(model_parameters, list | tuple):
+            raise TypeError("model_parameters must be a list or a tuple of torch tensors.")
+
+        self.model_parameters = model_parameters
+        self.square_average = [torch.zeros_like(parameter) for parameter in self.model_parameters]
+        self.buffer = [torch.zeros_like(parameter) for parameter in self.model_parameters]
+        self.g_average = [torch.zeros_like(parameter) for parameter in self.model_parameters]
+    
+    def update_parameters(self):
+        """
+        Takes a step towards the optimum for each parameter.
+        """
+        for i, parameter in enumerate(self.model_parameters):
+            if self.weight_decay > 0: parameter.grad += self.weight_decay * parameter
+            self.square_average[i] = self.alpha * self.square_average[i] + (1 - self.alpha) * parameter.grad ** 2
+            v_tilde = self.square_average[i]
+            if self.centered:
+                self.g_average[i] = self.alpha * self.g_average[i] + (1 - self.alpha) * parameter.grad
+                v_tilde -= self.g_average[i] ** 2
+            if self.momentum > 0:
+                self.buffer[i] = self.momentum * self.buffer[i] + parameter.grad / (torch.sqrt(v_tilde) + 1e-10)
+                parameter -= self.learning_rate * self.buffer[i]
+            else:
+                parameter -= self.learning_rate * parameter.grad / (torch.sqrt(v_tilde) + 1e-10)