PyDLLib 1.0__py3-none-any.whl

DLL/DeepLearning/Callbacks/_ReduceLROnPlateau.py

@@ -0,0 +1,90 @@
+from warnings import warn
+
+from . import Callback
+from ...Exceptions import NotCompiledError
+
+
+class ReduceLROnPlateau(Callback):
+    """
+    The reduce learning rate on plateau callback. Reduces the learning rate of the model optimiser if no improvement is made.
+
+    Args:
+        monitor (str, optional): The quantity, which is monitored. Must be in the metrics of the model. Defaults to "val_loss".
+        patience (int, optional): The number of epochs the model waits without improvement. Must be a positive integer. Defaults to 0.
+        mode (str, optional): Determines if the monitored metric better if it is maximized or minimized. Must be in ["min", "max"]. Defaults to "min".
+        warmup_length (int, optional): Determines how many epochs are trained on the original learning rate. Must be non-negative. Defaults to 0.
+        factor (float, optional): The learning rate is multiplied by this factor when training. Must be in range (0, 1). Defaults to 0.5.
+        verbose (bool, optional): Determines if information about the callbacks is printed. Must be a boolean. Defaults to False.
+    """
+    def __init__(self, monitor="val_loss", patience=0, mode="min", warmup_length=0, factor=0.5, verbose=False):
+        if not isinstance(patience, int) or patience <= 0:
+            raise ValueError("patience must be a positive integer.")
+        if mode not in ["min", "max"]:
+            raise ValueError('mode must be in ["min", "max"].')
+        if not isinstance(warmup_length, int) or warmup_length < 0:
+            raise ValueError("warmup_length must be non-negative.")
+        if not isinstance(factor, float) or factor <= 0 or factor >= 1:
+            raise ValueError("factor must be in range (0, 1).")
+        if not isinstance(verbose, bool):
+            raise TypeError("verbose must be a boolean.")
+
+        self.monitor = monitor
+        self.patience = patience
+        self.mode = 1 if mode == "min" else -1
+        self.warmup_length = warmup_length
+        self.factor = factor
+        self.verbose = verbose
+
+    def set_model(self, model):
+        """
+        Lets the callback know about the chosen model. Is automatically called when using Model.fit()
+
+        Args:
+            Model (:ref:`models_section_label`): The chosen model.
+        """
+        if not hasattr(model, "metrics") or self.monitor not in model.metrics:
+            raise ValueError("monitor must be in the metrics of the model.")
+        self.model = model
+
+    def on_train_start(self):
+        """
+        Initializes the needed attributes.
+
+        Raises:
+            NotCompiledError: callback.set_model must be called before the training starts
+        """
+        if not hasattr(self, "model"):
+            raise NotCompiledError("callback.set_model must be called before the training starts.")
+        if not hasattr(self.model.optimiser, "learning_rate"):
+            warn("The chosen optimizer has no learning_rate attribute. The ReduceLROnPlateau callback has no effect.")
+
+        self.wait = 0
+        self.best_value = float("inf")
+
+    def on_epoch_end(self, epoch, metrics):
+        """
+        Calculates if the learning rate should be removed.
+
+        Args:
+            epoch (int): The current epoch.
+            metrics (dict[str, float]): The values of the chosen metrics of the last epoch.
+
+        Raises:
+            NotCompiledError: callback.set_model must be called before the training starts
+        """
+        if not hasattr(self, "model"):
+            raise NotCompiledError("callback.set_model must be called before the training starts.")
+
+        value = metrics[self.monitor]
+        if self.mode * value < self.best_value:
+            self.wait = 0
+            self.best_value = self.mode * value
+        elif epoch > self.warmup_length:
+            self.wait += 1
+            if self.wait >= self.patience:
+                new_lr_message = None
+                if hasattr(self.model.optimiser, "learning_rate"):
+                    self.wait = 0
+                    self.model.optimiser.learning_rate *= self.factor
+                    new_lr_message = self.model.optimiser.learning_rate
+                    if self.verbose: print(f"Learning rate reduced at epoch {epoch + 1} after {self.patience} epochs of no improvement. The monitored metric is currently {value:.3f} and the best model has the value of {self.mode * self.best_value:.3f}. The current learning rate is {new_lr_message}.")

DLL/DeepLearning/Callbacks/__init__.py

@@ -0,0 +1,4 @@
+from ._BaseCallback import Callback
+from ._EarlyStopping import EarlyStopping
+from ._BackUp import BackUp
+from ._ReduceLROnPlateau import ReduceLROnPlateau

DLL/DeepLearning/Initialisers/_Initialiser.py

@@ -0,0 +1,24 @@
+from abc import ABC, abstractmethod
+
+
+class Initialiser(ABC):
+    @abstractmethod
+    def initialise(self, shape):
+        pass
+
+    def _get_dims(self, shape):
+        shape = tuple(shape)
+        if len(shape) < 1:
+            input_dim = output_dim = 1
+        elif len(shape) == 1:
+            input_dim = output_dim = shape[0]
+        elif len(shape) == 2:
+            input_dim = shape[0]
+            output_dim = shape[0]
+        else:  # convolutional layers (output_depth, input_depth, ...)
+            product = 1
+            for dim in shape[2:]:
+                product *= dim
+            input_dim = product * shape[1]
+            output_dim = product * shape[0]
+        return int(input_dim), int(output_dim)

DLL/DeepLearning/Initialisers/_Kaiming_He.py

@@ -0,0 +1,55 @@
+import torch
+from math import sqrt
+
+from ._Initialiser import Initialiser
+
+
+class Kaiming_Uniform(Initialiser):
+    """
+    The Kaiming He uniform initialiser. Kaiming He initialiser should be used for the ReLU or any other activation, which is nonlinear close to origin.
+
+    Args:
+        mode (str, optional): Determines if the variance is constant in the forward or backward propagation. If "input_dim", variance is constant in forward propagation, while if "output_dim", the variance is constant in back propagation. Defaults to "input_dim".
+    """
+    def __init__(self, mode="input_dim"):
+        if mode not in ["input_dim", "output_dim"]:
+            raise ValueError('mode must be one of "input_dim" or "output_dim".')
+        self.mode = mode
+
+    def initialise(self, shape, data_type=torch.float32, device=torch.device("cpu")):
+        """
+        Initialises a tensor of the wanted shape with values in :math:`U(-a, a)`, where :math:`a = \\sqrt{\\frac{6}{d}}`.
+
+        Args:
+            shape (torch.Size): The shape of the wanted tensor.
+            data_type (torch.dtype, optional): The data type used in the returned tensor. Defaults to torch.float32.
+            device (torch.device, optional): The device of the tensor. Determines if the computation is made using the gpu or the cpu. Defaults to torch.device("cpu").
+        """
+        input_dim, output_dim = self._get_dims(shape)
+        a = sqrt(6/(input_dim if self.mode == "input_dim" else output_dim))
+        return 2 * a * torch.rand(size=shape, dtype=data_type, device=device) - a
+
+
+class Kaiming_Normal(Initialiser):
+    """
+    The Kaiming He normal initialiser. Kaiming He initialiser should be used for the ReLU or any other activation, which is nonlinear close to origin.
+
+    Args:
+        mode (str, optional): Determines if the variance is constant in the forward or backward propagation. If "input_dim", variance is constant in forward propagation, while if "output_dim", the variance is constant in back propagation. Defaults to "input_dim".
+    """
+    def __init__(self, mode="input_dim"):
+        if mode not in ["input_dim", "output_dim"]:
+            raise ValueError('mode must be one of "input_dim" or "output_dim".')
+        self.mode = mode
+
+    def initialise(self, shape, data_type=torch.float32, device=torch.device("cpu")):
+        """
+        Initialises a tensor of the wanted shape with values in :math:`N(0, \\sigma^2)`, where :math:`\\sigma = \\sqrt{\\frac{2}{d_{\\text{input}} + d_{\\text{output}}}}`.
+
+        Args:
+            shape (torch.Size): The shape of the wanted tensor.
+            data_type (torch.dtype, optional): The data type used in the returned tensor. Defaults to torch.float32.
+            device (torch.device, optional): The device of the tensor. Determines if the computation is made using the gpu or the cpu. Defaults to torch.device("cpu").
+        """
+        input_dim, output_dim = self._get_dims(shape)
+        return torch.normal(mean=0, std=sqrt(2/(input_dim if self.mode == "input_dim" else output_dim)), size=shape, dtype=data_type, device=device)

DLL/DeepLearning/Initialisers/_Xavier_Glorot.py

@@ -0,0 +1,39 @@
+import torch
+from math import sqrt
+
+from ._Initialiser import Initialiser
+
+
+class Xavier_Uniform(Initialiser):
+    """
+    The Xavier Glorot uniform initialiser. Xavier Glorot initialiser should be used for tanh, sigmoid, softmax or other activations, which are approximately linear close to origin.
+    """
+    def initialise(self, shape, data_type=torch.float32, device=torch.device("cpu")):
+        """
+        Initialises a tensor of the wanted shape with values in :math:`U(-a, a)`, where :math:`a = \\sqrt{\\frac{6}{d_{\\text{input}} + d_{\\text{output}}}}`.
+
+        Args:
+            shape (torch.Size): The shape of the wanted tensor.
+            data_type (torch.dtype, optional): The data type used in the returned tensor. Defaults to torch.float32.
+            device (torch.device, optional): The device of the tensor. Determines if the computation is made using the gpu or the cpu. Defaults to torch.device("cpu").
+        """
+        input_dim, output_dim = self._get_dims(shape)
+        a = sqrt(6/(input_dim + output_dim))
+        return 2 * a * torch.rand(size=shape, dtype=data_type, device=device) - a
+
+
+class Xavier_Normal(Initialiser):
+    """
+    The Xavier Glorot normal initialiser. Xavier Glorot initialiser should be used for tanh, sigmoid, softmax or other activations, which are approximately linear close to origin.
+    """
+    def initialise(self, shape, data_type=torch.float32, device=torch.device("cpu")):
+        """
+        Initialises a tensor of the wanted shape with values in :math:`N(0, \\sigma^2)`, where :math:`\\sigma = \\sqrt{\\frac{2}{d_{\\text{input}} + d_{\\text{output}}}}`.
+
+        Args:
+            shape (torch.Size): The shape of the wanted tensor.
+            data_type (torch.dtype, optional): The data type used in the returned tensor. Defaults to torch.float32.
+            device (torch.device, optional): The device of the tensor. Determines if the computation is made using the gpu or the cpu. Defaults to torch.device("cpu").
+        """
+        input_dim, output_dim = self._get_dims(shape)
+        return torch.normal(mean=0, std=sqrt(2/(input_dim + output_dim)), size=shape, dtype=data_type, device=device)

DLL/DeepLearning/Initialisers/__init__.py

@@ -0,0 +1,2 @@
+from ._Xavier_Glorot import Xavier_Normal, Xavier_Uniform
+from ._Kaiming_He import Kaiming_Normal, Kaiming_Uniform

DLL/DeepLearning/Layers/Activations/_Activation.py

@@ -0,0 +1,23 @@
+from .._BaseLayer import BaseLayer
+
+
+class Activation(BaseLayer):
+    def __init__(self, **kwargs):
+        super().__init__(None, None, **kwargs)
+
+        if self.output_shape is not None:
+            raise ValueError("The output_shape should be None for activation layers.")
+        if self.activation is not None:
+            raise ValueError("Activation layer must not have an activation function.")
+        if self.normalisation is not None:
+            raise ValueError("Activation layer must not have a normalisation layer.")
+
+        self.name = "Activation"
+
+    def initialise_layer(self, input_shape, data_type, device):
+        self.output_shape = input_shape
+        super().initialise_layer(input_shape, data_type, device)
+
+    def summary(self, offset=""):
+        output_shape = self.output_shape[0] if len(self.output_shape) == 1 else self.output_shape
+        return offset + f"{self.name} - Output: ({output_shape})"

DLL/DeepLearning/Layers/Activations/_ReLU.py

@@ -0,0 +1,51 @@
+import torch
+
+from ._Activation import Activation
+
+
+class ReLU(Activation):
+    """
+    The basic rectified linear unit activation function.
+    """
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.name = "ReLU"
+
+    def forward(self, input, **kwargs):
+        """
+        Calculates the following function for every element of the input matrix:
+
+        .. math::
+        
+            \\text{ReLU}(x) = \\text{max}(0, x).
+
+        Args:
+            input (torch.Tensor of shape (batch_size, ...)): The input to the layer. Must be a torch.Tensor of any shape.
+
+        Returns:
+            torch.Tensor: The output tensor after applying the activation function of the same shape as the input.
+        """
+        if not isinstance(input, torch.Tensor):
+            raise TypeError("input must be a torch.Tensor.")
+
+        self.input = input
+        output = torch.maximum(self.input, torch.zeros_like(self.input, device=self.input.device, dtype=self.input.dtype))
+        return output
+    
+    def backward(self, dCdy, **kwargs):
+        """
+        Calculates the gradient of the loss function with respect to the input of the layer.
+
+        Args:
+            dCdy (torch.Tensor of the same shape as returned from the forward method): The gradient given by the next layer.
+            
+        Returns:
+            torch.Tensor of shape (batch_size, ...): The new gradient after backpropagation through the layer.
+        """
+        if not isinstance(dCdy, torch.Tensor):
+            raise TypeError("dCdy must be a torch.Tensor.")
+        if dCdy.shape != self.input.shape:
+            raise ValueError(f"dCdy is not the same shape as the spesified output_shape ({dCdy.shape[1:], self.output_shape}).")
+
+        dCdx = dCdy * (self.input > 0)
+        return dCdx

DLL/DeepLearning/Layers/Activations/_Sigmoid.py

@@ -0,0 +1,51 @@
+import torch
+
+from ._Activation import Activation
+
+
+class Sigmoid(Activation):
+    """
+    The sigmoid activation function.
+    """
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.name = "Sigmoid"
+
+    def forward(self, input, **kwargs):
+        """
+        Calculates the following function for every element of the input matrix:
+
+        .. math::
+        
+            \\sigma(x) = \\frac{1}{1 + e^{-x}}.
+
+        Args:
+            input (torch.Tensor of shape (batch_size, ...)): The input to the layer. Must be a torch.Tensor of any shape.
+
+        Returns:
+            torch.Tensor: The output tensor after applying the activation function of the same shape as the input.
+        """
+        if not isinstance(input, torch.Tensor):
+            raise TypeError("input must be a torch.Tensor.")
+        
+        self.input = input
+        self.output = 1 / (1 + torch.exp(-self.input))
+        return self.output
+    
+    def backward(self, dCdy, **kwargs):
+        """
+        Calculates the gradient of the loss function with respect to the input of the layer.
+
+        Args:
+            dCdy (torch.Tensor of the same shape as returned from the forward method): The gradient given by the next layer.
+            
+        Returns:
+            torch.Tensor of shape (batch_size, ...): The new gradient after backpropagation through the layer.
+        """
+        if not isinstance(dCdy, torch.Tensor):
+            raise TypeError("dCdy must be a torch.Tensor.")
+        if dCdy.shape != self.output.shape:
+            raise ValueError(f"dCdy is not the same shape as the spesified output_shape ({dCdy.shape[1:], self.output_shape}).")
+
+        dCdx = (self.output * (1 - self.output)) * dCdy
+        return dCdx

DLL/DeepLearning/Layers/Activations/_SoftMax.py

@@ -0,0 +1,83 @@
+import torch
+
+from ._Activation import Activation
+
+
+class SoftMax(Activation):
+    """
+    The softmax activation function.
+
+    Args:
+        dim (int): The dimension on which the softmax is calculated. If the data is (n_samples, n_channels, n_features) and one wants to calculate the softmax on the channels, one should select dim=1 or dim=-2.
+    """
+    def __init__(self, dim=-1, **kwargs):
+        super().__init__(**kwargs)
+        self.dim = dim
+        self.name = "Softmax"
+
+    def forward(self, input, **kwargs):
+        """
+        Calculates the following function for every element of the input matrix:
+
+        .. math::
+        
+            \\text{Softmax}(x)_i = \\frac{e^{x_i}}{\\sum_{j=1}^{K} e^{x_j}},
+        
+        where :math:`K` is the number of features of the input.
+
+        Args:
+            input (torch.Tensor of shape (n_samples, n_features)): The input to the layer. Must be a torch.Tensor of the spesified shape.
+
+        Returns:
+            torch.Tensor: The output tensor after applying the activation function of the same shape as the input.
+        """
+        if not isinstance(input, torch.Tensor):
+            raise TypeError("input must be a torch.Tensor.")
+        if input.ndim <= self.dim or -input.ndim > self.dim:
+            raise ValueError(f"dim must be in range [{-input.ndim}, {input.ndim}) for input of this shape, but is currently {self.dim}.")
+
+        self.input = input
+        exponential_input = torch.exp(self.input - torch.max(self.input, dim=self.dim, keepdim=True).values)
+        self.output = exponential_input / torch.sum(exponential_input, dim=self.dim, keepdim=True)
+        return self.output
+    
+    def backward(self, dCdy, **kwargs):
+        """
+        Calculates the gradient of the loss function with respect to the input of the layer.
+
+        Args:
+            dCdy (torch.Tensor of the same shape as returned from the forward method): The gradient given by the next layer.
+            
+        Returns:
+            torch.Tensor of shape (n_samples, n_features): The new gradient after backpropagation through the layer.
+        """
+        if not isinstance(dCdy, torch.Tensor):
+            raise TypeError("dCdy must be a torch.Tensor.")
+        if dCdy.shape != self.output.shape:
+            raise ValueError(f"dCdy is not the same shape as the spesified output_shape ({dCdy.shape[1:], self.output_shape}).")
+
+        # v1 - 2 dimensional
+        # n = dCdy.shape[1]
+        # # dCdx = torch.stack([(dCdy[i] @ (torch.tile(datapoint, (n, 1)).T * (torch.eye(n, device=self.device, dtype=self.data_type) - torch.tile(datapoint, (n, 1))))) for i, datapoint in enumerate(self.output)])
+
+        # v2 - 2 dimensional, but faster
+        # datapoints_expanded = self.output.unsqueeze(1).repeat(1, n, 1)
+        # identity_matrix = torch.eye(n, device=dCdy.device, dtype=dCdy.dtype)
+        # matrix_diff = identity_matrix - datapoints_expanded
+        # dCdx = dCdy.unsqueeze(1) @ (datapoints_expanded.transpose(1, 2) * matrix_diff)
+        # return dCdx.squeeze(1)
+
+        # v3 - n dimensional
+        # dim = self.dim if self.dim >= 0 else dCdy.ndim + self.dim
+        # n = self.output.shape[dim]
+        # output_expanded = self.output.unsqueeze(dim)
+        # identity_matrix = torch.eye(n, device=dCdy.device, dtype=dCdy.dtype).reshape((1,) * dim + (n, n) + (1,) * (dCdy.ndim - dim - 1))
+        # matrix_diff = identity_matrix - output_expanded
+        # # probably not optimal permuting the two wanted dimensions to the end for the matrix multiplication
+        # dCdx = dCdy.unsqueeze(dim).permute(*range(dim), *range(dim + 2, dCdy.ndim + 1), dim, dim + 1) @ (output_expanded.transpose(dim, dim + 1) * matrix_diff).permute(*range(dim), *range(dim + 2, dCdy.ndim + 1), dim, dim + 1)
+        # return dCdx.permute(*range(dim), dCdy.ndim - 1, dCdy.ndim, *range(dim, dCdy.ndim - 1)).squeeze(dim)
+
+        # v4 - n dimensional, but faster
+        output = self.output
+        dCdx = output * (dCdy - torch.sum(dCdy * output, dim=self.dim, keepdim=True))
+        return dCdx

DLL/DeepLearning/Layers/Activations/_Tanh.py

@@ -0,0 +1,47 @@
+import torch
+
+from ._Activation import Activation
+
+
+class Tanh(Activation):
+    """
+    The hyperbolic tangent activation function.
+    """
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.name = "Tanh"
+
+    def forward(self, input, **kwargs):
+        """
+        Calculates the hyperbolic tangent function for every element of the input matrix.
+
+        Args:
+            input (torch.Tensor of shape (batch_size, ...)): The input to the layer. Must be a torch.Tensor of any shape.
+
+        Returns:
+            torch.Tensor: The output tensor after applying the activation function of the same shape as the input.
+        """
+        if not isinstance(input, torch.Tensor):
+            raise TypeError("input must be a torch.Tensor.")
+
+        self.input = input
+        output = torch.tanh(input)
+        return output
+    
+    def backward(self, dCdy, **kwargs):
+        """
+        Calculates the gradient of the loss function with respect to the input of the layer.
+
+        Args:
+            dCdy (torch.Tensor of the same shape as returned from the forward method): The gradient given by the next layer.
+            
+        Returns:
+            torch.Tensor of shape (batch_size, ...): The new gradient after backpropagation through the layer.
+        """
+        if not isinstance(dCdy, torch.Tensor):
+            raise TypeError("dCdy must be a torch.Tensor.")
+        if dCdy.shape != self.input.shape:
+            raise ValueError(f"dCdy is not the same shape as the spesified output_shape ({dCdy.shape[1:], self.output_shape}).")
+
+        dCdx = dCdy * (1 - torch.tanh(self.input) ** 2)
+        return dCdx

DLL/DeepLearning/Layers/Activations/__init__.py

@@ -0,0 +1,4 @@
+from ._ReLU import ReLU
+from ._Sigmoid import Sigmoid
+from ._SoftMax import SoftMax
+from ._Tanh import Tanh

DLL/DeepLearning/Layers/Regularisation/_BaseRegularisation.py

@@ -0,0 +1,12 @@
+from ..Activations._Activation import Activation
+from ....Exceptions import NotCompiledError
+
+
+class BaseRegularisation(Activation):
+    def summary(self, offset=""):
+        if not hasattr(self, "input_shape"):
+            raise NotCompiledError("layer must be initialized correctly before calling layer.summary().")
+
+        output_shape = self.output_shape[0] if len(self.output_shape) == 1 else self.output_shape
+        params_summary = " - Parameters: " + str(self.nparams) if self.nparams > 0 else ""
+        return offset + f"{self.name} - Output: ({output_shape})" + params_summary

DLL/DeepLearning/Layers/Regularisation/_BatchNormalisation.py

@@ -0,0 +1,106 @@
+import torch
+import numpy as np
+
+from ._BaseRegularisation import BaseRegularisation
+
+
+class BatchNorm(BaseRegularisation):
+    """
+    The batch normalisation layer for neural networks.
+
+    Args:
+        patience (float, optional): The number deciding how fast the mean and variance in training. Must be strictly between 0 and 1. Defaults to 0.9.
+    """
+    def __init__(self, patience=0.9, **kwargs):
+        if patience <= 0 or 1 <= patience:
+            raise ValueError("patience must be strictly between 0 and 1.")
+        
+        super().__init__(**kwargs)
+        self.patience = patience
+        self.epsilon = 1e-6
+        self.name = "Batch normalisation"
+    
+    def initialise_layer(self, **kwargs):
+        """
+        :meta private:
+        """
+
+        super().initialise_layer(**kwargs)
+
+        self.gamma = torch.ones(self.output_shape, dtype=self.data_type, device=self.device)
+        self.beta = torch.zeros(self.output_shape, dtype=self.data_type, device=self.device)
+        self.running_var = torch.ones(self.output_shape, dtype=self.data_type, device=self.device)
+        self.running_mean = torch.zeros(self.output_shape, dtype=self.data_type, device=self.device)
+        self.nparams = 2 * np.prod(self.output_shape)
+
+    def forward(self, input, training=False, **kwargs):
+        """
+        Normalises the input to have zero mean and one variance with the following equation:
+        
+        .. math::
+            y = \\gamma\\frac{x - \\mathbb{E}[x]}{\\sqrt{\\text{var}(x) + \\epsilon}} + \\beta,
+        
+        where :math:`x` is the input, :math:`\\mathbb{E}[x]` is the expected value or the mean accross the batch dimension, :math:`\\text{var}(x)` is the variance accross the variance accross the batch dimension, :math:`\\epsilon` is a small constant and :math:`\\gamma` and :math:`\\beta` are trainable parameters.
+
+        Args:
+            input (torch.Tensor of shape (batch_size, channels, ...)): The input to the layer. Must be a torch.Tensor of the spesified shape given by layer.input_shape.
+            training (bool, optional): The boolean flag deciding if the model is in training mode. Defaults to False.
+
+        Returns:
+            torch.Tensor: The output tensor after the normalisation with the same shape as the input.
+        """
+        if not isinstance(input, torch.Tensor):
+            raise TypeError("input must be a torch.Tensor.")
+        if input.shape[1:] != self.input_shape:
+            raise ValueError(f"input is not the same shape as the spesified input_shape ({input.shape[1:], self.input_shape}).")
+        if not isinstance(training, bool):
+            raise TypeError("training must be a boolean.")
+        if input.shape[0] <= 1:
+            raise ValueError("The batch size must be atleast 2.")
+        
+        self.input = input
+        if training:
+            mean = torch.mean(input, axis=0)
+            variance = torch.var(input, axis=0, unbiased=True)
+            self.std = torch.sqrt(variance + self.epsilon)
+            self.running_mean = self.patience * self.running_mean + (1 - self.patience) * mean
+            self.running_var = self.patience * self.running_var + (1 - self.patience) * variance
+            self.x_centered = (self.input - mean)
+            self.x_norm = self.x_centered / self.std
+            self.output = self.gamma * self.x_norm + self.beta
+        else:
+            self.output = self.gamma * ((self.input - self.running_mean) / torch.sqrt(self.running_var + self.epsilon)) + self.beta
+        return self.output
+    
+    def backward(self, dCdy, **kwargs):
+        """
+        Calculates the gradient of the loss function with respect to the input of the layer. Also calculates the gradients of the loss function with respect to the model parameters.
+
+        Args:
+            dCdy (torch.Tensor of the same shape as returned from the forward method): The gradient given by the next layer.
+            
+        Returns:
+            torch.Tensor of shape (batch_size, channels, ...): The new gradient after backpropagation through the layer.
+        """
+        if not isinstance(dCdy, torch.Tensor):
+            raise TypeError("dCdy must be a torch.Tensor.")
+        if dCdy.shape[1:] != self.output.shape[1:]:
+            raise ValueError(f"dCdy is not the same shape as the spesified output_shape ({dCdy.shape[1:], self.output.shape[1:]}).")
+        
+        batch_size = self.output.shape[0]
+        dCdx_norm = dCdy * self.gamma
+        dCdgamma = (dCdy * self.x_norm).mean(axis=0)
+        dCdbeta = dCdy.mean(axis=0)
+        dCdvar = (dCdx_norm * self.x_centered * -self.std**(-3) / 2).sum(axis=0)
+        dCdmean = -((dCdx_norm / self.std).sum(axis=0) + dCdvar * (2 / batch_size) * self.x_centered.sum(axis=0))
+        dCdx = dCdx_norm / self.std + dCdvar * 2 * self.x_centered / (batch_size - 1) + dCdmean / batch_size
+
+        self.gamma.grad += dCdgamma
+        self.beta.grad += dCdbeta
+        return dCdx
+    
+    def get_parameters(self):
+        """
+        :meta private:
+        """
+        return (self.gamma, self.beta)