PyDLLib 1.0__py3-none-any.whl

DLL/DeepLearning/Layers/_Bidirectional.py

@@ -0,0 +1,113 @@
+import torch
+from copy import deepcopy
+
+from ._BaseLayer import BaseLayer
+from . import RNN, LSTM
+from ...Exceptions import NotCompiledError
+
+
+class Bidirectional(BaseLayer):
+    """
+    The bidirectional wrapper for LSTM or RNN layers.
+
+    Args:
+        layer (DLL.DeepLearning.Layers.RNN or LSTM object): The input is passed to this layer in forward and reverse. The results of each layer are concatanated together along the feature axis.
+    """
+    def __init__(self, layer, **kwargs):
+        if not isinstance(layer, RNN) and not isinstance(layer, LSTM):
+            raise TypeError("layer must be an instance of DLL.DeepLearning.Layers.RNN or LSTM")
+
+        # Change the layers output shape to have at least 1 dimensional features.
+        if len(layer.output_shape) == 0 or (len(layer.output_shape) == 1 and not layer.return_last):
+            layer.output_shape = (*layer.output_shape, 1)
+        output_shape = (*layer.output_shape[:-1], 2 * layer.output_shape[-1])
+
+        super().__init__(output_shape, **kwargs)
+        self.name = "Bidirectional"
+        self.forward_layer = layer
+        self.backward_layer = deepcopy(layer)
+
+    def initialise_layer(self, input_shape, data_type, device):
+        """
+        :meta private:
+        """
+        if not isinstance(input_shape, tuple | list) or len(input_shape) != 2:
+            raise ValueError("input_shape must be a tuple of length 2.")
+        if not isinstance(data_type, torch.dtype):
+            raise TypeError("data_type must be an instance of torch.dtype")
+        if not isinstance(device, torch.device):
+            raise TypeError('device must be one of torch.device("cpu") or torch.device("cuda")')
+    
+        self.forward_layer.initialise_layer(input_shape, data_type, device)
+        self.backward_layer.initialise_layer(input_shape, data_type, device)
+
+        super().initialise_layer(input_shape, data_type, device)
+
+    def forward(self, input, training=False, **kwargs):
+        """
+        Computes the forward values of the RNN or LSTM layer for both normal input and reverse input and concatanates the results along the feature axis.
+
+        Args:
+            input (torch.Tensor of shape (batch_size, sequence_length, input_size)): The input to the layer. Must be a torch.Tensor of the spesified shape given by layer.input_shape.
+            training (bool, optional): The boolean flag deciding if the model is in training mode. Defaults to False.
+
+        Returns:
+            torch.Tensor of shape (n_samples, 2 * RNN.output_shape[-1]) or (n_samples, sequence_length, 2 * RNN.output_shape[-1]): The output tensor after the transformations with the spesified shape.
+        """
+        if not isinstance(input, torch.Tensor):
+            raise TypeError("input must be a torch.Tensor.")
+        if input.shape[1:] != self.input_shape:
+            raise ValueError(f"Input shape {input.shape[1:]} does not match the expected shape {self.input_shape}.")
+        if not isinstance(training, bool):
+            raise TypeError("training must be a boolean.")
+
+        forward_val = self.forward_layer.forward(input, training=training)
+        backward_val = self.backward_layer.forward(input.flip(1), training=training)
+        if not self.backward_layer.return_last:
+            backward_val = backward_val.flip(1)
+        self.output = torch.cat((forward_val, backward_val), dim=-1)
+        return self.output
+
+    def backward(self, dCdy, **kwargs):
+        """
+        Calculates the gradient of the loss function with respect to the input of the layer. Also calculates the gradients of the loss function with respect to the model parameters.
+
+        Args:
+            dCdy (torch.Tensor of the same shape as returned from the forward method): The gradient given by the next layer.
+
+        Returns:
+            torch.Tensor of shape (n_samples, sequence_length, input_size): The new gradient after backpropagation through the layer.
+        """
+        if not isinstance(dCdy, torch.Tensor):
+            raise TypeError("dCdy must be a torch.Tensor.")
+        if dCdy.shape[1:] != self.output.shape[1:]:
+            raise ValueError(f"dCdy is not the same shape as the spesified output_shape ({dCdy.shape[1:], self.output.shape[1:]}).")
+        
+        forward_grad = dCdy[..., :self.output_shape[-1] // 2]
+        backward_grad = dCdy[..., self.output_shape[-1] // 2:]
+        dCdx_forward = self.forward_layer.backward(forward_grad, **kwargs)
+        dCdx_backward = self.backward_layer.backward(backward_grad.flip(1), **kwargs)
+        if not self.backward_layer.return_last:
+            dCdx_backward = dCdx_backward.flip(1)
+
+        dCdx = dCdx_backward + dCdx_forward
+        return dCdx
+    
+    def get_parameters(self):
+        """
+        :meta private:
+        """
+        return (*self.forward_layer.get_parameters(), *self.backward_layer.get_parameters(), *super().get_parameters())
+
+    def get_nparams(self):
+        return self.forward_layer.get_nparams() + self.backward_layer.get_nparams()
+    
+    def summary(self, offset=""):
+        if not hasattr(self, "input_shape"):
+            raise NotCompiledError("layer must be initialized correctly before calling layer.summary().")
+
+        super_summary = offset + f"{self.name} - (Input, Output): ({self.input_shape}, {self.output_shape})"
+        sublayer_offset = offset + "    "
+        forward_summary = "\n" + offset + self.forward_layer.summary(sublayer_offset)
+        backward_summary = "\n" + offset + self.backward_layer.summary(sublayer_offset)
+        return super_summary + forward_summary + backward_summary

DLL/DeepLearning/Layers/_Conv2D.py

@@ -0,0 +1,133 @@
+import torch
+import torch.nn.functional as F
+import numpy as np
+
+from ._BaseLayer import BaseLayer
+from .Activations._Activation import Activation
+from .Regularisation._BaseRegularisation import BaseRegularisation
+from ..Initialisers import Xavier_Uniform
+from ..Initialisers._Initialiser import Initialiser
+
+
+class Conv2D(BaseLayer):
+    """
+    The convolutional layer for a neural network.
+
+    Args:
+        kernel_size (int): The kernel size used for the model. The kernel is automatically square. Must be a positive integer.
+        output_depth (int): The output depth of the layer. Must be a positive integer.
+        initialiser (:ref:`initialisers_section_label`, optional): The initialisation method for models weights. Defaults to Xavier_uniform.
+        activation (:ref:`activations_section_label` | None, optional): The activation used after this layer. If is set to None, no activation is used. Defaults to None. If both activation and regularisation is used, the regularisation is performed first in the forward propagation.
+        normalisation (:ref:`regularisation_layers_section_label` | None, optional): The regularisation layer used fter this layer. If is set to None, no regularisation is used. Defaults to None. If both activation and regularisation is used, the regularisation is performed first in the forward propagation.
+    """
+    def __init__(self, kernel_size, output_depth, initialiser=Xavier_Uniform(), activation=None, normalisation=None, **kwargs):
+        if not isinstance(kernel_size, int) or kernel_size <= 0:
+            raise ValueError(f"kernel_size must be a positive integer. Currently {kernel_size}.")
+        if not isinstance(output_depth, int) or output_depth <= 0:
+            raise ValueError(f"output_depth must be a positive integer. Currently {output_depth}.")
+        if not isinstance(initialiser, Initialiser):
+            raise ValueError('initialiser must be an instance of DLL.DeepLearning.Initialisers')
+        if not isinstance(activation, Activation) and activation is not None:
+            raise ValueError("activation must be from DLL.DeepLearning.Layers.Activations or None.")
+        if not isinstance(normalisation, BaseRegularisation) and normalisation is not None:
+            raise ValueError("normalisation must be from DLL.DeepLearning.Layers.Regularisation or None.")
+
+        super().__init__(output_shape=None, activation=activation, normalisation=normalisation, **kwargs)
+        self.output_depth = output_depth
+        self.kernel_size = kernel_size
+        self.initialiser = initialiser
+        self.name = "Conv2D"
+
+    def initialise_layer(self, input_shape, data_type, device):
+        """
+        :meta private:
+        """
+        input_depth, input_height, input_width = input_shape
+        self.input_depth = input_depth
+        self.output_shape = (self.output_depth, input_height - self.kernel_size + 1, input_width - self.kernel_size + 1)
+        self.kernels_shape = (self.output_depth, input_depth, self.kernel_size, self.kernel_size)
+        self.nparams = np.prod(self.kernels_shape) + np.prod(self.output_shape)
+        
+        super().initialise_layer(input_shape, data_type, device)
+
+        self.kernels = self.initialiser.initialise(self.kernels_shape, data_type=self.data_type, device=self.device)
+        self.biases = torch.zeros(self.output_shape)
+        
+        if self.activation:
+            self.activation.initialise_layer(self.output_shape, self.data_type, self.device)
+        if self.normalisation:
+            self.normalisation.initialise_layer(self.output_shape, self.data_type, self.device)
+    
+    def forward(self, input, training=False, **kwargs):
+        """
+        Applies the convolutional transformation.
+
+        .. math::
+            \\begin{align*}
+                y_{i, j} &= \\text{bias}_j + \\sum_{k = 1}^{\\text{d_in}} \\text{kernel}(j, k) \star \\text{input}(i, k),\\\\
+                y_{reg_{i, j}} &= f(y_{i, j}),\\\\
+                y_{activ_{i, j}} &= g(y_{reg}),
+            \\end{align*}
+        
+        where :math:`\star` is the cross-correlation operator, :math:`\\text{d_in}` is the input_depth, :math:`i\in [1,\dots, \\text{batch_size}]`, :math:`j\in[1,\dots, \\text{output_depth}]`, :math:`f` is the possible regularisation function and :math:`g` is the possible activation function.
+
+        Args:
+            input (torch.Tensor of shape (n_samples, input_depth, input_height, input_width)): The input to the layer. Must be a torch.Tensor of the spesified shape.
+            training (bool, optional): The boolean flag deciding if the model is in training mode. Defaults to False.
+            
+        Returns:
+            torch.Tensor of shape (n_samples, output_depth, height - kernel_size + 1, width - kernel_size + 1): The output tensor after the transformations with the spesified shape.
+        """
+        if not isinstance(input, torch.Tensor):
+            raise TypeError("input must be a torch.Tensor.")
+        if input.shape[1:] != self.input_shape:
+            raise ValueError(f"input is not the same shape as the spesified input_shape ({input.shape[1:], self.input_shape}).")
+        if not isinstance(training, bool):
+            raise TypeError("training must be a boolean.")
+
+        batch_size = input.shape[0]
+        self.input = input
+        self.output = self.biases.clone().repeat(batch_size, 1, 1, 1)
+        for i in range(self.output_depth):
+            for j in range(self.input_depth):
+                conv_output = F.conv2d(input[:, j:j+1, :, :], self.kernels[i:i+1, j:j+1, :, :], padding="valid")
+                self.output[:, i, :, :] += conv_output[:, 0, :, :]
+                
+        if self.normalisation: self.output = self.normalisation.forward(self.output, training=training)
+        if self.activation: self.output = self.activation.forward(self.output)
+        return self.output
+    
+    def backward(self, dCdy, **kwargs):
+        """
+        Calculates the gradient of the loss function with respect to the input of the layer. Also calculates the gradients of the loss function with respect to the model parameters.
+
+        Args:
+            dCdy (torch.Tensor of shape (n_samples, output_depth, output_height, output_width) : The gradient given by the next layer.
+
+        Returns:
+            torch.Tensor of shape (n_samples, input_depth, input_height, input_width): The new gradient after backpropagation through the layer.
+        """
+        if not isinstance(dCdy, torch.Tensor):
+            raise TypeError("dCdy must be a torch.Tensor.")
+        if dCdy.shape[1:] != self.output_shape:
+            raise ValueError(f"dCdy is not the same shape as the spesified output_shape ({dCdy.shape[1:], self.output_shape}).")
+
+        if self.activation: dCdy = self.activation.backward(dCdy)
+        if self.normalisation: dCdy = self.normalisation.backward(dCdy)
+        kernel_gradient = torch.zeros_like(self.kernels, device=self.device, dtype=self.data_type)
+        dCdx = torch.zeros_like(self.input, device=self.device, dtype=self.data_type)
+        batch_size = self.input.shape[0]
+        for i in range(self.output_depth):
+            for j in range(self.input_depth):
+                kernel_gradient[i, j] = F.conv2d(self.input[:, j:j+1, :, :], dCdy[:, i:i+1, :, :], padding="valid")[0, 0, :, :]
+                dCdx[:, j] += F.conv2d(dCdy[:, i:i+1, :, :], torch.flip(self.kernels[i:i+1, j:j+1, :, :], dims=(2, 3)), padding=[self.kernel_size - 1, self.kernel_size - 1])[0, 0, :, :]
+                
+        self.biases.grad += dCdy.mean(dim=0)
+        self.kernels.grad += kernel_gradient / batch_size
+        return dCdx
+    
+    def get_parameters(self):
+        """
+        :meta private:
+        """
+        return (self.kernels, self.biases, *super().get_parameters())

DLL/DeepLearning/Layers/_Dense.py

@@ -0,0 +1,109 @@
+import torch
+
+from ._BaseLayer import BaseLayer
+from .Activations._Activation import Activation
+from .Regularisation._BaseRegularisation import BaseRegularisation
+from ..Initialisers import Xavier_Uniform
+from ..Initialisers._Initialiser import Initialiser
+
+
+class Dense(BaseLayer):
+    """
+    The basic dense linear layer.
+
+    Args:
+        output_shape (tuple[int] or int): The output_shape of the model not containing the batch_size dimension. Must contain non-negative integers. If is int, returned shape is (n_samples, int). If is the length is zero, the returned tensor is of shape (n_samples,). Otherwise the returned tensor is of shape (n_samples, *output_shape).
+        initialiser (:ref:`initialisers_section_label`, optional): The initialisation method for models weights. Defaults to Xavier_uniform.
+        activation (:ref:`activations_section_label` | None, optional): The activation used after this layer. If is set to None, no activation is used. Defaults to None. If both activation and regularisation is used, the regularisation is performed first in the forward propagation.
+        normalisation (:ref:`regularisation_layers_section_label` | None, optional): The regularisation layer used after this layer. If is set to None, no regularisation is used. Defaults to None. If both activation and regularisation is used, the regularisation is performed first in the forward propagation.
+    """
+    def __init__(self, output_shape, bias=True, initialiser=Xavier_Uniform(), activation=None, normalisation=None, **kwargs):
+        if not isinstance(initialiser, Initialiser):
+            raise ValueError('initialiser must be an instance of DLL.DeepLearning.Initialisers')
+        if not isinstance(activation, Activation) and activation is not None:
+            raise ValueError("activation must be from DLL.DeepLearning.Layers.Activations or None.")
+        if not isinstance(normalisation, BaseRegularisation) and normalisation is not None:
+            raise ValueError("normalisation must be from DLL.DeepLearning.Layers.Regularisation or None.")
+
+        super().__init__(output_shape, activation=activation, normalisation=normalisation, **kwargs)
+        self.name = "Dense"
+        self.initialiser = initialiser
+        self.bias = bias
+
+    def initialise_layer(self, input_shape, data_type, device):
+        """
+        :meta private:
+        """
+        super().initialise_layer(input_shape, data_type, device)
+        
+        input_dim = input_shape[-1]
+        output_dim = 1 if len(self.output_shape) == 0 else self.output_shape[-1]
+        
+        self.weights = self.initialiser.initialise((input_dim, output_dim), data_type=self.data_type, device=self.device)
+        self.biases = torch.zeros(output_dim, dtype=self.data_type, device=self.device)
+        self.nparams = output_dim * input_dim + output_dim
+
+    def forward(self, input, training=False, **kwargs):
+        """
+        Applies the basic linear transformation
+
+        .. math::
+        
+            \\begin{align*}
+                y_{lin} = xW + b,\\\\
+                y_{reg} = f(y_{lin}),\\\\
+                y_{activ} = g(y_{reg}),
+            \\end{align*}
+        
+        where :math:`f` is the possible regularisation function and :math:`g` is the possible activation function.
+
+        Args:
+            input (torch.Tensor of shape (n_samples, n_features)): The input to the dense layer. Must be a torch.Tensor of the spesified shape given by layer.input_shape.
+            training (bool, optional): The boolean flag deciding if the model is in training mode. Defaults to False.
+            
+        Returns:
+            torch.Tensor of shape (n_samples,) if len(layer.output_shape) == 0 else (n_samples, output_shape): The output tensor after the transformations with the spesified shape.
+        """
+        if not isinstance(input, torch.Tensor):
+            raise TypeError("input must be a torch.Tensor.")
+        if input.shape[1:] != self.input_shape:
+            raise ValueError(f"input is not the same shape as the spesified input_shape ({input.shape[1:], self.input_shape}).")
+        if not isinstance(training, bool):
+            raise TypeError("training must be a boolean.")
+
+        self.input = input
+        output = self.input @ self.weights + self.biases
+        if self.normalisation: output = self.normalisation.forward(output, training=training)
+        if self.activation: output = self.activation.forward(output)
+        if len(self.output_shape) == 0: output = output.squeeze(dim=1)  # If the output_shape is a 1d tensor, remove the last dimension
+        return output
+
+    def backward(self, dCdy, **kwargs):
+        """
+        Calculates the gradient of the loss function with respect to the input of the layer. Also calculates the gradients of the loss function with respect to the model parameters.
+
+        Args:
+            dCdy (torch.Tensor of shape (n_samples,) if len(layer.output_shape) == 0 else (n_samples, output_shape)): The gradient given by the next layer.
+            
+        Returns:
+            torch.Tensor of shape (n_samples, layer.input_shape[0]): The new gradient after backpropagation through the layer.
+        """
+        if not isinstance(dCdy, torch.Tensor):
+            raise TypeError("dCdy must be a torch.Tensor.")
+        if dCdy.shape[1:] != self.output_shape:
+            raise ValueError(f"dCdy is not the same shape as the spesified output_shape ({dCdy.shape[1:], self.output_shape}).")
+        
+        if len(self.output_shape) == 0: dCdy = dCdy.unsqueeze(dim=1)  # If the output shape was a 1d tensor, add an extra dimension to the end
+        if self.activation: dCdy = self.activation.backward(dCdy)
+        if self.normalisation: dCdy = self.normalisation.backward(dCdy)
+        dCdx = dCdy @ self.weights.T
+        dCdW = self.input.transpose(-1, -2) @ dCdy
+        self.weights.grad += torch.mean(dCdW, dim=tuple(range(self.input.ndim - 2))) if self.input.ndim > 2 else dCdW
+        self.biases.grad += torch.mean(dCdy, dim=tuple(range(self.input.ndim - 1))) if self.bias else torch.zeros_like(self.biases)
+        return dCdx
+    
+    def get_parameters(self):
+        """
+        :meta private:
+        """
+        return (self.weights, self.biases, *super().get_parameters())

DLL/DeepLearning/Layers/_DenseKAN.py

@@ -0,0 +1,184 @@
+import torch
+from functools import lru_cache, partial
+
+from ._BaseLayer import BaseLayer
+from .Activations._Activation import Activation
+from .Regularisation._BaseRegularisation import BaseRegularisation
+from ..Initialisers import Xavier_Uniform
+from ..Initialisers._Initialiser import Initialiser
+
+
+@lru_cache
+def _bspline_basis(i, n_fun, knots, x):
+    if n_fun == 0:
+        return ((knots[i] <= x) & (x < knots[i + 1])).float()
+    
+    denom1 = knots[i + n_fun] - knots[i]
+    denom2 = knots[i + n_fun + 1] - knots[i + 1]
+
+    term1 = (x - knots[i]) / denom1 * _bspline_basis(i, n_fun - 1, knots, x) if denom1 > 0 else torch.zeros_like(x)
+    term2 = (knots[i + n_fun + 1] - x) / denom2 * _bspline_basis(i + 1, n_fun - 1, knots, x) if denom2 > 0 else torch.zeros_like(x)
+    return term1 + term2
+
+@lru_cache
+def _bspline_basis_derivative(i, n_fun, knots, x):
+    if n_fun == 0:
+        return torch.zeros(x)
+
+    denom1 = knots[i + n_fun] - knots[i]
+    denom2 = knots[i + n_fun + 1] - knots[i + 1]
+
+    term1 = _bspline_basis(i, n_fun - 1, knots, x) / denom1 if denom1 > 0 else torch.zeros_like(x)
+    term2 = _bspline_basis(i + 1, n_fun - 1, knots, x) / denom2 if denom2 > 0 else torch.zeros_like(x)
+
+    return n_fun * (term1 - term2)
+
+def _SiLU(x):
+    return x / (1 + torch.exp(-x))
+
+def _SiLU_der(x):
+    return (1 + torch.exp(-x) + x * torch.exp(-x)) / (1 + torch.exp(-x)) ** 2
+
+def make_basis_func(i, degree, t, x):
+    return _bspline_basis(i, degree, t, x)
+
+def make_basis_func_der(i, degree, t, x):
+    return _bspline_basis_derivative(i, degree, t, x)
+
+def _get_basis_functions(n_fun, degree=3, bounds=(-1, 1)):
+    grid_len = n_fun - degree + 1
+    step = (bounds[1] - bounds[0]) / (grid_len - 1)
+    edge_funcs, edge_func_ders = [], []
+
+    # SiLU bias function
+    edge_funcs.append(_SiLU)
+    edge_func_ders.append(_SiLU_der)
+
+    # B-splines
+    t = torch.linspace(bounds[0] - degree * step, bounds[1] + degree * step, grid_len + 2 * degree)
+    t[degree], t[-degree - 1] = bounds[0], bounds[1]
+    for ind_spline in range(n_fun - 1):
+        edge_funcs.append(partial(make_basis_func, ind_spline, degree, t))
+        edge_func_ders.append(partial(make_basis_func_der, ind_spline, degree, t))
+    return edge_funcs, edge_func_ders
+
+
+class _NeuronKAN:
+    def __init__(self, input_length, n_basis_funcs, bounds, initialiser, data_type, device, basis_func_degree=3):
+        self.edge_fun, self.edge_fun_der = _get_basis_functions(n_fun=n_basis_funcs, degree=basis_func_degree, bounds=bounds)
+        self.weights = initialiser.initialise((n_basis_funcs, input_length), data_type=data_type, device=device)
+
+    def forward(self, input, **kwargs):
+        self.input = input
+        self.edge_func_values = torch.stack([func(input) for func in self.edge_fun], dim=1)  # (n, n_basis_funcs, input_length)
+        output = (self.weights.unsqueeze(0) * self.edge_func_values).sum(dim=(1, 2))
+        return output
+
+    def backward(self, dCdy, **kwargs):
+        dCdy = dCdy.view(len(dCdy), 1, 1)  # (n, 1, 1)
+        self.weights.grad += torch.sum(dCdy * self.edge_func_values, dim=0)
+        dCdx = dCdy * self.weights.unsqueeze(0)  # (n, n_basis_funcs, input_length)
+        edge_func_derivatives = torch.stack([func(self.input) for func in self.edge_fun_der], dim=1)
+        dCdx = torch.sum(dCdx * edge_func_derivatives, dim=1)
+        return dCdx
+
+
+class DenseKAN(BaseLayer):
+    """
+    The dense Kolmogorov-Arnold network layer. The implementation is based on `this paper <https://arxiv.org/pdf/2404.19756>`_ and `this article <https://mlwithouttears.com/2024/05/15/a-from-scratch-implementation-of-kolmogorov-arnold-networks-kan/>`_.
+
+    Args:
+        output_shape (tuple[int] or int): The output_shape of the model not containing the batch_size dimension. Must contain non-negative integers. If is int, returned shape is (n_samples, int). If is the length is zero, the returned tensor is of shape (n_samples,). Otherwise the returned tensor is of shape (n_samples, *output_shape).
+        n_basis_funcs (int, optional): The number of basis functions used for fitting. If 1, only SiLU is used. Otherwise 1 SiLU and n_basis_funcs - 1 Bsplines basis functions are used. Must be a positive integer. Defaults to 10.
+        bounds (tuple[int], optional): The theoretical min and max of the data that will be passed to the forward method. Must be a tuple containing two integers. Defaults to (-1, 1).
+        basis_func_degree (int, optional): The degree of the Bspline basis functions. Must be positive. Defaults to 3.
+        initialiser (:ref:`initialisers_section_label`, optional): The initialisation method for models weights. Defaults to Xavier_uniform.
+        activation (:ref:`activations_section_label` | None, optional): The activation used after this layer. If is set to None, no activation is used. Defaults to None. If both activation and regularisation is used, the regularisation is performed first in the forward propagation.
+        normalisation (:ref:`regularisation_layers_section_label` | None, optional): The regularisation layer used after this layer. If is set to None, no regularisation is used. Defaults to None. If both activation and regularisation is used, the regularisation is performed first in the forward propagation.
+    Note:
+        n_basis_funcs and basis_func_degree should be different to avoid certain errors.
+    """
+    def __init__(self, output_shape, n_basis_funcs=10, bounds=(-1, 1), basis_func_degree=3, initialiser=Xavier_Uniform(), activation=None, normalisation=None, **kwargs):
+        if not isinstance(n_basis_funcs, int) or n_basis_funcs <= 0:
+            raise ValueError("n_basis_funcs must be a positive integer.")
+        if not isinstance(bounds, tuple) or len(bounds) != 2 or not isinstance(bounds[0], int) or not isinstance(bounds[1], int) or bounds[0] >= bounds[1]:
+            raise ValueError("bounds should be a tuple of length 2 containing the theoretical minimum and maximum of the data that will be passed in.")
+        if not isinstance(basis_func_degree, int) or basis_func_degree <= 0:
+            raise ValueError("basis_func_degree must be a positive integer.")
+        if n_basis_funcs == basis_func_degree:
+            raise ValueError("n_basis_funcs and basis_func_degree should be different.")
+        if not isinstance(initialiser, Initialiser):
+            raise ValueError('initialiser must be an instance of DLL.DeepLearning.Initialisers')
+        if not isinstance(activation, Activation) and activation is not None:
+            raise ValueError("activation must be from DLL.DeepLearning.Layers.Activations or None.")
+        if not isinstance(normalisation, BaseRegularisation) and normalisation is not None:
+            raise ValueError("normalisation must be from DLL.DeepLearning.Layers.Regularisation or None.")
+
+        super().__init__(output_shape, activation=activation, normalisation=normalisation, **kwargs)
+        self.name = "DenseKAN"
+        self.initialiser = initialiser
+        self.n_basis_funcs = n_basis_funcs
+
+    def initialise_layer(self, input_shape, data_type, device):
+        """
+        :meta private:
+        """
+        super().initialise_layer(input_shape, data_type, device)
+        
+        input_dim = input_shape[-1]
+        output_dim = 1 if len(self.output_shape) == 0 else self.output_shape[-1]
+        
+        self.neurons = [_NeuronKAN(input_dim, self.n_basis_funcs, (-1, 1), self.initialiser, data_type, device, 3) for _ in range(output_dim)]  # TODO: Change the hard coded values to actual parameters
+        self.nparams = self.n_basis_funcs * output_dim * input_dim
+
+    def forward(self, input, training=False, **kwargs):
+        """
+        Applies the forward equation of the Kolmogorov-Arnold network.
+
+        Args:
+            input (torch.Tensor of shape (n_samples, n_features)): The input to the dense layer. Must be a torch.Tensor of the spesified shape given by layer.input_shape.
+            training (bool, optional): The boolean flag deciding if the model is in training mode. Defaults to False.
+            
+        Returns:
+            torch.Tensor of shape (n_samples,) if len(layer.output_shape) == 0 else (n_samples, output_shape): The output tensor after the transformations with the spesified shape.
+        """
+        if not isinstance(input, torch.Tensor):
+            raise TypeError("input must be a torch.Tensor.")
+        if input.shape[1:] != self.input_shape:
+            raise ValueError(f"input is not the same shape as the spesified input_shape ({input.shape[1:], self.input_shape}).")
+        if not isinstance(training, bool):
+            raise TypeError("training must be a boolean.")
+
+        self.input = input
+        output = torch.stack([neuron.forward(input) for neuron in self.neurons], dim=1)
+        if self.normalisation: output = self.normalisation.forward(output, training=training)
+        if self.activation: output = self.activation.forward(output)
+        if len(self.output_shape) == 0: output = output.squeeze(dim=1)  # If the output_shape is a 1d tensor, remove the last dimension
+        return output
+
+    def backward(self, dCdy, **kwargs):
+        """
+        Calculates the gradient of the loss function with respect to the input of the layer. Also calculates the gradients of the loss function with respect to the model parameters.
+
+        Args:
+            dCdy (torch.Tensor of shape (n_samples,) if len(layer.output_shape) == 0 else (n_samples, output_shape)): The gradient given by the next layer.
+            
+        Returns:
+            torch.Tensor of shape (n_samples, layer.input_shape[0]): The new gradient after backpropagation through the layer.
+        """
+        if not isinstance(dCdy, torch.Tensor):
+            raise TypeError("dCdy must be a torch.Tensor.")
+        if dCdy.shape[1:] != self.output_shape:
+            raise ValueError(f"dCdy is not the same shape as the spesified output_shape ({dCdy.shape[1:], self.output_shape}).")
+        
+        if len(self.output_shape) == 0: dCdy = dCdy.unsqueeze(dim=1)  # If the output shape was a 1d tensor, add an extra dimension to the end
+        if self.activation: dCdy = self.activation.backward(dCdy)
+        if self.normalisation: dCdy = self.normalisation.backward(dCdy)
+        dCdx = sum([neuron.backward(dCdy[:, i]) for i, neuron in enumerate(self.neurons)])
+        return dCdx
+
+    def get_parameters(self):
+        """
+        :meta private:
+        """
+        return (*(neuron.weights for neuron in self.neurons), *super().get_parameters())

DLL/DeepLearning/Layers/_Flatten.py

@@ -0,0 +1,55 @@
+import numpy as np
+import torch
+
+from ._BaseLayer import BaseLayer
+
+
+class Flatten(BaseLayer):
+    """
+    The flattening layer.
+    """
+    def __init__(self, **kwargs):
+        super().__init__(None, **kwargs)
+        self.name = "Flatten"
+    
+    def initialise_layer(self, input_shape, data_type, device):
+        """
+        :meta private:
+        """
+        self.output_shape = (np.prod(input_shape),)
+        super().initialise_layer(input_shape, data_type, device)
+    
+    def forward(self, input, **kwargs):
+        """
+        Flattens the input tensor into a 2 dimensional tensor.
+
+        Args:
+            input (torch.Tensor of shape (n_samples, ...)): The input to the layer. Must be a torch.Tensor of the spesified shape given by layer.input_shape.
+            
+        Returns:
+            torch.Tensor of shape (n_samples, product_of_other_dimensions): The output tensor after flattening the input tensor.
+        """
+        if not isinstance(input, torch.Tensor):
+            raise TypeError("input must be a torch.Tensor.")
+        if input.shape[-len(self.input_shape):] != self.input_shape:
+            raise ValueError(f"input is not the same shape as the spesified input_shape ({input.shape[1:], self.input_shape}).")
+
+        self.input = input
+        return input.reshape(input.shape[0], -1)
+    
+    def backward(self, dCdy, **kwargs):
+        """
+        Reshapes the gradient to the original shape.
+
+        Args:
+            dCdy (torch.Tensor of shape (n_samples, product_of_other_dimensions): The gradient given by the next layer.
+            
+        Returns:
+            torch.Tensor of shape (n_samples, *layer.input_shape): The reshaped gradient after backpropagation through the layer.
+        """
+        if not isinstance(dCdy, torch.Tensor):
+            raise TypeError("dCdy must be a torch.Tensor.")
+        if dCdy.shape[1:] != self.output_shape:
+            raise ValueError(f"dCdy is not the same shape as the spesified output_shape ({dCdy.shape[1:], self.output_shape}).")
+
+        return dCdy.reshape(*self.input.shape)

DLL/DeepLearning/Layers/_Identity.py

@@ -0,0 +1,50 @@
+import torch
+
+from .Activations._Activation import Activation
+
+
+class Identity(Activation):
+    """
+    The identity layer.
+    """
+    def __init__(self, **kwargs):
+
+        super().__init__(**kwargs)
+        self.name = "Identity"
+
+    def forward(self, input, training=False, **kwargs):
+        """
+        Returns the input.
+
+        Args:
+            input (torch.Tensor of shape (n_samples, n_features)): The input to the dense layer. Must be a torch.Tensor of the spesified shape given by layer.input_shape.
+            training (bool, optional): The boolean flag deciding if the model is in training mode. Defaults to False.
+            
+        Returns:
+            torch.Tensor: The same tensor as the input
+        """
+        if not isinstance(input, torch.Tensor):
+            raise TypeError("input must be a torch.Tensor.")
+        if input.shape[1:] != self.input_shape:
+            raise ValueError(f"input is not the same shape as the spesified input_shape ({input.shape[1:], self.input_shape}).")
+        if not isinstance(training, bool):
+            raise TypeError("training must be a boolean.")
+
+        return input
+
+    def backward(self, dCdy, **kwargs):
+        """
+        Returns the gradient.
+
+        Args:
+            dCdy (torch.Tensor): The gradient given by the next layer.
+            
+        Returns:
+            torch.Tensor: The same tensor as the input gradient
+        """
+        if not isinstance(dCdy, torch.Tensor):
+            raise TypeError("dCdy must be a torch.Tensor.")
+        if dCdy.shape[1:] != self.output_shape:
+            raise ValueError(f"dCdy is not the same shape as the spesified output_shape ({dCdy.shape[1:], self.output_shape}).")
+        
+        return dCdy

DLL/DeepLearning/Layers/_Input.py

@@ -0,0 +1,11 @@
+from ._BaseLayer import BaseLayer
+
+
+class Input(BaseLayer):
+    def __init__(self, output_shape, **kwargs):
+        super().__init__(output_shape=output_shape, input_shape=output_shape, **kwargs)
+        self.name = "Input"
+
+    def summary(self, offset=""):
+        output_shape = self.output_shape[0] if len(self.output_shape) == 1 else self.output_shape
+        return offset + f"{self.name} - Output: ({output_shape})"