PyDLLib 1.0__py3-none-any.whl

DLL/DeepLearning/Optimisers/_SGD.py

@@ -0,0 +1,44 @@
+import torch
+
+from ._BaseOptimiser import BaseOptimiser
+
+
+class SGD(BaseOptimiser):
+    """
+    Stochastic gradient descent optimiser with momentum. A first order method and therefore does not use information on second gradients, i.e. the hessian matrix. Hence, does not require a lot of memory.
+
+    Args:
+        learning_rate (float, optional): The learning rate of the optimiser. Must be positive. Defaults to 0.001.
+        momentum (float, optional): Determines how long the previous gradients affect the current direction. Must be in range [0, 1). Defaults to 0.9.
+    """
+    def __init__(self, learning_rate=0.001, momentum=0.9):
+        if not isinstance(learning_rate, int | float) or learning_rate <= 0:
+            raise ValueError("learning_rate must be a positive real number.")
+        if not isinstance(momentum, int | float) or momentum < 0 or momentum >= 1:
+            raise ValueError("momentum must be in range [0, 1).")
+
+        self.learning_rate = learning_rate
+        self.momentum = momentum
+    
+    def initialise_parameters(self, model_parameters):
+        """
+        Initialises the optimiser with the parameters that need to be optimised.
+
+        Args:
+            model_parameters (list[torch.Tensor]): The parameters that will be optimised. Must be a list or a tuple of torch tensors.
+        """
+
+        if not isinstance(model_parameters, list | tuple):
+            raise TypeError("model_parameters must be a list or a tuple of torch tensors.")
+
+        self.model_parameters = model_parameters
+        self.changes = [torch.zeros_like(parameter) for parameter in self.model_parameters]
+    
+    def update_parameters(self):
+        """
+        Takes a step towards the optimum for each parameter.
+        """
+        for i, parameter in enumerate(self.model_parameters):
+            change = self.learning_rate * parameter.grad + self.momentum * self.changes[i]
+            parameter -= change
+            self.changes[i] = change

DLL/DeepLearning/Optimisers/__init__.py

@@ -0,0 +1,6 @@
+from ._ADAM import ADAM
+from ._SGD import SGD
+from ._LBFGS import LBFGS
+from ._ADAGRAD import ADAGRAD
+from ._ADADELTA import ADADELTA
+from ._RMSPROP import RMSPROP

DLL/Exceptions/_Exceptions.py

@@ -0,0 +1,19 @@
+class NotFittedError(Exception):
+    """
+    Exception raised when an action is called before the model is fitted.
+
+    Args:
+        message (string): The message which is passed to the Exception class.
+    """
+    def __init__(self, message):
+        super().__init__(message)
+
+class NotCompiledError(Exception):
+    """
+    Exception raised when an action is called before the model is not compiled.
+
+    Args:
+        message (string): The message which is passed to the Exception class.
+    """
+    def __init__(self, message):
+        super().__init__(message)

DLL/Exceptions/__init__.py

@@ -0,0 +1,2 @@
+from ._Exceptions import NotFittedError
+from ._Exceptions import NotCompiledError

DLL/MachineLearning/SupervisedLearning/Calibration/_CalibratedClassifier.py

@@ -0,0 +1,88 @@
+import torch
+
+from ....Data.Preprocessing import data_split
+from ....Data.Metrics import prob_to_pred
+from . import IsotonicRegression
+from ..LinearModels import LogisticRegression
+
+
+class CalibratedClassifier:
+    """
+    The CalibratedClassifier for probability calibration.
+
+    Args:
+        estimator (DLL.Machinelearning classifier): An estimator to calibrate. The estimator must have the predict_proba method defined.
+        method (str, optional): The method used for the calibration. Must be one of "logistic" or "isotonic". Defaults to "logistic".
+        learning_rate (float, optional): The learning rate for fitting the logistic regression. Must be a positive real number. If method == "isotonic", this parameter is ignored. Defaults to 0.01.
+    """
+    def __init__(self, estimator, method="logistic", learning_rate=0.01):
+        if not hasattr(estimator, "predict_proba"):
+            raise TypeError("The estimator must have the predict_proba method defined.")
+        if method not in ["isotonic", "logistic"]:
+            raise ValueError('The chosen method be one of "logistic" or "isotonic".')
+        if not isinstance(learning_rate, int | float) or learning_rate <= 0:
+            raise ValueError("learning_rate must be a positive real number.")
+
+        self.estimator = estimator
+        self.method = method
+        self.calibrator = LogisticRegression(learning_rate) if method == "logistic" else IsotonicRegression()
+    
+    def fit(self, X, y, calibration_size=0.2, **kwargs):
+        """
+        Fits chosen classifier to the input data and the output labels trying to be as accurate as possible about the probabilities of each class. Uses part of the data for the calibration and not for the fitting.
+
+        Args:
+            X (torch.Tensor of shape (n_samples, n_features)): The input data, where each row is a sample and each column is a feature.
+            y (torch.Tensor of shape (n_samples,)): The labels corresponding to each sample. Every element must be in [0, ..., n_classes - 1].
+            calibration_size (float, optional): The amount of data used for the model calibration. Defaults to 0.2.
+            **kwargs: Key word arguments, which are passed to the fit method of the chosen estimator.
+        """
+        if not isinstance(X, torch.Tensor) or not isinstance(y, torch.Tensor):
+            raise TypeError("The inpu matrix X and the label matrix y must be PyTorch tensors.")
+        if X.ndim != 2:
+            raise ValueError("The input matrix must be a 2 dimensional tensor.")
+        if y.ndim != 1 or len(y) != len(X):
+            raise ValueError("The labels must be 1 dimensional with the same number of samples as the input data")
+        if not isinstance(calibration_size, int | float) or calibration_size <= 0 or calibration_size >= 1:
+            raise ValueError("calibration_size must be in range (0, 1).")
+        vals = torch.unique(y).numpy()
+        if set(vals) != {*range(len(vals))}:
+            raise ValueError("y must only contain the values in [0, ..., n_classes - 1].")
+
+        X_train, y_train, _, _, X_cal, y_cal = data_split(X, y, train_split=1 - calibration_size, validation_split=0.0)
+        self.estimator.fit(X_train, y_train, **kwargs)
+        probs = self.estimator.predict_proba(X_cal)
+        if self.method == "logistic": probs = probs.unsqueeze(1)
+        self.calibrator.fit(self._logit(probs), y_cal, epochs=1000) if self.method == "logistic" else self.calibrator.fit(probs, y_cal)
+
+    def predict_proba(self, X):
+        """
+        Predicts the calibrated probabilities of the data points.
+
+        Args:
+            X (torch.Tensor of shape (n_samples, n_features)): The input data, where each row is a sample and each column is a feature.
+
+        Returns:
+            torch.Tensor of shape (n_samples, n_classes) or (n_samples,): The predicted probabilities.
+        """
+        probs = self.estimator.predict_proba(X)
+        if self.method == "logistic": probs = probs.unsqueeze(1)
+        calibrated_probs = self.calibrator.predict_proba(self._logit(probs)) if self.method == "logistic" else self.calibrator.predict(probs)
+        return calibrated_probs
+
+    def predict(self, X):
+        """
+        Predicts the classes of each data point using the calibrated probabilities.
+
+        Args:
+            X (torch.Tensor of shape (n_samples, n_features)): The input data, where each row is a sample and each column is a feature.
+
+        Returns:
+            torch.Tensor of shape (n_samples, n_classes) or (n_samples,): The predicted probabilities.
+        """
+        return prob_to_pred(self.predict_proba(X))
+
+    def _logit(self, X):
+        X[X < 1e-6] = 1e-6
+        X[X > 1 - 1e-6] = 1 - 1e-6
+        return torch.log(X / (1 - X))

DLL/MachineLearning/SupervisedLearning/Calibration/_IsotonicRegression.py

@@ -0,0 +1,114 @@
+import torch
+from functools import partial
+
+
+class IsotonicRegression:
+    """
+    Isotonic regression model.
+    """
+    def fit(self, X, y, weight=None, increasing=True):
+        """
+        Fits a monotonic function to the trainin data using the Pool-Adjacent-Violators (PAV) algorithm.
+
+        Args:
+            X (torch.Tensor of shape (n_samples,)): Independent variable (not necessarily sorted).
+            y (torch.Tensor of shape (n_samples,)): Dependent variable.
+            weight (torch.Tensor of shape (n_samples,) | None, optional): Sample weights. Must be a non-negative torch tensor. Defaults to None, which corresponds to uniform weights.
+            increasing (bool, optional): Deterimines if the final predictions should be increasing or decreasing. Must be a boolean. Defaults to True.
+        """
+        if not isinstance(X, torch.Tensor) or not isinstance(y, torch.Tensor):
+            raise TypeError("The input matrix X and the label matrix y must be PyTorch tensors.")
+        if not isinstance(weight, torch.Tensor | None):
+            raise TypeError("The weight matrix must be a PyTorch tensor or None.")
+        if X.ndim != 1:
+            raise ValueError("The input matrix must be a 1 dimensional tensor.")
+        if y.ndim != 1 or len(y) != len(X):
+            raise ValueError("The labels must be 1 dimensional with the same number of samples as the input data.")
+        if weight is not None and (weight.ndim != 1 or len(weight) != len(X)):
+            raise ValueError("The weights must be 1 dimensional with the same number of samples as the input data.")
+        if not isinstance(increasing, bool):
+            raise TypeError("increasing must be a boolean.")
+
+        sorted_idx = torch.argsort(X)
+        X_sorted = X[sorted_idx]
+        y_sorted = y[sorted_idx]
+        if weight is None:
+            weight = torch.ones_like(y_sorted)
+        else:
+            weight = weight[sorted_idx]
+        if not increasing:
+            y_sorted = y_sorted.flip(0)
+            weight = weight.flip(0)
+
+        solution, _ = self._pava(y_sorted, weight)
+        if not increasing:
+            solution = solution.flip(0)
+        self._interp_func = partial(self._linear_interpolation, X_sorted, solution)
+    
+    def _pava(self, y, w):
+        """
+        Adapted from sklearn docs: https://github.com/scikit-learn/scikit-learn/blob/main/sklearn/_isotonic.pyx
+        """
+        n = len(y)
+        y, w = y.clone(), w.clone()
+        target = torch.arange(n, dtype=torch.int32)
+        i = 0
+        while i < n:
+            k = target[i].item() + 1
+            if k == n:
+                break
+            if y[i] < y[k]:
+                i = k
+                continue
+            sum_wy = w[i] * y[i]
+            sum_w = w[i]
+            while True:
+                prev_y = y[k]
+                sum_wy += w[k] * y[k]
+                sum_w += w[k]
+                k = target[k].item() + 1
+                if k == n or prev_y < y[k]:
+                    y[i] = sum_wy / sum_w
+                    w[i] = sum_w
+                    target[i] = k - 1
+                    target[k - 1] = i
+                    if i > 0:
+                        i = target[i - 1].item()
+                    break
+        i = 0
+        while i < n:
+            k = target[i] + 1
+            y[i + 1:k] = y[i]
+            i = k
+        return y, w
+    
+    def _linear_interpolation(self, X_fit, y_fit, X_new):
+        X_new = X_new.unsqueeze(0) if X_new.dim() == 0 else X_new
+        y_interp = torch.zeros_like(X_new).float()
+
+        for i, x in enumerate(X_new):
+            if x <= X_fit[0]:
+                y_interp[i] = y_fit[0]
+            elif x >= X_fit[-1]:
+                y_interp[i] = y_fit[-1]
+            else:
+                idx = torch.searchsorted(X_fit, x, right=True) - 1
+                x0, x1 = X_fit[idx], X_fit[min(idx + 1, len(X_fit) - 1)]
+                y0, y1 = y_fit[idx], y_fit[min(idx + 1, len(X_fit) - 1)]
+                y_interp[i] = y0 + (y1 - y0) * (x - x0) / (x1 - x0)
+        return y_interp
+
+    def predict(self, X):
+        """
+        Predicts values using the fitted isotonic regression model with linear interpolation.
+
+        Parameters:
+            X (torch.Tensor): New independent variable values.
+
+        Returns:
+            torch.Tensor: Predicted values.
+        """
+        if not isinstance(X, torch.Tensor) or X.ndim != 1:
+            raise ValueError("The input matrix must be a torch.Tensor of shape (n_samples,).")
+
+        return self._interp_func(X)

DLL/MachineLearning/SupervisedLearning/Calibration/__init__.py

@@ -0,0 +1,2 @@
+from ._IsotonicRegression import IsotonicRegression
+from ._CalibratedClassifier import CalibratedClassifier

DLL/MachineLearning/SupervisedLearning/GaussianProcesses/_GaussianProcessRegressor.py

@@ -0,0 +1,190 @@
+import torch
+from math import floor
+
+from ..Kernels import _Base
+from ....DeepLearning.Optimisers import ADAM
+from ....DeepLearning.Optimisers._BaseOptimiser import BaseOptimiser
+from ....Exceptions import NotFittedError
+from ....Data.Metrics import _round_dictionary
+
+
+class GaussianProcessRegressor:
+    """
+    Implements the Gaussian process regression model.
+
+    Args:
+        covariance_function (:ref:`kernel_section_label`, optional): The kernel function expressing how similar are different samples.
+        noise (int | float, optional): The artificially added noise to the model. Is added as variance to each sample. Must be non-negative. Defaults to 0.
+        epsilon (float, optional): Implemented similarly to noise. Makes sure the covariance matrix is positive definite and hence invertible. Must be positive. Defaults to 1e-5. If one gets a RunTimeError for a matrix not being invertible, one should increase this parameter.
+        device (torch.device, optional): The device of all matricies. Defaults to torch.device("cpu").
+        
+    Attributes:
+        n_features (int): The number of features. Available after fitting.
+    """
+    def __init__(self, covariance_function, noise=0, epsilon=1e-5, device=torch.device("cpu")):
+        if not isinstance(covariance_function, _Base):
+            raise TypeError("covariance_function must be from DLL.MachineLearning.Supervisedlearning.Kernels.")
+        if not isinstance(noise, int | float) or noise < 0:
+            raise ValueError("noise must b non-negative.")
+        if not isinstance(epsilon, int | float) or epsilon <= 0:
+            raise ValueError("epsilon must be positive.")
+        if not isinstance(device, torch.device):
+            raise TypeError("device must be an instance of torch.device.")
+
+        self.covariance_function = covariance_function
+        self.noise = noise
+        self.epsilon = epsilon
+        self.device = device
+
+    def _get_covariance_matrix(self, X1, X2):
+        return self.covariance_function(X1, X2).to(X1.dtype).to(self.device)
+
+    def fit(self, X, y):
+        """
+        Fits the GaussianProcessRegressor model to the input data.
+
+        Args:
+            X (torch.Tensor of shape (n_samples, n_features)): The input data, where each row is a sample and each column is a feature.
+            y (torch.Tensor of shape (n_samples,)): The target values corresponding to each sample. Should be normalized to zero mean and one variance.
+        Returns:
+            None
+        Raises:
+            TypeError: If the input matrix or the target matrix is not a PyTorch tensor.
+            ValueError: If the input matrix or the target matrix is not the correct shape.
+        """
+        if not isinstance(X, torch.Tensor) or not isinstance(y, torch.Tensor):
+            raise TypeError("The input matrix and the target matrix must be a PyTorch tensor.")
+        if X.ndim != 2:
+            raise ValueError("The input matrix must be a 2 dimensional tensor.")
+        if y.ndim != 1 or len(y) != len(X):
+            raise ValueError("The targets must be 1 dimensional with the same number of samples as the input data")
+        if len(y) < 2:
+            raise ValueError("There must be atleast 2 samples.")
+        # variance = torch.var(y)
+        # mean = torch.mean(y)
+        # if not torch.allclose(variance, torch.ones_like(variance)):
+        #     raise ValueError("y must have one variance. Use DLL.Data.Preprocessing.StandardScaler to scale the data.")
+        # if not torch.allclose(mean, torch.zeros_like(mean)):
+        #     raise ValueError("y must have zero mean. Use DLL.Data.Preprocessing.StandardScaler to scale the data.")
+
+        self.n_features = X.shape[1]
+        self.X = X
+        self.Y = y.unsqueeze(dim=1)
+        self.prior_covariance_matrix = self._get_covariance_matrix(X, X) + (self.noise + self.epsilon) * torch.eye(len(X), device=self.device)
+        self.inverse_prior_covariance_matrix = torch.linalg.inv(self.prior_covariance_matrix)
+
+    def predict(self, X):
+        """
+        Applies the fitted GaussianProcessRegressor model to the input data, predicting the correct values.
+
+        Args:
+            X (torch.Tensor of shape (n_samples, n_features)): The input data to be regressed.
+        Returns:
+            mean, covariance (tuple[torch.Tensor of shape (n_samples,), torch.Tensor of shape (n_samples, n_samples)): A tuple containing the posterior mean and posterior covariance. As the prediction, one should use the mean.
+        Raises:
+            NotFittedError: If the GaussianProcessRegressor model has not been fitted before predicting.
+            TypeError: If the input matrix is not a PyTorch tensor.
+            ValueError: If the input matrix is not the correct shape.
+        """
+        if not hasattr(self, "inverse_prior_covariance_matrix"):
+            raise NotFittedError("GaussianProcessRegressor.fit() must be called before predicting.")
+        if not isinstance(X, torch.Tensor):
+            raise TypeError("The input matrix must be a PyTorch tensor.")
+        if X.ndim != 2 or X.shape[1] != self.n_features:
+            raise ValueError("The input matrix must be a 2 dimensional tensor with the same number of features as the fitted tensor.")
+
+        k_1 = self._get_covariance_matrix(self.X, X)
+        k_2 = self._get_covariance_matrix(X, X) + (self.noise + self.epsilon) * torch.eye(len(X), device=self.device)
+        mean = k_1.T @ self.inverse_prior_covariance_matrix @ self.Y
+        posterior_covariance = k_2 - k_1.T @ self.inverse_prior_covariance_matrix @ k_1
+        return mean, posterior_covariance
+    
+    def log_marginal_likelihood(self):
+        """
+        Computes the log marginal likelihood of the current model. This value is used to optimize hyperparameters.
+
+        Returns:
+            log marginal likelihood (float): The log marginal likelihood of the current model.
+        """
+        if not hasattr(self, "inverse_prior_covariance_matrix"):
+            raise NotFittedError("GaussianProcessRegressor.fit() must be called before calculating the log marginal likelihood.")
+
+        L = torch.linalg.cholesky(self.prior_covariance_matrix)
+        alpha = torch.cholesky_solve(self.Y, L)
+        lml = -0.5 * self.Y.T @ alpha - torch.sum(torch.log(torch.diagonal(L))) - 0.5 * len(self.Y) * torch.log(torch.tensor(2 * torch.pi, device=self.device))
+        return lml.item()
+    
+    def train_kernel(self, epochs=10, optimiser=None, callback_frequency=1, verbose=False):
+        """
+        Trains the current covariance function parameters by maximizing the log marginal likelihood.
+
+        Args:
+            epochs (int, optional): The number of optimisation rounds. Must be a positive integer. Defaults to 10.
+            optimiser (:ref:`optimisers_section_label` | None, optional): The optimiser used for training the model. If None, the Adam optimiser is used.
+            callback_frequency (int, optional): The number of iterations between printing info from training. Must be a positive integer. Defaults to 1, which means that every iteration, info is printed assuming verbose=True.
+            verbose (bool, optional): If True, prints the log marginal likelihood of the model during training. Defaults to False.
+        
+        Returns:
+            history (dict[str, torch.Tensor], the tensor is floor(epochs / callback_frequency) long.): A dictionary tracking the evolution of the log marginal likelihood at intervals defined by callback_frequency. The tensor can be accessed with history["log marginal likelihood"].
+
+        Raises:
+            NotFittedError: If the GaussianProcessRegressor model has not been fitted before training the kernel.
+            TypeError: If the parameters are of wrong type.
+            ValueError: If epochs is not a positive integer.
+        """
+        if not hasattr(self, "inverse_prior_covariance_matrix"):
+            raise NotFittedError("GaussianProcessRegressor.fit() must be called before calculating the log marginal likelihood.")
+        if not isinstance(epochs, int) or epochs <= 0:
+            raise ValueError("epochs must be a positive integer.")
+        if not isinstance(optimiser, BaseOptimiser) and optimiser is not None:
+            raise TypeError("optimiser must be from DLL.DeepLearning.Optimisers")
+        if not isinstance(verbose, bool):
+            raise TypeError("verbose must be a boolean.")
+
+        optimiser = ADAM() if optimiser is None else optimiser
+        optimiser.initialise_parameters(list(self.covariance_function.parameters().values()))
+
+        history = {"log marginal likelihood": torch.zeros(floor(epochs / callback_frequency))}
+
+        for epoch in range(epochs):
+            # form the derivative function
+            def derivative(parameter_derivative):
+                derivative = 0.5 * self.Y.T @ self.inverse_prior_covariance_matrix @ parameter_derivative @ self.inverse_prior_covariance_matrix @ self.Y
+
+                if parameter_derivative.ndim == 2:
+                    derivative = derivative.squeeze(1)
+                    derivative -= 0.5 * torch.trace(self.inverse_prior_covariance_matrix @ parameter_derivative)
+                else:
+                    derivative = derivative.squeeze((1, 2))
+                    derivative -= 0.5 * (self.inverse_prior_covariance_matrix @ parameter_derivative).diagonal(dim1=1, dim2=2).sum(dim=-1)
+                # minus sign, since goal is to maximise the log_marginal_likelihood
+                return -derivative
+
+            # calculate the derivatives
+            self.covariance_function.update(derivative, self.X)
+
+            # update the parameters
+            optimiser.update_parameters()
+
+            self.prior_covariance_matrix = self._get_covariance_matrix(self.X, self.X) + (self.noise + self.epsilon) * torch.eye(len(self.X), device=self.device)
+            self.inverse_prior_covariance_matrix = torch.linalg.inv(self.prior_covariance_matrix)
+            if epoch % callback_frequency == 0:
+                lml = self.log_marginal_likelihood()
+                history["log marginal likelihood"][int(epoch / callback_frequency)] = lml
+                if verbose:
+                    params = {}
+                    for key, param in self.covariance_function.parameters().items():
+                        if len(param) == 1:
+                            params[key] = round(param.item(), 3)
+                            continue
+                        for i, param_ in enumerate(param):
+                            params[key + "_" + str(i + 1)] = round(param_.item(), 3)
+                    print(f"Epoch: {epoch + 1} - Log marginal likelihood: {lml} - Parameters: {params}")
+        return history
+
+    # def is_positive_definite(matrix):
+    #     print("-----------------")
+    #     if not torch.allclose(matrix, matrix.T):
+    #         print(f"Matrix not symmetric: {torch.linalg.norm(matrix.T - matrix)}")
+    #     eigenvalues = torch.linalg.eigvalsh(matrix)
+    #     print(f"Minimum eigenvalue: {torch.min(eigenvalues).item()}")

DLL/MachineLearning/SupervisedLearning/GaussianProcesses/__init__.py

@@ -0,0 +1 @@
+from ._GaussianProcessRegressor import GaussianProcessRegressor