PyDLLib 1.0__py3-none-any.whl

DLL/MachineLearning/UnsupervisedLearning/DimensionalityReduction/_QDA.py

@@ -0,0 +1,69 @@
+import torch
+
+from ....Exceptions import NotFittedError
+
+
+class QDA:
+    """
+    Quadratic discriminant analysis (LDA) class for classification.
+
+    Attributes:
+        n_features (int): The number of features in the input.
+    """
+
+    def fit(self, X, y):
+        """
+        Fits the QDA model to the input data by calculating the class means and covariances.
+        
+        Args:
+            X (torch.Tensor of shape (n_samples, n_features)): The input data, where each row is a sample and each column is a feature.
+        Raises:
+            TypeError: If the input matrix is not a PyTorch tensor.
+            ValueError: If the input matrix is not the correct shape.
+        """
+        if not isinstance(X, torch.Tensor):
+            raise TypeError("The input matrix must be a PyTorch tensor.")
+        if X.ndim != 2 or X.shape[0] == 1:
+            raise ValueError("The input matrix must be a 2 dimensional tensor with atleast 2 samples.")
+        
+        self.classes = torch.unique(y)
+        self.n_features = X.shape[1]
+
+        self.class_means = []
+        self.class_covariances = []
+
+        for current_class in self.classes:
+            X_c = X[y == current_class]
+            self.class_means.append(torch.mean(X_c, dim=0))
+            self.class_covariances.append(torch.cov(X_c.T, correction=0))
+
+    def predict(self, X):
+        """
+        Applies the fitted QDA model to the input data, predicting the correct classes.
+
+        Args:
+            X (torch.Tensor of shape (n_samples, n_features)): The input data to be transformed.
+        Returns:
+            y (torch.Tensor of shape (n_samples,)): The predicted labels.
+        Raises:
+            NotFittedError: If the QDA model has not been fitted before transforming.
+        """
+        if not hasattr(self, "class_means"):
+            raise NotFittedError("QDA.fit() must be called before predicting.")
+        if not isinstance(X, torch.Tensor):
+            raise TypeError("The input matrix must be a PyTorch tensor.")
+        if X.ndim != 2 or X.shape[0] == 1:
+            raise ValueError("The input matrix must be a 2 dimensional tensor with atleast 2 samples.")
+
+        min_dists = torch.full((len(X),), torch.inf, dtype=X.dtype)
+        y = torch.zeros_like(min_dists, dtype=X.dtype)
+        
+        for current_class, class_mean, class_covariance in zip(self.classes, self.class_means, self.class_covariances):
+            diff = X - class_mean
+            # mahalanobis = torch.sum(diff @ torch.linalg.lstsq(class_covariance, diff.T).solution, dim=1)
+            # mahalanobis = torch.sum(diff @ torch.linalg.inv(class_covariance) * diff, dim=1)
+            mahalanobis = torch.sum(torch.linalg.lstsq(class_covariance, diff.T).solution.T * diff, dim=1)
+            mask = mahalanobis < min_dists
+            y[mask] = current_class
+            min_dists[mask] = mahalanobis[mask]
+        return y

DLL/MachineLearning/UnsupervisedLearning/DimensionalityReduction/_RobustPCA.py

@@ -0,0 +1,143 @@
+import torch
+from math import sqrt, floor
+
+from ....Exceptions import NotFittedError
+from . import PCA
+
+
+class RobustPCA:
+    """
+    Robust version of Principal Component Analysis (PCA).
+
+    Args:
+        n_components (int): Number of principal components to keep. The number must be a positive integer.
+        method (str, optional): The method used for the robust estimator. Must be one of "mcd" or "decomposition". Defaults to "mcd".
+    Attributes:
+        components (torch.Tensor): Principal components extracted from the data.
+        explained_variance (torch.Tensor): Variance explained by the selected components.
+    """
+    def __init__(self, n_components=2, method="mcd", epsilon=1e-10):
+        if not isinstance(n_components, int) or n_components < 1:
+            raise ValueError("n_components must be a positive integer.")
+        if method not in ["mcd", "decomposition"]:
+            raise ValueError('method must be one of "mcd" or "decomposition".')
+        
+        self.n_components = n_components
+        self.method = method
+        self.epsilon = epsilon
+        self.pca = PCA(n_components, epsilon)
+
+    def fit(self, X, epochs=1000, normalize=True, proportion=0.5, n_tries=20):
+        """
+        Fits the RobustPCA model to the input data by calculating the principal components. If method is "decomposition", finds matricies L and S such that X = L + S, where L is low rank and S is sparse. Then applies PCA to L. If method is mcd, tries to find a subset of the data that has the minimum covariance determinant and uses that as an estimate of the covariance.
+    
+        The input data is always centered and if `normalize=True`, also normalized so that the standard deviation is 1 along each axis.
+
+        Args:
+            X (torch.Tensor of shape (n_samples, n_features)): The input data, where each row is a sample and each column is a feature.
+            epochs (int, optional): Determines how many iterations are used for finding the L and S matricies. Defaults to 1000. Must be a positive integer.
+            normalize (bool, optional): Whether to normalize the data before computing the PCA. Defaults to True.
+            proportion (float, optional): The proportion of data, which is used to approximate the covariance matrix. Is ignored unless method is "mcd". Must be in range (0, 1). Defaults to 0.5.
+            n_tries (int, optional): The amount of attempts of randomly selecting the samples for the covariance estimate. Must be a positive integer. Defaults to 20.
+        """
+        if not isinstance(X, torch.Tensor):
+            raise TypeError("The input matrix must be a PyTorch tensor.")
+        if not isinstance(epochs, int) or epochs <= 0:
+            raise ValueError("epochs must be positive integer.")
+        if not isinstance(normalize, bool):
+            raise TypeError("The normalize parameter must be a boolean.")
+        if X.ndim != 2 or X.shape[0] == 1:
+            raise ValueError("The input matrix must be a 2 dimensional tensor with atleast 2 samples.")
+        if not isinstance(proportion, float) or proportion <= 0 or proportion >= 1:
+            raise ValueError("proportion must be in range (0, 1).")
+        if not isinstance(n_tries, int) or n_tries <= 0:
+            raise ValueError("n_tries must be a positive integer.")
+
+        self.normalize = normalize
+        if self.method == "decomposition":
+            n1, n2 = X.shape
+            mu = n1 * n2 / (4 * torch.sum(torch.abs(X)))
+            l = 1 / sqrt(max(n1, n2))
+            thresh = 1e-7 * torch.linalg.norm(X, ord="fro")
+            L = torch.zeros_like(X)
+            S = torch.zeros_like(X)
+            Y = torch.zeros_like(X)
+            i = 0
+            while torch.linalg.norm(X - L - S, ord="fro") > thresh and i < epochs:
+                L = self._SVT(X - S + (1 / mu) * Y, 1 / mu)
+                S = self._shrink(X - L + (1 / mu) * Y, l / mu)
+                Y = Y + mu * (X - L - S)
+                i = i + 1
+            self.pca.fit(L, normalize, False)
+        elif self.method == "mcd":
+            covariance, self.mean, self.std = self._min_cov_det(X, proportion, normalize, n_tries)
+            self.pca.fit(covariance, normalize, True)
+
+    def _shrink(self, X, tau):
+        return torch.sign(X) * torch.max(torch.abs(X) - tau, torch.zeros_like(X))
+
+    def _SVT(self, X, tau):
+        U, S, Vh = torch.linalg.svd(X, full_matrices=False)
+        return U @ self._shrink(torch.diag(S), tau) @ Vh
+    
+    def _min_cov_det(self, X, proportion, normalize, n_tries):
+        N = floor(proportion * len(X))
+        best_covariance_matrix = torch.cov(X.T)
+        lowest_determinant = torch.linalg.det(best_covariance_matrix)
+        best_mean = X.mean(dim=0)
+        best_std = X.std(dim=0, unbiased=True) if normalize else None
+        for _ in range(n_tries):
+            indices = torch.randperm(len(X))[:N]
+            mean = X[indices].mean(dim=0)
+            X_centered = X[indices] - mean
+            if normalize:
+                std = X_centered.std(dim=0, unbiased=True)
+                X_centered = X_centered / (std + self.epsilon)
+            covariance = torch.cov(X_centered.T)
+            determinant = torch.linalg.det(covariance)
+            if determinant < lowest_determinant:
+                best_covariance_matrix = covariance
+                lowest_determinant = determinant
+                best_mean = mean
+                if normalize: best_std = std
+        return best_covariance_matrix, best_mean, best_std
+    
+    def transform(self, X):
+        """
+        Applies the fitted RobustPCA model to the input data, transforming it into the reduced feature space.
+
+        Args:
+            X (torch.Tensor of shape (n_samples, n_features)): The input data to be transformed.
+        Returns:
+            X_new (torch.Tensor of shape (n_samples, n_components)): The data transformed into the principal component space.
+        Raises:
+            NotFittedError: If the RobustPCA model has not been fitted before transforming.
+            TypeError: If the input matrix is not a PyTorch tensor.
+            ValueError: If the input matrix is not the correct shape.
+        """
+        if not hasattr(self.pca, "from_covariance"):
+            raise NotFittedError("RobustPCA.fit() must be called before transforming.")
+        if not isinstance(X, torch.Tensor):
+            raise TypeError("The input matrix must be a PyTorch tensor.")
+        if X.ndim != 2 or X.shape[1] != self.pca.components.shape[1]:
+            raise ValueError("The input matrix must be a 2 dimensional tensor with the same number of features as the fitted tensor.")
+
+        if self.method == "mcd":
+            X = X - self.mean
+            if self.normalize: X = X / (self.std + self.epsilon)
+        return self.pca.transform(X)
+    
+    def fit_transform(self, X, epochs=1000, normalize=True):
+        """
+        First finds the principal components of X and then transforms X to fitted space.
+
+        Args:
+            X (torch.Tensor of shape (n_samples, n_features)): The input data to be transformed.
+            epochs (int, optional): Determines how many iterations are used for finding the L and S matricies. Defaults to 1000. Must be a positive integer.
+            normalize (bool, optional): Whether to normalize the data before computing the PCA. Defaults to True.
+        Returns:
+            X_new (torch.Tensor of shape (n_samples, n_components)): The data transformed into the principal component space.
+        """
+        
+        self.fit(X, epochs=epochs, normalize=normalize)
+        return self.transform(X)

DLL/MachineLearning/UnsupervisedLearning/DimensionalityReduction/_TSNE.py

@@ -0,0 +1,151 @@
+import torch
+
+from . import PCA
+from ....DeepLearning.Optimisers import SGD
+
+
+class TSNE:
+    """
+    T-distributed Stochastic Neighbor Embedding (T-SNE) class for dimensionality reduction. This implementation is based on `this paper <https://jmlr.org/papers/volume9/vandermaaten08a/vandermaaten08a.pdf>`_ and `this article <https://towardsdatascience.com/t-sne-from-scratch-ft-numpy-172ee2a61df7/>`_. The main difference is that this implementation uses vectorized matrix operations making it considerably faster than the loop approach used in the article.
+
+    Args:
+        n_components (int): Number of principal components to keep. The number must be a positive integer.
+        init (str, optional): The method for initializing the embedding. Must be in ``["pca", "random"]``. Defaults to ``"pca"``.
+        p (int, optional): The order of the chosen metric. Must be a positive integer. Defaults to 2, which corresponds to the Euclidian metric.
+        early_exaggeration (float | int, optional): Determines how far apart the clusters are in the embedding space. Must be a positive real number. Defaults to 12.0.
+        perplexity (float | int, optional): Determines how far can samples be from one another to be considered neighbors. Must be a positive real number. Defaults to 30.0. One should consider using something between 5 and 50 to begin with.
+        learning_rate (float | int, optional): Determines how long steps do we take towards the gradient. Must be a positive real number. It is recommended to use a value between 10.0 and 1000.0. Defaults to ``"auto"``, where we use a value of ``max(n_samples / (4 * early_exaggeration), 50)``.
+    
+    Attributes:
+        history (list[float]): The history of KL-divergence loss function each epoch. Available after fitting the model.
+    """
+    def __init__(self, n_components=2, init="pca", p=2, early_exaggeration=12.0, perplexity=30.0, learning_rate="auto"):
+        if not isinstance(n_components, int) or n_components < 1:
+            raise ValueError("n_components must be a positive integer.")
+        if not init in ["pca", "random"]:
+            raise ValueError('init must be in ["pca", "random"].')
+        if not isinstance(p, int) or p <= 0:
+            raise ValueError("p must be a positive integer.")
+        if not isinstance(early_exaggeration, float | int) or early_exaggeration <= 0:
+            raise ValueError("early_exaggeration must be a positive real number.")
+        if not isinstance(perplexity, float | int) or perplexity <= 0:
+            raise ValueError("perplexity must be a positive real number.")
+        if (learning_rate != "auto" and (not isinstance(learning_rate, float | int) or learning_rate <= 0)):
+            raise ValueError('learning_rate must be a positive real number or "auto".')
+
+        self.n_components = n_components
+        self.init = init
+        self.p = p
+        self.early_exaggeration = early_exaggeration
+        self.perplexity = perplexity
+        self.learning_rate = learning_rate
+    
+    def _initialize(self, X):
+        if self.init == "pca":
+            return PCA(n_components=self.n_components).fit_transform(X)
+        else:
+            return torch.normal(0, 1e-4, size=(len(X), self.n_components))
+    
+    def _symmetric_affinities(self, X):
+        affinities = self._pairwise_affinities(X)
+        p_ij_symmetric = (affinities + affinities.T) / (2 * len(affinities))
+        eps = torch.tensor(torch.finfo(p_ij_symmetric.dtype).tiny)
+        p_ij_symmetric = torch.maximum(p_ij_symmetric, eps)
+        return p_ij_symmetric
+    
+    def _pairwise_affinities(self, X):
+        diff = X.unsqueeze(1) - X.unsqueeze(0)
+        norm = torch.norm(diff, dim=2, p=self.p)
+        variance = self._grid_search(norm)
+        affinities = torch.exp(-norm ** self.p / (2 * variance[:, None] ** 2))
+        affinities.fill_diagonal_(0)
+        row_sums = affinities.sum(dim=1, keepdim=True)
+        zero_rows = row_sums == 0
+        eps = torch.finfo(affinities.dtype).tiny
+        row_sums[zero_rows] = eps
+        affinities /= row_sums
+        affinities[affinities == 0] = eps
+        return affinities
+
+    def _grid_search(self, norm):
+        n_samples = len(norm)
+        std_norm = torch.std(norm, dim=1, keepdim=True)
+        resolution = 200
+        sigma_search_values = torch.linspace(0.01, 5.0, resolution)
+        sigma_search_values = std_norm * sigma_search_values
+        p_matrix = torch.exp(-norm.unsqueeze(2) ** self.p / (2 * sigma_search_values.pow(2).unsqueeze(1)))
+        eps = torch.tensor(torch.finfo(p_matrix.dtype).tiny)
+        p_matrix[torch.eye(n_samples, dtype=torch.bool)] = eps  # do not set to 0 to avoid division by zero errors
+        p_new_matrix = torch.maximum(p_matrix / p_matrix.sum(dim=1, keepdim=True), eps)
+        H_values = -torch.sum(p_new_matrix * torch.log2(p_new_matrix), dim=1)
+        result = torch.abs(torch.log(torch.tensor(self.perplexity)) - H_values * torch.log(torch.tensor(2.0)))
+        sigma_indices = torch.argmin(result, dim=1)
+        return sigma_search_values[torch.arange(n_samples), sigma_indices]
+
+    def _low_dimensional_affinities(self, low_dimensional_representation):
+        diff = low_dimensional_representation.unsqueeze(1) - low_dimensional_representation.unsqueeze(0)
+        norm = torch.norm(diff, dim=2, p=self.p)
+        result = (1 + norm ** self.p).pow(-1)
+        result.fill_diagonal_(0)
+        result_sum = result.sum()
+        result = result / result_sum if result_sum > 0 else result
+        eps = torch.tensor(torch.finfo(result.dtype).tiny)
+        return torch.maximum(result, eps)
+
+    def _gradient(self, affinities, low_dimensional_affinities, low_dimensional_representation):
+        diff = low_dimensional_representation.unsqueeze(1) - low_dimensional_representation.unsqueeze(0)
+        A = affinities - low_dimensional_affinities
+        B = (1 + torch.norm(diff, dim=2, p=self.p)).pow(-1)
+        return 4 * torch.sum(((A * B).unsqueeze(2) * diff), dim=1)
+
+    def fit_transform(self, X, epochs=100, verbose=False):
+        """
+        Fits the T-SNE model to the input data.
+
+        Args:
+            X (torch.Tensor of shape (n_samples, n_features)): The input data, where each row is a sample and each column is a feature.
+            epochs (int, optional): The number of training epochs after early exaggeration. Must be a positive integer. Defaults to 100. Due to early exaggeration, the embedding is updated epochs + 250 times.
+            verbose (bool, optional): Determines if the loss is printed each epoch. Must be a boolean. Defaults to ``False``.
+        Returns:
+            embedding (torch.tensor): The embedded samples of shape (n_samples, n_components).
+        """
+        if not isinstance(X, torch.Tensor):
+            raise TypeError("The input matrix must be a PyTorch tensor.")
+        if X.ndim != 2 or X.shape[0] == 1:
+            raise ValueError("The input matrix must be a 2 dimensional tensor with atleast 2 samples.")
+        if not isinstance(epochs, int) or epochs <= 0:
+            raise ValueError("epochs must be a positive integer.")
+        if not isinstance(verbose, bool):
+            raise ValueError("verbose must be a boolean.")
+
+        result = self._initialize(X)
+        symmetric_affinities = self._symmetric_affinities(X)
+        learning_rate = max(len(X) / (4 * self.early_exaggeration), 50) if self.learning_rate == "auto" else self.learning_rate
+        optimiser = SGD(learning_rate=learning_rate, momentum=0.5)
+        optimiser.initialise_parameters([result])
+        self.history = []
+        early_exaggeration = self.early_exaggeration
+
+        for epoch in range(epochs + 250):
+            if epoch > 250:
+                optimiser.momentum = 0.8
+                early_exaggeration = 1
+            low_dimensional_affinities = self._low_dimensional_affinities(result)
+            result.grad = self._gradient(early_exaggeration * symmetric_affinities, low_dimensional_affinities, result)
+            loss = torch.sum(symmetric_affinities * torch.log(symmetric_affinities / low_dimensional_affinities))
+            self.history.append(loss)
+            if verbose: print(f"Epoch: {epoch + 1} - KL-divergence loss: {loss:.3f}")
+            optimiser.update_parameters()
+
+        return result
+
+    def fit(self, X, epochs=100, verbose=False):
+        """
+        Wrapper for the TSNE.fit_transform(X) method.
+        
+        Args:
+            X (torch.Tensor of shape (n_samples, n_features)): The input data, where each row is a sample and each column is a feature.
+            epochs (int, optional): The number of training epochs after early exaggeration. Must be a positive integer. Defaults to 100.
+            verbose (bool, optional): Determines if the loss is printed each epoch. Must be a boolean. Defaults to ``False``.
+        """
+        return self.fit_transform(X, epochs, verbose)

DLL/MachineLearning/UnsupervisedLearning/DimensionalityReduction/_UMAP.py

@@ -0,0 +1,176 @@
+import torch
+from scipy.optimize import curve_fit
+from functools import partial
+
+from . import PCA
+from ....DeepLearning.Optimisers import SGD
+from ...SupervisedLearning.Kernels import RBF
+
+
+class UMAP:
+    """
+    Uniform Manifold Approximation and Projection (UMAP) class for dimensionality reduction. This implementation is based on `this paper <https://arxiv.org/pdf/1802.03426>`_ and `this article <https://github.com/NikolayOskolkov/HowUMAPWorks/blob/master/HowUMAPWorks.ipynb>`_.
+
+    Args:
+        n_components (int): Number of principal components to keep. The number must be a positive integer.
+        init (str, optional): The method for initializing the embedding. Must be in ``["spectral", "pca", "random"]``. Defaults to ``"spectral"``.
+        p (int, optional): The order of the chosen metric. Must be a positive integer. Defaults to 2, which corresponds to the Euclidian metric.
+        n_neighbor (int, optional): Controls how UMAP balances local and global structure in data. The larger this parameter is the better the global structure is conserved. A small value conserves fine details well, but may lose global structure. Must be a positive integer. Defaults to 15.
+        min_dist (float | int, optional): Controls the minimum distance between samples in the low dimensional space. Must be a non-negative real number. Defaults to 0.25.
+        learning_rate (float | int, optional): Determines how long steps do we take towards the gradient. Must be a positive real number. Defaults to 1.
+
+    Attributes:
+        history (list[float]): The history of the cross entropy loss function each epoch. Available after fitting the model.
+    """
+    def __init__(self, n_components=2, init="spectral", p=2, n_neighbor=15, min_dist=0.25, learning_rate=1):
+        if not isinstance(n_components, int) or n_components < 1:
+            raise ValueError("n_components must be a positive integer.")
+        if not init in ["spectral", "pca", "random"]:
+            raise ValueError('init must be in ["spectral", "pca", "random"].')
+        if not isinstance(p, int) or p <= 0:
+            raise ValueError("p must be a positive integer.")
+        if not isinstance(n_neighbor, int) or n_neighbor <= 0:
+            raise ValueError("n_neighbor must be a positive integer.")
+        if not isinstance(min_dist, int | float) or min_dist < 0:
+            raise ValueError("min_dist must be a non-negative real number.")
+        if (not isinstance(learning_rate, float | int) or learning_rate <= 0):
+            raise ValueError('learning_rate must be a positive real number.')
+
+        self.n_components = n_components
+        self.init = init
+        self.p = p
+        self.learning_rate = learning_rate
+        self.n_neighbor = n_neighbor
+        self.min_dist = min_dist
+        self.a, self.b = self._solve_params()
+
+    def _solve_params(self):
+        def func(d, a, b):
+            return 1 / (1 + a * (d + 1e-10) ** (2 * b))
+        
+        x = torch.linspace(0, 3, 300)
+        y = torch.full_like(x, 1)
+        mask = x > self.min_dist
+        y[mask] = torch.exp(-x[mask] + self.min_dist)
+        p, _ = curve_fit(func, x.numpy(), y.numpy())
+        return torch.tensor(p[0], dtype=torch.float32), torch.tensor(p[1], dtype=torch.float32)  # a, b
+
+    def _initialize(self, X):
+        if self.init == "pca":
+            return PCA(n_components=self.n_components).fit_transform(X)
+        elif self.init == "spectral":
+            W = RBF()(X, X)
+            D = torch.diag(W.sum(dim=1))
+            L = D - W
+            _, eigenvectors = torch.linalg.eigh(L)
+            embedding = eigenvectors[:, 1:self.n_components+1]
+            embedding /= torch.linalg.norm(embedding, dim=1, keepdim=True) + 1e-10
+            return embedding
+        else:
+            return torch.normal(0, 1e-4, size=(len(X), self.n_components))
+
+    def _pairwise_affinities(self, norm, rho, sigma, row):
+        d = norm[row] - rho[row]
+        d[d < 0] = 0
+        return torch.exp(- d / sigma)
+
+    def _k(self, prob):
+        return 2 ** torch.sum(prob)
+
+    def _sigma(self, k_of_sigma, fixed_k):
+        sigma_lower_limit = 0
+        sigma_upper_limit = 1000
+        for _ in range(20):
+            approx_sigma = (sigma_lower_limit + sigma_upper_limit) / 2
+            if k_of_sigma(approx_sigma) < fixed_k:
+                sigma_lower_limit = approx_sigma
+            else:
+                sigma_upper_limit = approx_sigma
+            if torch.abs(fixed_k - k_of_sigma(approx_sigma)) <= 1e-5:
+                break
+        return approx_sigma
+
+    def _local_fuzzy_simplicial_set(self, norm, rho):
+        n = len(norm)
+        prob = torch.zeros((n, n))
+        for dist_row in range(n):
+            _func = partial(self._k_of_sigma, norm, rho, dist_row)
+            prob[dist_row] = self._pairwise_affinities(norm, rho, self._sigma(_func, self.n_neighbor), dist_row)
+        return prob
+    
+    def _k_of_sigma(self, norm, rho, dist_row, sigma):
+        return self._k(self._pairwise_affinities(norm, rho, sigma, dist_row))
+
+    def _symmetric_affinities(self, X):
+        diff = X.unsqueeze(1) - X.unsqueeze(0)
+        norm = torch.norm(diff, dim=2, p=self.p) ** 2
+        rho = [sorted(norm[i])[1] for i in range(len(norm))]  # distance to nearest neighbor
+        affinities = self._local_fuzzy_simplicial_set(norm, rho)
+        return (affinities + affinities.T) / 2  # suggested by article
+        # return affinities + affinities.T - affinities * affinities.T  # suggested by paper
+
+    def _low_dimensional_affinities(self, low_dimensional_representation):
+        diff = low_dimensional_representation.unsqueeze(1) - low_dimensional_representation.unsqueeze(0)
+        norm = torch.norm(diff, dim=2, p=self.p)
+        inv_distances = 1 / (1 + self.a * norm ** (2 * self.b))
+        return inv_distances
+
+    def _CE(self, symmetric_affinities, low_dimensional_representation):
+        low_dimensional_affinities = self._low_dimensional_affinities(low_dimensional_representation)
+        return torch.mean(-symmetric_affinities * torch.log(low_dimensional_affinities + 1e-5) - (1 - symmetric_affinities) * torch.log(1 - low_dimensional_affinities + 1e-5))
+
+    def _CE_gradient(self, symmetric_affinities, low_dimensional_representation):
+        diff = low_dimensional_representation.unsqueeze(1) - low_dimensional_representation.unsqueeze(0)
+        norm = torch.norm(diff, dim=2, p=self.p) ** 2
+        inv_dist = 1 / (1 + self.a * norm ** self.b)
+        Q = (1 - symmetric_affinities) @ (1 / (norm + 1e-5))
+        Q.fill_diagonal_(0)
+        Q /= torch.sum(Q, dim=1, keepdims=True)
+        fact = (self.a * symmetric_affinities * (1e-8 + norm) ** (self.b - 1) - Q).unsqueeze(2)
+        return 2 * self.b * torch.sum(fact * diff * inv_dist.unsqueeze(2), dim=1)
+
+    def fit_transform(self, X, epochs=100, verbose=False):
+        """
+        Fits the UMAP model to the input data.
+
+        Args:
+            X (torch.Tensor of shape (n_samples, n_features)): The input data, where each row is a sample and each column is a feature.
+            epochs (int, optional): The number of training epochs. Must be a positive integer. Defaults to 100.
+            verbose (bool, optional): Determines if the loss is printed each epoch. Must be a boolean. Defaults to ``False``.
+        Returns:
+            embedding (torch.tensor): The embedded samples of shape (n_samples, n_components).
+        """
+        if not isinstance(X, torch.Tensor):
+            raise TypeError("The input matrix must be a PyTorch tensor.")
+        if X.ndim != 2 or X.shape[0] == 1:
+            raise ValueError("The input matrix must be a 2 dimensional tensor with atleast 2 samples.")
+        if not isinstance(epochs, int) or epochs <= 0:
+            raise ValueError("epochs must be a positive integer.")
+        if not isinstance(verbose, bool):
+            raise ValueError("verbose must be a boolean.")
+
+        result = self._initialize(X)
+        symmetric_affinities = self._symmetric_affinities(X)
+        optimiser = SGD(learning_rate=self.learning_rate, momentum=0.5)
+        optimiser.initialise_parameters([result])
+        self.history = []
+
+        for epoch in range(epochs):
+            result.grad = self._CE_gradient(symmetric_affinities, result)
+            optimiser.update_parameters()
+            loss = self._CE(symmetric_affinities, result)
+            self.history.append(loss)
+            if verbose: print(f"Epoch: {epoch + 1} - Cross-Entropy loss: {loss:.3f}")
+
+        return result
+
+    def fit(self, X, epochs=100, verbose=False):
+        """
+        Wrapper for the TSNE.fit_transform(X) method.
+        
+        Args:
+            X (torch.Tensor of shape (n_samples, n_features)): The input data, where each row is a sample and each column is a feature.
+            epochs (int, optional): The number of training epochs after early exaggeration. Must be a positive integer. Defaults to 100.
+            verbose (bool, optional): Determines if the loss is printed each epoch. Must be a boolean. Defaults to ``False``.
+        """
+        return self.fit_transform(X, epochs, verbose)

DLL/MachineLearning/UnsupervisedLearning/DimensionalityReduction/__init__.py

@@ -0,0 +1,6 @@
+from ._PCA import PCA
+from ._LDA import LDA
+from ._RobustPCA import RobustPCA
+from ._TSNE import TSNE
+from ._UMAP import UMAP
+from ._QDA import QDA