PyDLLib 1.0__py3-none-any.whl

DLL/MachineLearning/UnsupervisedLearning/Clustering/_DBScan.py

@@ -0,0 +1,87 @@
+import torch
+from scipy.spatial import KDTree
+
+from ....Exceptions import NotFittedError
+
+
+class DBScan:
+    """
+    Density-based spatial clustering of applications with noise (DBSCAN) algorithm.
+
+    Args:
+        eps (float | int, optional): The distance inside of which datapoints are considered to be neighbours. Must be a positive real number. Defaults to 0.5.
+        min_samples (int, optional): The minimum number of neighbours for a non-leaf node. Must be a positive integer. Defaults to 5.
+    
+    Note:
+        The algorithm is very sensitive to changes in `eps`. One should fine-tune the value of eps for optimal results.
+    """
+    def __init__(self, eps=0.5, min_samples=5):
+        if not isinstance(eps, int | float) or eps <= 0:
+            raise ValueError("eps must be a positive real number.")
+        if not isinstance(min_samples, int) or min_samples < 1:
+            raise ValueError("min_samples must be a positive integer.")
+        
+        self.eps = eps
+        self.min_samples = min_samples
+
+    def fit(self, X):
+        """
+        Fits the algorithm to the given data. Recursively finds clusters by connecting near-by samples into the same cluster.
+
+        Args:
+            X (torch.Tensor of shape (n_samples, n_features)): The input data, where each row is a sample and each column is a feature.
+        """
+        if not isinstance(X, torch.Tensor):
+            raise TypeError("The input matrix must be a PyTorch tensor.")
+        if X.ndim != 2:
+            raise ValueError("The input matrix must be a 2 dimensional tensor.")
+        
+        X = X.numpy()
+        tree = KDTree(X)
+        n = len(X)
+        labels = torch.zeros(n)
+        current_class = 0
+
+        # loop over samples to find roots for clusters.
+        for i in range(n):
+            if labels[i] != 0:
+                continue
+            
+            neighbour_indicies = tree.query_ball_point(X[i], r=self.eps)
+            
+            if len(neighbour_indicies) < self.min_samples:
+                labels[i] = -1
+            else:
+                current_class += 1
+                labels[i] = current_class
+                self._grow_cluster(X, tree, labels, neighbour_indicies, current_class)
+        labels[labels != -1] -= 1  # transform the classes to start from 0.
+        self._labels = labels
+
+    def _grow_cluster(self, X, tree, labels, neighbour_indicies, current_class):
+        j = 0
+        while j < len(neighbour_indicies):
+            neighbour_index = neighbour_indicies[j]
+            
+            if labels[neighbour_index] == -1:
+                labels[neighbour_index] = current_class
+            elif labels[neighbour_index] == 0:
+                labels[neighbour_index] = current_class
+                new_neighbours = tree.query_ball_point(X[neighbour_index], r=self.eps)
+                if len(new_neighbours) >= self.min_samples:
+                    neighbour_indicies.extend(new_neighbours)
+            j += 1
+
+    def predict(self):
+        """
+        Applies the fitted DBScan model to the input data. Splits the training data into clusters.
+
+        Returns:
+            labels (torch.Tensor of shape (n_samples,)): The cluster corresponding to each sample. Label -1 indicates, that the algorithm considers that spesific samples as noise.
+        Raises:
+            NotFittedError: If the DBScan model has not been fitted before predicting.
+        """
+        if not hasattr(self, "_labels"):
+            raise NotFittedError("DBScan.fit() must be called before predicting.")
+        
+        return self._labels

DLL/MachineLearning/UnsupervisedLearning/Clustering/_GaussianMixtureModels.py

@@ -0,0 +1,121 @@
+import torch
+
+from ....Exceptions import NotFittedError
+
+
+class GaussianMixture:
+    """
+    Gaussian mixture model. Fits `k` Gaussian distributions onto the data using maximum likelihood estimation.
+
+    Args:
+        k (int, optional): The number of Gaussian distributions (clusters). Must be a positive integer. Defaults to 3.
+        max_iters (int, optional): The maximum number of iterations. Must be a positive integer. Defaults to 10.
+    """
+    def __init__(self, k=3, max_iters=10, tol=1e-5):
+        if not isinstance(k, int) or k <= 0:
+            raise ValueError("k must be a positive integer.")
+        if not isinstance(max_iters, int) or max_iters <= 0:
+            raise ValueError("max_iters must be a positive integer.")
+
+        self.k = k
+        self.max_iters = max_iters
+        self.tol = tol
+
+    def _pdf(self, X):
+        # X (n_samples, n_features)
+        X_minus_mu = X[:, None, :] - self.mus
+        exponent = torch.einsum('nkd,kde,nke->nk', X_minus_mu, torch.linalg.inv(self.sigmas), X_minus_mu)
+        const = (2 * torch.pi) ** (-0.5 * self.n_features) * torch.linalg.det(self.sigmas) ** (-0.5)
+        return const * torch.exp(-0.5 * exponent)
+
+    def _expectation_step(self, X):
+        # Compute the posterior given prior, mean and covariance.
+        likelihood = self._pdf(X)
+        posterior = likelihood * self.prior[None, :]
+        norm_const = torch.sum(posterior, axis=1)
+        normalised_posterior = posterior / norm_const[:, None]
+        return normalised_posterior
+
+    def _maximization_step(self, X, posterior):
+        posterior_sum = torch.sum(posterior, axis=0)
+        new_prior = posterior_sum / len(X)
+
+        posterior_mus = (X.T @ posterior / posterior_sum).T
+
+        X_centered = X[:, None, :] - posterior_mus
+        X_weighted = X_centered * posterior[:, :, None]
+        posterior_sigmas = torch.einsum('nkd,nke->kde', X_weighted, X_centered) / posterior_sum[:, None, None]
+        return posterior_mus, posterior_sigmas, new_prior
+    
+    def _log_likelihood(self, X):
+        return torch.log(torch.sum(self._pdf(X) * self.prior))
+
+    def fit(self, X, verbose=False):
+        """
+        Fits the k gaussian distributions to the data using maximum likelihood estimation.
+
+        Args:
+            X (torch.Tensor of shape (n_samples, n_features)): The input data to be clustered.
+            verbose (bool, optional): Determines if the likelihood should be calculated and printed during training. Must be a boolean. Defaults to False.
+        """
+        if not isinstance(X, torch.Tensor) or X.ndim != 2:
+            raise TypeError("X must be a 2 dimensional torch tensor.")
+        if not isinstance(verbose, bool):
+            raise TypeError("verbose must be a boolean.")
+
+        self.n_features = X.shape[1]
+        # self.mus = torch.randn(self.k, n_features, dtype=X.dtype, device=X.device)
+        random_indices = torch.randperm(len(X))[:self.k]  # Random permutation of indices, pick first k
+        self.mus = X[random_indices].clone()
+        self.sigmas = torch.stack([torch.eye(self.n_features, dtype=X.dtype, device=X.device) for _ in range(self.k)])
+        self.prior = torch.ones(self.k, dtype=X.dtype, device=X.device)/self.k
+        if verbose: print(f"Initial log-likelihood: {self._log_likelihood(X)}")
+
+        for i in range(self.max_iters):
+            posterior = self._expectation_step(X)
+            mus, sigmas, prior = self._maximization_step(X, posterior)
+            if torch.all(mus - self.mus < self.tol) and torch.all(sigmas - self.sigmas < self.tol):
+                break
+            self.taus, self.mus, self.sigmas = prior, mus, sigmas
+            if verbose: print(f"Epoch: {i + 1} - Log-likelihood: {self._log_likelihood(X)}")
+
+    def predict(self, X):
+        """
+        Predicts the clusters of the data according to the fitted distributions.
+
+        Args:
+            X (torch.Tensor of shape (n_samples, n_features)): The input data to be clustered.
+
+        Returns:
+            torch.Tensor of shape (n_samples,): A tensor of labels corresponding to classes.
+        """
+        if not isinstance(X, torch.Tensor) or X.ndim != 2:
+            raise TypeError("X must be a 2 dimensional torch tensor.")
+        if X.shape[1] != self.n_features:
+            raise ValueError("X must have the same number of features as the training data.")
+        if not hasattr(self, "mus"):
+            raise NotFittedError("GaussianMixture.fit() must be fitted before predicting.")
+
+
+        posterior = self._expectation_step(X)
+        return torch.argmax(posterior, dim=1)
+
+    def predict_proba(self, X):
+        """
+        Predicts the probabilities of the data being in the fitted distributions.
+
+        Args:
+            X (torch.Tensor of shape (n_samples, n_features)): The input data to be clustered.
+
+        Returns:
+            torch.Tensor of shape (n_samples, k): A tensor of probabilities of the data being in the fitted distributions.
+        """
+        if not isinstance(X, torch.Tensor) or X.ndim != 2:
+            raise TypeError("X must be a 2 dimensional torch tensor.")
+        if X.shape[1] != self.n_features:
+            raise ValueError("X must have the same number of features as the training data.")
+        if not hasattr(self, "mus"):
+            raise NotFittedError("GaussianMixture.fit() must be fitted before predicting.")
+
+        posterior = self._expectation_step(X)
+        return posterior

DLL/MachineLearning/UnsupervisedLearning/Clustering/_KMeans.py

@@ -0,0 +1,120 @@
+import torch
+
+from ....Exceptions import NotFittedError
+
+
+class KMeansClustering:
+    """
+    KMeansClustering implements the K-Means clustering algorithm, which partitions data points into `k` clusters.
+
+    Args:
+        k (int, optional): The number of clusters. Defaults to 3. Must be a positive integer.
+        max_iters (int, optional): The maximum number of iterations for training the model. Defaults to 100. Must be a positive integer.
+        init ({`kmeans++`, `random`}, optional): The method for initialising the centroids. Defaults to `kmeans++`.
+        n_init (int, optional): The number of differently initialized centroids. Defaults to 10. Must be a positive integer.
+    Attributes:
+        centroids (torch.Tensor): The final chosen centroids.
+        inertia (float): The total squared distance to the nearest centroid.
+    """
+    def __init__(self, k=3, max_iters=100, init="kmeans++", n_init=10, tol=1e-5):
+        if not isinstance(k, int) or k < 1:
+            raise ValueError("k must be a positive integer.")
+        if not isinstance(max_iters, int) or max_iters < 1:
+            raise ValueError("max_iters must be a positive integer.")
+        if not isinstance(n_init, int) or n_init < 1:
+            raise ValueError("n_init must be a positive integer.")
+        if init not in ["random", "kmeans++"]:
+            raise ValueError('init must be one of "random" or "kmeans++".')
+        self.k = k
+        self.max_iters = max_iters
+        self.init = init
+        self.n_init = n_init
+        self.tol = tol
+
+    def fit(self, X):
+        """
+        Fits the KMeansClustering model to the input data by finding the best centroids.
+
+        Args:
+            X (torch.Tensor of shape (n_samples, n_features)): The input data, where each row is a sample and each column is a feature. The number of samples must be atleast `k`.
+        Returns:
+            None
+        Raises:
+            TypeError: If the input matrix is not a PyTorch tensor.
+            ValueError: If the input matrix is not the correct shape.
+        """
+        if not isinstance(X, torch.Tensor):
+            raise TypeError("The input matrix must be a PyTorch tensor.")
+        if X.ndim != 2 or X.shape[0] < self.k:
+            raise ValueError("The input matrix must be a 2 dimensional tensor with atleast k samples.")
+        centroids = self._initialize_centroids(X) # (n_init, k, m)
+        best_centroid_index = -1
+        lowest_inertia = float("inf")
+        for i in range(self.n_init):
+            for _ in range(self.max_iters):
+                indicies = self._cluster(centroids[i], X)
+                old_centroids = centroids[i].clone()
+                centroids[i] = self._get_centroids(indicies, X)
+                if torch.norm(centroids[i] - old_centroids) < self.tol:
+                    break
+            # select the centroids with the lowest inertia (total squared error)
+            inertia = ((X - centroids[i][indicies]) ** 2).sum().item()
+            if inertia < lowest_inertia:
+                lowest_inertia = inertia
+                best_centroid_index = i
+            else:
+                best_centroid_index = best_centroid_index
+        self.inertia = lowest_inertia
+        self.centroids = centroids[best_centroid_index]
+
+    def predict(self, X):
+        """
+        Applies the fitted KMeansClustering model to the input data, partitioning it to `k` clusters.
+
+        Args:
+            X (torch.Tensor of shape (n_samples, n_features)): The input data to be partitioned.
+        Returns:
+            labels (torch.Tensor of shape (n_samples,)): The cluster corresponding to each sample.
+        Raises:
+            NotFittedError: If the KMeansClustering model has not been fitted before predicting.
+            TypeError: If the input matrix is not a PyTorch tensor.
+            ValueError: If the input matrix is not the correct shape.
+        """
+        if not hasattr(self, "centroids"):
+            raise NotFittedError("KMeansClustering.fit() must be called before predicting.")
+        if not isinstance(X, torch.Tensor):
+            raise TypeError("The input matrix must be a PyTorch tensor.")
+        if X.ndim != 2 or X.shape[1] != self.centroids.shape[1]:
+            raise ValueError("The input matrix must be a 2 dimensional tensor with the same number of features as the fitted tensor.")
+        return self._cluster(self.centroids, X)
+    
+    def _initialize_centroids(self, X):
+        if self.init == "random":
+            initial_centroid_indicies = torch.randint(0, len(X), (self.n_init, self.k))
+            centroids = torch.stack([X[initial_centroid_indicies[i]] for i in range(self.n_init)])
+        elif self.init == "kmeans++":
+            # https://en.wikipedia.org/wiki/K-means%2B%2B
+            centroids = torch.zeros((self.n_init, self.k, X.shape[1]), dtype=X.dtype)
+            centroids[:, 0] = X[torch.randint(0, len(X), (self.n_init,))]
+            for i in range(self.n_init):
+                for k in range(1, self.k):
+                    # Calculate distance from points to the clostest of the chosen centroids
+                    chosen_centroids = centroids[i, :k]
+                    distances = torch.cdist(chosen_centroids, X, p=2).min(dim=0).values ** 2
+                    probabilities = distances / distances.sum()
+                    # Choose remaining points based on their distances
+                    new_centroid_index = torch.multinomial(probabilities, num_samples=1)
+                    centroids[i, k] = X[new_centroid_index]
+        return centroids
+
+    def _cluster(self, centroids, X):
+        distances = ((centroids.unsqueeze(0) - X.unsqueeze(1)) ** 2).sum(dim=2).sqrt() # (n, k)
+        indicies = torch.argmin(distances, dim=1)
+        return indicies
+
+    def _get_centroids(self, indicies, X):
+        new_centroids = torch.zeros((self.k, X.shape[1]))
+        for k in range(self.k):
+            X_cls = X[indicies == k]
+            new_centroids[k] = X_cls.mean(dim=0)
+        return new_centroids

DLL/MachineLearning/UnsupervisedLearning/Clustering/_SpectralClustering.py

@@ -0,0 +1,105 @@
+import torch
+
+from ...SupervisedLearning.Kernels import RBF, _Base
+from . import KMeansClustering
+from ....Exceptions import NotFittedError
+
+
+class SpectralClustering:
+    """
+    SpectralClustering implements the spectral clustering algorithm, which partitions data points into `k` clusters.
+
+    Args:
+        kernel (:ref:`kernel_section_label`, optional): The similarity function for fitting the model. Defaults to RBF(correlation_length=0.1).
+        k (int, optional): The number of clusters. Defaults to 3. Must be a positive integer.
+        max_iters (int, optional): The maximum number of iterations for training the model. Defaults to 100. Must be a positive integer.
+        normalise (bool, optional): Determines if the laplacian matrix is calculated using L = I - sqrt(inv(D)) A sqrt(inv(D)) or just L = D - A. Defaults to True.
+        use_kmeans (bool, optional): Determines if the clustring in embedded space is done using kmeans or discretisation. Defaults to True.
+        **kwargs: Other arguments are passed into the `KMeansClustering` algorithm.
+
+    Note:
+        The result depends heavily on the chosen kernel function. Especially the correlation_length parameter should be fine-tuned for optimal performance.
+    """
+    def __init__(self, kernel=RBF(correlation_length=0.1), k=3, max_iters=100, normalise=True, use_kmeans=True, **kwargs):
+        if not isinstance(kernel, _Base):
+            raise ValueError("kernel must be from DLL.MachineLearning.SupervisedLearning.Kernels")
+        if not isinstance(k, int) or k < 1:
+            raise ValueError("k must be a positive integer.")
+        if not isinstance(max_iters, int) or max_iters < 1:
+            raise ValueError("max_iters must be a positive integer.")
+        if not isinstance(normalise, bool):
+            raise TypeError("normalise must be a boolean.")
+        if not isinstance(use_kmeans, bool):
+            raise TypeError("use_kmeans must be a boolean.")
+
+        self.kernel = kernel
+        self.k = k
+        self.max_iters = max_iters
+        self.normalise = normalise
+        self.use_kmeans = use_kmeans
+        if self.use_kmeans: self.kmeans = KMeansClustering(k=self.k, max_iters=self.max_iters, **kwargs)
+    
+    def _transform_data(self, X):
+        similarity_matrix = self.kernel(X, X)
+        if self.normalise:
+            D_sqrt_inv = torch.diag(1.0 / torch.sqrt(similarity_matrix.sum(axis=1)))
+            laplacian_matrix = torch.eye(X.shape[0]) - D_sqrt_inv @ similarity_matrix @ D_sqrt_inv
+        else:
+            degree_matrix = torch.diag(similarity_matrix.sum(axis=1))
+            laplacian_matrix = degree_matrix - similarity_matrix
+        
+        _, eigenvectors = torch.linalg.eigh(laplacian_matrix)
+        selected_eigenvectors = eigenvectors[:, :self.k]
+        return selected_eigenvectors
+    
+    def _discretize_spectral_embedding(self, selected_eigenvectors):
+        selected_eigenvectors = selected_eigenvectors / torch.linalg.norm(selected_eigenvectors, axis=1, keepdims=True)
+        n_samples = len(selected_eigenvectors)
+        H = torch.zeros((n_samples, self.k), dtype=selected_eigenvectors.dtype)
+        indices = torch.argmax(selected_eigenvectors, axis=1)
+        H[torch.arange(n_samples), indices] = 1
+
+        for _ in range(self.max_iters):
+            U, _, Vt = torch.linalg.svd(selected_eigenvectors.T @ H)
+            Q = U @ Vt
+            H_new = torch.argmax(selected_eigenvectors @ Q, axis=1)
+            H = torch.zeros((n_samples, self.k), dtype=selected_eigenvectors.dtype)
+            H[torch.arange(n_samples), H_new] = 1
+        
+        labels = torch.argmax(H, axis=1)
+        return labels
+    
+    def fit(self, X):
+        """
+        Fits the algorithm to the given data. Transforms the data into the embedding space using the kernel function and clusters the data in the space.
+
+        Args:
+            X (torch.Tensor of shape (n_samples, n_features)): The input data, where each row is a sample and each column is a feature. The number of samples must be atleast `k`.
+        """
+        if not isinstance(X, torch.Tensor):
+            raise TypeError("The input matrix must be a PyTorch tensor.")
+        if X.ndim != 2 or X.shape[0] < self.k:
+            raise ValueError("The input matrix must be a 2 dimensional tensor with atleast k samples.")
+        
+        self.selected_eigenvectors = self._transform_data(X)
+        if self.use_kmeans:
+            self.kmeans.fit(self.selected_eigenvectors)
+        else:
+            self._labels = self._discretize_spectral_embedding(self.selected_eigenvectors)
+    
+    def predict(self):
+        """
+        Applies the fitted SpectralClustering model to the input data, partitioning it to `k` clusters.
+
+        Returns:
+            labels (torch.Tensor of shape (n_samples,)): The cluster corresponding to each sample.
+        Raises:
+            NotFittedError: If the SpectralClustering model has not been fitted before predicting.
+        """
+        if not hasattr(self, "selected_eigenvectors"):
+            raise NotFittedError("SpectralClustering.fit() must be called before predicting.")
+        
+        if self.use_kmeans:
+            return self.kmeans.predict(self.selected_eigenvectors)
+        else:
+            return self._labels

DLL/MachineLearning/UnsupervisedLearning/Clustering/__init__.py

@@ -0,0 +1,4 @@
+from ._KMeans import KMeansClustering
+from ._GaussianMixtureModels import GaussianMixture
+from ._SpectralClustering import SpectralClustering
+from ._DBScan import DBScan

DLL/MachineLearning/UnsupervisedLearning/DimensionalityReduction/_LDA.py

@@ -0,0 +1,118 @@
+import torch
+
+from ....Exceptions import NotFittedError
+
+
+class LDA:
+    """
+    Linear discriminant analysis (LDA) class for dimensionality reduction.
+
+    Args:
+        n_components (int): Number of principal components to keep. The number must be a positive integer.
+    Attributes:
+        components (torch.Tensor): Components extracted from the data.
+        n_features (int): The number of features in the input.
+    """
+    def __init__(self, n_components=2):
+        if not isinstance(n_components, int) or n_components < 1:
+            raise ValueError("n_components must be a positive integer.")
+        
+        self.n_components = n_components
+    
+    def fit(self, X, y):
+        """
+        Fits the LDA model to the input data by calculating the components.
+        
+        Args:
+            X (torch.Tensor of shape (n_samples, n_features)): The input data, where each row is a sample and each column is a feature.
+        Raises:
+            TypeError: If the input matrix is not a PyTorch tensor.
+            ValueError: If the input matrix is not the correct shape.
+        """
+        if not isinstance(X, torch.Tensor):
+            raise TypeError("The input matrix must be a PyTorch tensor.")
+        if X.ndim != 2 or X.shape[0] == 1:
+            raise ValueError("The input matrix must be a 2 dimensional tensor with atleast 2 samples.")
+        
+        self.classes = torch.unique(y)
+        self.n_features = X.shape[1]
+        X_mean = torch.mean(X, dim=0)
+
+        Sw = torch.zeros((self.n_features, self.n_features), dtype=X.dtype)
+        Sb = torch.zeros((self.n_features, self.n_features), dtype=X.dtype)
+
+        self.class_means = []
+
+        for current_class in self.classes:
+            X_c = X[y == current_class]
+            C_mean = torch.mean(X_c, dim=0)
+            self.class_means.append(C_mean)
+            Sw += torch.cov(X_c.T, correction=0) * len(X_c)
+            mean_diff = (C_mean - X_mean).reshape(-1, 1)
+            Sb += len(X_c) * (mean_diff @ mean_diff.T)
+
+        A = torch.linalg.lstsq(Sw, Sb).solution
+        eig_vals, eig_vecs = torch.linalg.eig(A)
+        indicies = torch.argsort(eig_vals.real, descending=True)
+        self.components = eig_vecs.real.T[indicies][:self.n_components].T
+
+    def transform(self, X):
+        """
+        Applies the fitted LDA model to the input data, transforming it into the reduced feature space.
+
+        Args:
+            X (torch.Tensor of shape (n_samples, n_features)): The input data to be transformed.
+        Returns:
+            X_new (torch.Tensor of shape (n_samples, n_components)): The data transformed into the component space.
+        Raises:
+            NotFittedError: If the LDA model has not been fitted before transforming.
+            TypeError: If the input matrix is not a PyTorch tensor.
+            ValueError: If the input matrix is not the correct shape.
+        """
+        if not hasattr(self, "components"):
+            raise NotFittedError("LDA.fit() must be called before transforming.")
+        if not isinstance(X, torch.Tensor):
+            raise TypeError("The input matrix must be a PyTorch tensor.")
+        if X.ndim != 2 or X.shape[1] != self.n_features:
+            raise ValueError("The input matrix must be a 2 dimensional tensor with the same number of features as the fitted tensor.")
+
+        return X @ self.components
+    
+    def fit_transform(self, X, y):
+        """
+        First finds the components of X and then transforms X to fitted space.
+
+        Args:
+            X (torch.Tensor of shape (n_samples, n_features)): The input data to be transformed.
+        Returns:
+            X_new (torch.Tensor of shape (n_samples, n_components)): The data transformed into the component space.
+        """
+        self.fit(X, y)
+        return self.transform(X)
+
+    def predict(self, X):
+        """
+        Applies the fitted LDA model to the input data, predicting the correct classes.
+
+        Args:
+            X (torch.Tensor of shape (n_samples, n_features)): The input data to be transformed.
+        Returns:
+            y (torch.Tensor of shape (n_samples,)): The predicted labels.
+        Raises:
+            NotFittedError: If the LDA model has not been fitted before transforming.
+        """
+        if not hasattr(self, "components"):
+            raise NotFittedError("LDA.fit() must be called before predicting.")
+        
+        X_projected = self.transform(X)
+        class_means_projected = torch.stack([self.transform(C_mean.unsqueeze(0)) for C_mean in self.class_means])
+
+        min_dists = torch.full((len(X),), torch.inf, dtype=X.dtype)
+        y = torch.zeros_like(min_dists, dtype=X.dtype)
+        
+        for i, C_mean in enumerate(class_means_projected):
+            class_dists = torch.sum((X_projected - C_mean) ** 2, dim=1)
+            mask = class_dists < min_dists
+            y[mask] = self.classes[i]
+            min_dists[mask] = class_dists[mask]
+        return y

DLL/MachineLearning/UnsupervisedLearning/DimensionalityReduction/_PCA.py

@@ -0,0 +1,101 @@
+import torch
+
+from ....Exceptions import NotFittedError
+
+
+class PCA:
+    """
+    Principal Component Analysis (PCA) class for dimensionality reduction.
+
+    Args:
+        n_components (int): Number of principal components to keep. The number must be a positive integer.
+    Attributes:
+        components (torch.Tensor): Principal components extracted from the data.
+        explained_variance (torch.Tensor): Variance explained by the selected components.
+    """
+    def __init__(self, n_components=2, epsilon=1e-10):
+        if not isinstance(n_components, int) or n_components < 1:
+            raise ValueError("n_components must be a positive integer.")
+        self.n_components = n_components
+        self.epsilon = epsilon
+
+    def fit(self, X, normalize=True, from_covariance=False):
+        """
+        Fits the PCA model to the input data by calculating the principal components.
+        
+        The input data is always centered and if `normalize=True`, also normalized so that the standard deviation is 1 along each axis.
+
+        Args:
+            X (torch.Tensor of shape (n_samples, n_features) or (n_features, n_features)): The input data, where each row is a sample and each column is a feature if from_covariance is set to False and the covariance matrix otherwise.
+            normalize (bool, optional): Whether to normalize the data before computing the PCA. Defaults to True. Is ignored if from_covariance is True.
+            from_covariance (bool, optional): Determines if X is considered as the data matrix or the covariance matrix. Must be a boolean. Defaults to False.
+        Returns:
+            None
+        Raises:
+            TypeError: If the input matrix is not a PyTorch tensor or if the `normalize` parameter is not boolean.
+            ValueError: If the input matrix is not the correct shape.
+        """
+        if not isinstance(X, torch.Tensor):
+            raise TypeError("The input matrix must be a PyTorch tensor.")
+        if not isinstance(normalize, bool):
+            raise TypeError("The normalize parameter must be a boolean.")
+        if X.ndim != 2 or (X.shape[0] == 1 and not from_covariance) or (from_covariance and (X.shape[0] != X.shape[1])):
+            if from_covariance:
+                raise ValueError("The input matrix must be 2 dimensional, with the shape (n_features, n_features).")
+            raise ValueError("The input matrix must be a 2 dimensional tensor with atleast 2 samples.")
+
+        self.from_covariance = from_covariance
+        if not from_covariance:
+            self.normalize = normalize
+            self.mean = X.mean(dim=0)
+            X = X - self.mean
+            if self.normalize:
+                self.standard_deviation = X.std(dim=0, unbiased=True)
+                X = X / (self.standard_deviation + self.epsilon)
+            covariance = torch.cov(X.T)
+        else:
+            covariance = X
+        eig_vals, eig_vecs = torch.linalg.eig(covariance)
+        indicies = torch.argsort(eig_vals.real, descending=True)
+        self.components = eig_vecs.real.T[indicies][:self.n_components]
+        self.explained_variance = eig_vals.real[indicies][:self.n_components]
+    
+    def transform(self, X):
+        """
+        Applies the fitted PCA model to the input data, transforming it into the reduced feature space. If covariance matrix was used when fitting, the input is assumed to be normalized appropriately.
+
+        Args:
+            X (torch.Tensor of shape (n_samples, n_features)): The input data to be transformed.
+        Returns:
+            X_new (torch.Tensor of shape (n_samples, n_components)): The data transformed into the principal component space.
+        Raises:
+            NotFittedError: If the PCA model has not been fitted before transforming.
+            TypeError: If the input matrix is not a PyTorch tensor.
+            ValueError: If the input matrix is not the correct shape.
+        """
+        if not hasattr(self, "from_covariance"):
+            raise NotFittedError("PCA.fit() must be called before transforming.")
+        if not isinstance(X, torch.Tensor):
+            raise TypeError("The input matrix must be a PyTorch tensor.")
+        if X.ndim != 2 or X.shape[1] != self.components.shape[1]:
+            raise ValueError("The input matrix must be a 2 dimensional tensor with the same number of features as the fitted tensor.")
+
+        if not self.from_covariance:
+            X = (X - self.mean)
+            if self.normalize:
+                X = X / (self.standard_deviation + self.epsilon)
+        return X @ self.components.T
+    
+    def fit_transform(self, X, normalize=True):
+        """
+        First finds the principal components of X and then transforms X to fitted space.
+
+        Args:
+            X (torch.Tensor of shape (n_samples, n_features)): The input data to be transformed.
+            normalize (bool, optional): Whether to normalize the data before computing the PCA. Defaults to True.
+        Returns:
+            X_new (torch.Tensor of shape (n_samples, n_components)): The data transformed into the principal component space.
+        """
+        
+        self.fit(X, normalize=normalize)
+        return self.transform(X)