scratchkit 0.2.0__py3-none-any.whl

mlscratch/unsupervised/kmeans.py

@@ -0,0 +1,135 @@
+r"""
+K-Means Clustering
+===================
+
+A classic unsupervised clustering algorithm using Lloyd's iteration with
+K-Means++ initialization.
+
+The objective minimized is:
+
+.. math::
+    J = \sum_{i=1}^n \min_{1 \leq k \leq K} \|x_i - \mu_k\|^2
+
+Complexity
+----------
+- Training: O(n K d \cdot n\_iter)
+- Inference: O(n K d)
+- Space:    O(K d)
+"""
+
+from __future__ import annotations
+
+import numpy as np
+from numpy.typing import ArrayLike, NDArray
+
+FloatArray = NDArray[np.float64]
+
+
+def _validate_input(X: ArrayLike) -> FloatArray:
+    X_arr = np.asarray(X, dtype=float)
+    if X_arr.ndim != 2:
+        raise ValueError("X must be a 2D array of shape (n_samples, n_features).")
+    return X_arr
+
+
+class KMeans:
+    """K-Means clustering with optional K-Means++ initialization.
+
+    Parameters
+    ----------
+    n_clusters : int
+        The number of clusters to form.
+    max_iter : int, default=300
+        Maximum number of iterations of the k-means algorithm for a single run.
+    tol : float, default=1e-4
+        Convergence tolerance. The algorithm stops when centroid movement is
+        less than this threshold.
+    random_state : int | None, default=None
+        Seed for centroid initialization.
+    """
+
+    def __init__(
+        self,
+        n_clusters: int = 8,
+        max_iter: int = 300,
+        tol: float = 1e-4,
+        random_state: int | None = None,
+    ) -> None:
+        self.n_clusters = int(n_clusters)
+        self.max_iter = int(max_iter)
+        self.tol = float(tol)
+        self.random_state = random_state
+        self.cluster_centers_: FloatArray | None = None
+        self.labels_: NDArray[np.int64] | None = None
+        self.inertia_: float | None = None
+        self.n_iter_: int | None = None
+
+    def fit(self, X: ArrayLike) -> "KMeans":
+        X_arr = _validate_input(X)
+        n_samples, n_features = X_arr.shape
+        if self.n_clusters <= 0 or self.n_clusters > n_samples:
+            raise ValueError("n_clusters must be between 1 and n_samples.")
+
+        rng = np.random.default_rng(self.random_state)
+        centers = self._initialize_centroids(X_arr, rng)
+
+        for iteration in range(1, self.max_iter + 1):
+            labels = self._assign_clusters(X_arr, centers)
+            new_centers = self._compute_centers(X_arr, labels, n_features)
+
+            shift = np.linalg.norm(centers - new_centers, axis=1).max()
+            centers = new_centers
+            if shift <= self.tol:
+                break
+
+        self.cluster_centers_ = centers
+        self.labels_ = labels
+        self.inertia_ = float(self._compute_inertia(X_arr, centers, labels))
+        self.n_iter_ = iteration
+        return self
+
+    def predict(self, X: ArrayLike) -> NDArray[np.int64]:
+        if self.cluster_centers_ is None:
+            raise RuntimeError("Call fit() before predict().")
+        X_arr = _validate_input(X)
+        if X_arr.shape[1] != self.cluster_centers_.shape[1]:
+            raise ValueError("X has a different number of features than the training data.")
+        return self._assign_clusters(X_arr, self.cluster_centers_)
+
+    def _initialize_centroids(self, X: FloatArray, rng: np.random.Generator) -> FloatArray:
+        centers = np.empty((self.n_clusters, X.shape[1]), dtype=float)
+        first_idx = rng.integers(X.shape[0])
+        centers[0] = X[first_idx]
+
+        distances = np.full(X.shape[0], np.inf, dtype=float)
+        for i in range(1, self.n_clusters):
+            squared_distances = np.sum((X - centers[i - 1]) ** 2, axis=1)
+            distances = np.minimum(distances, squared_distances)
+            probabilities = distances / distances.sum()
+            cumulative = np.cumsum(probabilities)
+            chosen = rng.random()
+            centers[i] = X[np.searchsorted(cumulative, chosen)]
+
+        return centers
+
+    def _assign_clusters(self, X: FloatArray, centers: FloatArray) -> NDArray[np.int64]:
+        distances = np.linalg.norm(X[:, np.newaxis, :] - centers[np.newaxis, :, :], axis=2)
+        return np.argmin(distances, axis=1).astype(np.int64)
+
+    def _compute_centers(
+        self, X: FloatArray, labels: NDArray[np.int64], n_features: int
+    ) -> FloatArray:
+        centers = np.zeros((self.n_clusters, n_features), dtype=float)
+        for cluster_index in range(self.n_clusters):
+            members = X[labels == cluster_index]
+            if members.size == 0:
+                centers[cluster_index] = X[np.random.default_rng(self.random_state).integers(X.shape[0])]
+            else:
+                centers[cluster_index] = members.mean(axis=0)
+        return centers
+
+    def _compute_inertia(
+        self, X: FloatArray, centers: FloatArray, labels: NDArray[np.int64]
+    ) -> float:
+        diff = X - centers[labels]
+        return float(np.sum(diff ** 2))

mlscratch/unsupervised/kmedoids.py

@@ -0,0 +1,133 @@
+"""
+K-Medoids Clustering (PAM — Partitioning Around Medoids)
+=========================================================
+Similar to K-Means but the cluster representatives (medoids) must be actual
+data points, making the algorithm more robust to outliers and compatible
+with non-Euclidean distances.
+
+Algorithm (simplified PAM)
+--------------------------
+1. Randomly initialise K medoids from the dataset.
+2. Assign every point to its nearest medoid.
+3. For each cluster, try every non-medoid point as a new medoid;
+   keep the swap if it reduces the total cluster cost.
+4. Repeat steps 2-3 until no swap improves the cost.
+
+Only numpy and Python stdlib are used.
+"""
+
+import numpy as np
+
+
+class KMedoids:
+    """
+    K-Medoids clustering.
+
+    Parameters
+    ----------
+    n_clusters : int
+        Number of clusters / medoids.
+    max_iter : int
+        Maximum number of swap iterations.
+    random_state : int or None
+        Seed for reproducible medoid initialisation.
+    """
+
+    def __init__(
+        self,
+        n_clusters: int = 3,
+        max_iter: int = 100,
+        random_state: int | None = None,
+    ):
+        self.n_clusters = n_clusters
+        self.max_iter = max_iter
+        self.random_state = random_state
+        self.medoid_indices_ = None   # indices into X of the medoids
+        self.labels_ = None
+        self.inertia_ = None
+
+    # ------------------------------------------------------------------
+    # Helpers
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _pairwise_distances(X: np.ndarray) -> np.ndarray:
+        """Return Euclidean distance matrix of shape (n, n)."""
+        n = len(X)
+        D = np.zeros((n, n))
+        for i in range(n):
+            for j in range(i + 1, n):
+                d = np.sqrt(np.sum((X[i] - X[j]) ** 2))
+                D[i, j] = D[j, i] = d
+        return D
+
+    def _assign_labels(self, D: np.ndarray, medoids: list) -> np.ndarray:
+        """Assign each point to its nearest medoid."""
+        dist_to_medoids = D[:, medoids]          # (n, K)
+        return np.argmin(dist_to_medoids, axis=1)
+
+    def _total_cost(self, D: np.ndarray, medoids: list, labels: np.ndarray) -> float:
+        """Sum of distances from each point to its medoid."""
+        cost = 0.0
+        for k, m in enumerate(medoids):
+            members = np.where(labels == k)[0]
+            cost += D[members, m].sum()
+        return float(cost)
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def fit(self, X: np.ndarray) -> "KMedoids":
+        """
+        Fit K-Medoids to X.
+
+        Parameters
+        ----------
+        X : ndarray of shape (n_samples, n_features)
+        """
+        rng = np.random.default_rng(self.random_state)
+        n_samples = len(X)
+        D = self._pairwise_distances(X)
+
+        # 1. Initialise medoids
+        medoids = rng.choice(n_samples, self.n_clusters, replace=False).tolist()
+
+        for _ in range(self.max_iter):
+            labels = self._assign_labels(D, medoids)
+            current_cost = self._total_cost(D, medoids, labels)
+            improved = False
+
+            for k in range(self.n_clusters):
+                cluster_members = np.where(labels == k)[0].tolist()
+                for candidate in cluster_members:
+                    if candidate in medoids:
+                        continue
+                    new_medoids = medoids.copy()
+                    new_medoids[k] = candidate
+                    new_labels = self._assign_labels(D, new_medoids)
+                    new_cost = self._total_cost(D, new_medoids, new_labels)
+
+                    if new_cost < current_cost:
+                        medoids = new_medoids
+                        labels = new_labels
+                        current_cost = new_cost
+                        improved = True
+
+            if not improved:
+                break
+
+        self.medoid_indices_ = np.array(medoids, dtype=int)
+        self.labels_ = self._assign_labels(D, medoids)
+        self.inertia_ = self._total_cost(D, medoids, self.labels_)
+        return self
+
+    def fit_predict(self, X: np.ndarray) -> np.ndarray:
+        """Fit and return cluster labels."""
+        self.fit(X)
+        return self.labels_
+
+    @property
+    def cluster_centers_(self) -> np.ndarray:
+        """Return the actual medoid data points."""
+        return None  # set by fit via fit_predict path; use medoid_indices_

mlscratch/unsupervised/pca.py

@@ -0,0 +1,103 @@
+"""
+Principal Component Analysis (PCA)
+====================================
+Linear dimensionality reduction via eigen-decomposition of the covariance
+matrix.  Projects data onto the directions of maximum variance.
+
+Key steps
+---------
+1. Centre the data (subtract column means).
+2. Compute the covariance matrix  C = X^T X / (n - 1).
+3. Eigendecompose C to get eigenvalues and eigenvectors.
+4. Sort eigenvectors by descending eigenvalue.
+5. Project: X_reduced = X_centered @ W,  where W holds the top-k eigenvectors.
+
+Only numpy is used; no scipy or sklearn.
+"""
+
+import numpy as np
+
+
+class PCA:
+    """
+    Principal Component Analysis.
+
+    Parameters
+    ----------
+    n_components : int or None
+        Number of components to keep.  If None, all components are kept.
+    """
+
+    def __init__(self, n_components: int | None = None):
+        self.n_components = n_components
+        self.components_ = None        # shape (n_components, n_features)
+        self.explained_variance_ = None
+        self.explained_variance_ratio_ = None
+        self.mean_ = None
+
+    # ------------------------------------------------------------------
+    # Fit
+    # ------------------------------------------------------------------
+
+    def fit(self, X: np.ndarray) -> "PCA":
+        """
+        Compute principal components from X.
+
+        Parameters
+        ----------
+        X : ndarray of shape (n_samples, n_features)
+        """
+        n_samples, n_features = X.shape
+
+        # 1. Centre
+        self.mean_ = np.mean(X, axis=0)
+        X_centered = X - self.mean_
+
+        # 2. Covariance matrix (unbiased, divide by n-1)
+        cov = np.dot(X_centered.T, X_centered) / (n_samples - 1)
+
+        # 3. Eigen-decomposition
+        eigenvalues, eigenvectors = np.linalg.eigh(cov)
+
+        # 4. Sort descending by eigenvalue
+        order = np.argsort(eigenvalues)[::-1]
+        eigenvalues = eigenvalues[order]
+        eigenvectors = eigenvectors[:, order]   # columns are eigenvectors
+
+        # 5. Keep top-k
+        k = self.n_components if self.n_components is not None else n_features
+        self.components_ = eigenvectors[:, :k].T          # (k, n_features)
+        self.explained_variance_ = eigenvalues[:k]
+        total_var = np.sum(eigenvalues)
+        self.explained_variance_ratio_ = (
+            self.explained_variance_ / total_var if total_var > 0
+            else np.zeros(k)
+        )
+        return self
+
+    # ------------------------------------------------------------------
+    # Transform / inverse_transform
+    # ------------------------------------------------------------------
+
+    def transform(self, X: np.ndarray) -> np.ndarray:
+        """
+        Project X onto the principal components.
+
+        Returns
+        -------
+        X_new : ndarray of shape (n_samples, n_components)
+        """
+        X_centered = X - self.mean_
+        return np.dot(X_centered, self.components_.T)
+
+    def fit_transform(self, X: np.ndarray) -> np.ndarray:
+        """Fit and immediately transform X."""
+        self.fit(X)
+        return self.transform(X)
+
+    def inverse_transform(self, X_reduced: np.ndarray) -> np.ndarray:
+        """
+        Map data from reduced space back to original feature space
+        (approximate reconstruction).
+        """
+        return np.dot(X_reduced, self.components_) + self.mean_

mlscratch/unsupervised/tsne.py

@@ -0,0 +1,200 @@
+"""
+t-Distributed Stochastic Neighbour Embedding (t-SNE)
+=====================================================
+Non-linear dimensionality reduction that preserves local structure.
+Converts high-dimensional Euclidean distances into conditional
+probabilities (Gaussian in high-dim, Student-t in low-dim) and minimises
+the KL divergence between the two distributions via gradient descent.
+
+Key steps
+---------
+1. Compute pairwise affinities p_{j|i} in the high-dimensional space using
+   a Gaussian kernel; perplexity controls the effective number of neighbours.
+2. Symmetrise: p_{ij} = (p_{j|i} + p_{i|j}) / 2n.
+3. Initialise low-dimensional embedding Y randomly.
+4. Compute q_{ij} in Y using a Student-t kernel (df=1).
+5. Gradient descent on KL(P || Q) with momentum.
+
+Reference: van der Maaten & Hinton (2008).
+Only numpy is used.
+"""
+
+import numpy as np
+
+
+class TSNE:
+    """
+    t-SNE dimensionality reduction.
+
+    Parameters
+    ----------
+    n_components : int
+        Dimension of the embedding (almost always 2 or 3).
+    perplexity : float
+        Effective number of neighbours; typical values 5–50.
+    n_iter : int
+        Number of gradient-descent iterations.
+    learning_rate : float
+        Step size for gradient descent.
+    momentum : float
+        Momentum coefficient for gradient updates.
+    random_state : int or None
+        Seed for reproducibility.
+    """
+
+    def __init__(
+        self,
+        n_components: int = 2,
+        perplexity: float = 30.0,
+        n_iter: int = 1000,
+        learning_rate: float = 200.0,
+        momentum: float = 0.9,
+        random_state: int | None = None,
+    ):
+        self.n_components = n_components
+        self.perplexity = perplexity
+        self.n_iter = n_iter
+        self.learning_rate = learning_rate
+        self.momentum = momentum
+        self.random_state = random_state
+        self.embedding_ = None
+
+    # ------------------------------------------------------------------
+    # High-dimensional affinities
+    # ------------------------------------------------------------------
+
+    def _pairwise_sq_distances(self, X: np.ndarray) -> np.ndarray:
+        """Return matrix of squared Euclidean distances."""
+        sum_sq = np.sum(X ** 2, axis=1, keepdims=True)
+        D_sq = sum_sq + sum_sq.T - 2.0 * (X @ X.T)
+        np.fill_diagonal(D_sq, 0.0)
+        return np.maximum(D_sq, 0.0)
+
+    def _conditional_probabilities(
+        self, D_sq: np.ndarray, sigma: float, i: int
+    ) -> np.ndarray:
+        """Compute p_{j|i} for a given bandwidth sigma."""
+        d = D_sq[i].copy()
+        d[i] = np.inf                  # exclude self
+        exp_d = np.exp(-d / (2.0 * sigma ** 2))
+        denom = exp_d.sum()
+        return exp_d / (denom + 1e-12)
+
+    def _binary_search_sigma(
+        self, D_sq: np.ndarray, i: int, target_perp: float,
+        tol: float = 1e-5, max_iter: int = 50
+    ) -> float:
+        """Find sigma_i such that perplexity(p_{.|i}) == target_perp."""
+        sigma_low, sigma_high = 1e-10, 1e5
+        sigma = 1.0
+
+        for _ in range(max_iter):
+            p = self._conditional_probabilities(D_sq, sigma, i)
+            # Shannon entropy
+            p_safe = np.maximum(p, 1e-12)
+            H = -np.sum(p_safe * np.log2(p_safe))
+            perp = 2.0 ** H
+
+            if abs(perp - target_perp) < tol:
+                break
+            if perp < target_perp:
+                sigma_low = sigma
+                sigma = (sigma + sigma_high) / 2.0
+            else:
+                sigma_high = sigma
+                sigma = (sigma + sigma_low) / 2.0
+
+        return sigma
+
+    def _compute_P(self, X: np.ndarray) -> np.ndarray:
+        """Compute symmetric joint probabilities P."""
+        n = len(X)
+        D_sq = self._pairwise_sq_distances(X)
+        P = np.zeros((n, n))
+
+        for i in range(n):
+            sigma = self._binary_search_sigma(D_sq, i, self.perplexity)
+            P[i] = self._conditional_probabilities(D_sq, sigma, i)
+
+        # Symmetrise and normalise
+        P = (P + P.T) / (2.0 * n)
+        P = np.maximum(P, 1e-12)
+        return P
+
+    # ------------------------------------------------------------------
+    # Low-dimensional affinities
+    # ------------------------------------------------------------------
+
+    def _compute_Q(self, Y: np.ndarray) -> tuple:
+        """
+        Compute Student-t affinities in the embedding.
+
+        Returns
+        -------
+        Q : normalised affinities
+        num : unnormalised numerator (needed for gradient)
+        """
+        D_sq = self._pairwise_sq_distances(Y)
+        num = 1.0 / (1.0 + D_sq)
+        np.fill_diagonal(num, 0.0)
+        denom = num.sum()
+        Q = num / (denom + 1e-12)
+        Q = np.maximum(Q, 1e-12)
+        return Q, num
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def fit_transform(self, X: np.ndarray) -> np.ndarray:
+        """
+        Fit t-SNE and return 2-D (or n_components-D) embedding.
+
+        Parameters
+        ----------
+        X : ndarray of shape (n_samples, n_features)
+
+        Returns
+        -------
+        Y : ndarray of shape (n_samples, n_components)
+        """
+        rng = np.random.default_rng(self.random_state)
+        n = len(X)
+
+        # Step 1: compute high-dim affinities
+        P = self._compute_P(X)
+        # Early exaggeration (first 250 iters)
+        P_exag = P * 4.0
+
+        # Step 2: random initialisation of embedding
+        Y = rng.standard_normal((n, self.n_components)) * 1e-4
+        velocity = np.zeros_like(Y)
+
+        for t in range(self.n_iter):
+            p_use = P_exag if t < 250 else P
+            Q, num = self._compute_Q(Y)
+
+            # Gradient of KL divergence
+            PQ_diff = p_use - Q           # (n, n)
+            grad = np.zeros_like(Y)
+            for i in range(n):
+                # dC/dY_i = 4 * sum_j (p_ij - q_ij) * (y_i - y_j) * (1 + ||y_i-y_j||^2)^-1
+                diff = Y[i] - Y           # (n, n_components)
+                grad[i] = 4.0 * (PQ_diff[i] * num[i] @ diff.reshape(n, -1)).sum(axis=0) \
+                    if self.n_components == 1 \
+                    else 4.0 * np.dot(PQ_diff[i] * num[i], diff)
+
+            # Momentum update
+            velocity = self.momentum * velocity - self.learning_rate * grad
+            Y = Y + velocity
+
+            # Centre embedding
+            Y -= Y.mean(axis=0)
+
+        self.embedding_ = Y
+        return Y
+
+    def fit(self, X: np.ndarray) -> "TSNE":
+        """Fit t-SNE (embedding stored in self.embedding_)."""
+        self.fit_transform(X)
+        return self