scratchkit 0.2.0__py3-none-any.whl

mlscratch/unsupervised/dbscan.py

@@ -0,0 +1,141 @@
+"""
+Density-Based Spatial Clustering of Applications with Noise (DBSCAN)
+=====================================================================
+Groups together points that are closely packed (high density regions)
+and marks points in low-density regions as outliers (noise).
+
+Key ideas
+---------
+- eps      : neighbourhood radius
+- min_samples : minimum number of points to form a dense region (core point)
+- Core point   : has >= min_samples neighbours within eps
+- Border point : within eps of a core point, but not a core point itself
+- Noise point  : neither core nor border
+
+Time complexity : O(n^2) with the naive distance matrix approach used here.
+Only numpy and basic Python stdlib are used.
+"""
+
+import numpy as np
+
+
+class DBSCAN:
+    """
+    DBSCAN clustering.
+
+    Parameters
+    ----------
+    eps : float
+        Maximum distance between two samples to be considered neighbours.
+    min_samples : int
+        Minimum number of samples in a neighbourhood for a point to be
+        labelled a core point (includes the point itself).
+    """
+
+    NOISE = -1
+    UNVISITED = 0
+
+    def __init__(self, eps: float = 0.5, min_samples: int = 5):
+        self.eps = eps
+        self.min_samples = min_samples
+        self.labels_ = None          # cluster label per sample (-1 = noise)
+        self.core_sample_indices_ = None
+
+    # ------------------------------------------------------------------
+    # Internal helpers
+    # ------------------------------------------------------------------
+
+    def _euclidean_distance(self, a: np.ndarray, b: np.ndarray) -> float:
+        """Euclidean distance between two 1-D vectors."""
+        return np.sqrt(np.sum((a - b) ** 2))
+
+    def _region_query(self, X: np.ndarray, point_idx: int) -> list:
+        """Return indices of all points within eps of X[point_idx]."""
+        neighbours = []
+        for idx in range(len(X)):
+            if self._euclidean_distance(X[point_idx], X[idx]) <= self.eps:
+                neighbours.append(idx)
+        return neighbours
+
+    def _expand_cluster(
+        self,
+        X: np.ndarray,
+        labels: np.ndarray,
+        point_idx: int,
+        neighbours: list,
+        cluster_id: int,
+    ) -> None:
+        """Grow a cluster starting from point_idx."""
+        labels[point_idx] = cluster_id
+        seed_set = list(neighbours)          # mutable working queue
+
+        i = 0
+        while i < len(seed_set):
+            current = seed_set[i]
+
+            # If this was previously labelled noise, reassign to cluster
+            if labels[current] == self.NOISE:
+                labels[current] = cluster_id
+
+            # If unvisited, visit it now
+            if labels[current] == self.UNVISITED:
+                labels[current] = cluster_id
+                current_neighbours = self._region_query(X, current)
+
+                # If it is itself a core point, add its neighbours to the queue
+                if len(current_neighbours) >= self.min_samples:
+                    seed_set += current_neighbours   # may add duplicates; OK
+
+            i += 1
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def fit(self, X: np.ndarray) -> "DBSCAN":
+        """
+        Fit DBSCAN on dataset X.
+
+        Parameters
+        ----------
+        X : ndarray of shape (n_samples, n_features)
+
+        Returns
+        -------
+        self
+        """
+        n_samples = len(X)
+        labels = np.full(n_samples, self.UNVISITED, dtype=int)
+        cluster_id = 0
+
+        for idx in range(n_samples):
+            if labels[idx] != self.UNVISITED:
+                continue  # already processed
+
+            neighbours = self._region_query(X, idx)
+
+            if len(neighbours) < self.min_samples:
+                labels[idx] = self.NOISE          # mark as noise for now
+            else:
+                cluster_id += 1
+                self._expand_cluster(X, labels, idx, neighbours, cluster_id)
+
+        self.labels_ = labels
+        self.core_sample_indices_ = np.array(
+            [i for i in range(n_samples)
+             if len(self._region_query(X, i)) >= self.min_samples],
+            dtype=int,
+        )
+        return self
+
+    def fit_predict(self, X: np.ndarray) -> np.ndarray:
+        """
+        Fit and return cluster labels.
+
+        Returns
+        -------
+        labels : ndarray of shape (n_samples,)
+            -1 for noise, integers >= 1 for clusters.
+        """
+        self.fit(X)
+        return self.labels_

mlscratch/unsupervised/gmm.py

@@ -0,0 +1,204 @@
+"""
+Gaussian Mixture Model (GMM) via Expectation-Maximization
+==========================================================
+Models data as a weighted mixture of K multivariate Gaussian distributions.
+Parameters (means, covariances, mixing weights) are found by iterating the
+E-step and M-step until the log-likelihood converges.
+
+E-step : compute responsibilities  r[i,k] = P(z=k | x_i)
+M-step : update pi_k, mu_k, Sigma_k using the responsibilities
+
+Only numpy is used.
+"""
+
+import numpy as np
+
+
+class GaussianMixtureModel:
+    """
+    Gaussian Mixture Model fitted by the EM algorithm.
+
+    Parameters
+    ----------
+    n_components : int
+        Number of mixture components (clusters).
+    max_iter : int
+        Maximum number of EM iterations.
+    tol : float
+        Convergence tolerance on the log-likelihood change.
+    reg_covar : float
+        Small value added to the diagonal of each covariance matrix for
+        numerical stability.
+    random_state : int or None
+        Seed for reproducible centroid initialisation.
+    """
+
+    def __init__(
+        self,
+        n_components: int = 3,
+        max_iter: int = 100,
+        tol: float = 1e-4,
+        reg_covar: float = 1e-6,
+        random_state: int | None = None,
+    ):
+        self.n_components = n_components
+        self.max_iter = max_iter
+        self.tol = tol
+        self.reg_covar = reg_covar
+        self.random_state = random_state
+
+        # Learned parameters
+        self.weights_ = None    # (K,)
+        self.means_ = None      # (K, n_features)
+        self.covariances_ = None  # (K, n_features, n_features)
+        self.converged_ = False
+        self.n_iter_ = 0
+        self.lower_bound_ = -np.inf
+
+    # ------------------------------------------------------------------
+    # Internal helpers
+    # ------------------------------------------------------------------
+
+    def _multivariate_gaussian(
+        self, X: np.ndarray, mean: np.ndarray, cov: np.ndarray
+    ) -> np.ndarray:
+        """
+        Evaluate the multivariate Gaussian PDF for each row of X.
+
+        Returns
+        -------
+        pdf : ndarray of shape (n_samples,)
+        """
+        n_features = X.shape[1]
+        diff = X - mean                          # (n, d)
+        try:
+            cov_inv = np.linalg.inv(cov)
+            cov_det = np.linalg.det(cov)
+        except np.linalg.LinAlgError:
+            cov_inv = np.linalg.pinv(cov)
+            cov_det = np.linalg.det(cov + np.eye(n_features) * self.reg_covar)
+
+        cov_det = max(cov_det, 1e-300)           # guard against log(0)
+        norm = 1.0 / (np.sqrt((2 * np.pi) ** n_features * cov_det))
+        exponent = -0.5 * np.einsum("ij,jk,ik->i", diff, cov_inv, diff)
+        return norm * np.exp(exponent)
+
+    def _e_step(self, X: np.ndarray) -> np.ndarray:
+        """
+        Compute responsibilities r[i, k] = P(z=k | x_i).
+
+        Returns
+        -------
+        r : ndarray of shape (n_samples, n_components)
+        """
+        n_samples = X.shape[0]
+        r = np.zeros((n_samples, self.n_components))
+
+        for k in range(self.n_components):
+            r[:, k] = self.weights_[k] * self._multivariate_gaussian(
+                X, self.means_[k], self.covariances_[k]
+            )
+
+        row_sums = r.sum(axis=1, keepdims=True)
+        row_sums = np.where(row_sums == 0, 1e-300, row_sums)
+        r /= row_sums
+        return r
+
+    def _m_step(self, X: np.ndarray, r: np.ndarray) -> None:
+        """Update parameters from responsibilities."""
+        n_samples, n_features = X.shape
+        N_k = r.sum(axis=0)            # effective number per component
+
+        self.weights_ = N_k / n_samples
+
+        for k in range(self.n_components):
+            if N_k[k] < 1e-8:
+                continue
+            self.means_[k] = (r[:, k] @ X) / N_k[k]
+
+            diff = X - self.means_[k]                        # (n, d)
+            weighted_diff = r[:, k:k+1] * diff              # (n, d)
+            self.covariances_[k] = (
+                weighted_diff.T @ diff / N_k[k]
+                + np.eye(n_features) * self.reg_covar
+            )
+
+    def _log_likelihood(self, X: np.ndarray) -> float:
+        """Compute the log-likelihood of the data under current parameters."""
+        n_samples = X.shape[0]
+        ll = 0.0
+        for i in range(n_samples):
+            point_ll = sum(
+                self.weights_[k]
+                * self._multivariate_gaussian(
+                    X[i:i+1], self.means_[k], self.covariances_[k]
+                )[0]
+                for k in range(self.n_components)
+            )
+            ll += np.log(max(point_ll, 1e-300))
+        return ll
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def fit(self, X: np.ndarray) -> "GaussianMixtureModel":
+        """
+        Fit GMM parameters to X via EM.
+
+        Parameters
+        ----------
+        X : ndarray of shape (n_samples, n_features)
+        """
+        rng = np.random.default_rng(self.random_state)
+        n_samples, n_features = X.shape
+        K = self.n_components
+
+        # Initialise parameters
+        self.weights_ = np.full(K, 1.0 / K)
+        # Pick K random data points as initial means
+        idx = rng.choice(n_samples, K, replace=False)
+        self.means_ = X[idx].copy().astype(float)
+        # Identity covariances scaled by data variance
+        var = np.var(X, axis=0).mean()
+        self.covariances_ = np.array(
+            [np.eye(n_features) * var for _ in range(K)]
+        )
+
+        prev_ll = -np.inf
+        for iteration in range(self.max_iter):
+            # E-step
+            r = self._e_step(X)
+            # M-step
+            self._m_step(X, r)
+            # Check convergence
+            ll = self._log_likelihood(X)
+            if abs(ll - prev_ll) < self.tol:
+                self.converged_ = True
+                break
+            prev_ll = ll
+            self.n_iter_ = iteration + 1
+
+        self.lower_bound_ = prev_ll
+        return self
+
+    def predict(self, X: np.ndarray) -> np.ndarray:
+        """
+        Assign each sample to the most likely component.
+
+        Returns
+        -------
+        labels : ndarray of shape (n_samples,)
+        """
+        r = self._e_step(X)
+        return np.argmax(r, axis=1)
+
+    def predict_proba(self, X: np.ndarray) -> np.ndarray:
+        """
+        Return responsibility (soft membership) for each component.
+
+        Returns
+        -------
+        r : ndarray of shape (n_samples, n_components)
+        """
+        return self._e_step(X)

mlscratch/unsupervised/hierarchical_clustering.py

@@ -0,0 +1,137 @@
+"""
+Hierarchical Agglomerative Clustering (HAC)
+============================================
+Builds a hierarchy of clusters bottom-up: each sample starts as its own
+cluster; at each step the two closest clusters are merged, until all
+samples belong to one cluster or a stopping criterion is met.
+
+Linkage criteria implemented
+-----------------------------
+- 'single'   : distance = min pairwise distance between clusters
+- 'complete' : distance = max pairwise distance between clusters
+- 'average'  : distance = mean pairwise distance between clusters
+- 'ward'     : distance = increase in total within-cluster variance on merge
+
+Only numpy is used.
+"""
+
+import numpy as np
+
+
+class AgglomerativeClustering:
+    """
+    Hierarchical agglomerative clustering.
+
+    Parameters
+    ----------
+    n_clusters : int
+        Target number of clusters to cut the dendrogram to.
+    linkage : str
+        One of {'single', 'complete', 'average', 'ward'}.
+    """
+
+    def __init__(self, n_clusters: int = 2, linkage: str = "ward"):
+        if linkage not in {"single", "complete", "average", "ward"}:
+            raise ValueError(
+                f"linkage must be one of 'single', 'complete', 'average', "
+                f"'ward'. Got '{linkage}'."
+            )
+        self.n_clusters = n_clusters
+        self.linkage = linkage
+        self.labels_ = None
+
+    # ------------------------------------------------------------------
+    # Distance helpers
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _pairwise_distances(X: np.ndarray) -> np.ndarray:
+        """Return symmetric Euclidean distance matrix (n x n)."""
+        n = len(X)
+        D = np.zeros((n, n))
+        for i in range(n):
+            for j in range(i + 1, n):
+                d = np.sqrt(np.sum((X[i] - X[j]) ** 2))
+                D[i, j] = D[j, i] = d
+        return D
+
+    def _cluster_distance(
+        self,
+        c1: list,
+        c2: list,
+        X: np.ndarray,
+        D: np.ndarray,
+    ) -> float:
+        """Compute linkage distance between two clusters."""
+        dists = [D[i, j] for i in c1 for j in c2]
+
+        if self.linkage == "single":
+            return min(dists)
+        if self.linkage == "complete":
+            return max(dists)
+        if self.linkage == "average":
+            return sum(dists) / len(dists)
+        if self.linkage == "ward":
+            # Increase in total within-cluster variance
+            combined = c1 + c2
+            centroid_c1 = X[c1].mean(axis=0)
+            centroid_c2 = X[c2].mean(axis=0)
+            centroid_merged = X[combined].mean(axis=0)
+            wcv_c1 = np.sum((X[c1] - centroid_c1) ** 2)
+            wcv_c2 = np.sum((X[c2] - centroid_c2) ** 2)
+            wcv_merged = np.sum((X[combined] - centroid_merged) ** 2)
+            return wcv_merged - wcv_c1 - wcv_c2
+        raise ValueError(f"Unknown linkage '{self.linkage}'")
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def fit(self, X: np.ndarray) -> "AgglomerativeClustering":
+        """
+        Perform hierarchical clustering on X.
+
+        Parameters
+        ----------
+        X : ndarray of shape (n_samples, n_features)
+        """
+        n_samples = len(X)
+        D = self._pairwise_distances(X)
+
+        # Each sample starts as its own cluster (stored as list of indices)
+        clusters = [[i] for i in range(n_samples)]
+
+        while len(clusters) > self.n_clusters:
+            min_dist = np.inf
+            merge_i, merge_j = 0, 1
+
+            # Find closest pair of clusters
+            for i in range(len(clusters)):
+                for j in range(i + 1, len(clusters)):
+                    d = self._cluster_distance(clusters[i], clusters[j], X, D)
+                    if d < min_dist:
+                        min_dist = d
+                        merge_i, merge_j = i, j
+
+            # Merge
+            merged = clusters[merge_i] + clusters[merge_j]
+            # Remove old clusters (higher index first to preserve positions)
+            clusters = [
+                c for idx, c in enumerate(clusters)
+                if idx != merge_i and idx != merge_j
+            ]
+            clusters.append(merged)
+
+        # Assign labels
+        labels = np.empty(n_samples, dtype=int)
+        for cluster_id, indices in enumerate(clusters):
+            for idx in indices:
+                labels[idx] = cluster_id
+
+        self.labels_ = labels
+        return self
+
+    def fit_predict(self, X: np.ndarray) -> np.ndarray:
+        """Fit and return cluster labels."""
+        self.fit(X)
+        return self.labels_

mlscratch/unsupervised/ica.py

@@ -0,0 +1,167 @@
+"""
+Independent Component Analysis (ICA) — FastICA
+================================================
+Separates a multivariate signal into additive, statistically independent
+components (blind source separation).  This is the FastICA algorithm using
+a fixed-point iteration to maximise non-Gaussianity (measured via kurtosis
+or negentropy via the logcosh or exp contrast functions).
+
+Steps
+-----
+1. Whiten X: remove correlations so components have unit variance.
+2. For each component, run a fixed-point update on a weight vector w:
+       w ← E[X g(w^T X)] − E[g'(w^T X)] w
+   then normalise w and orthogonalise against previous components.
+3. The independent components are  S = W X_white.
+
+Only numpy is used.
+"""
+
+import numpy as np
+
+
+class FastICA:
+    """
+    FastICA — Independent Component Analysis.
+
+    Parameters
+    ----------
+    n_components : int or None
+        Number of independent components to extract.
+        If None, uses min(n_samples, n_features).
+    max_iter : int
+        Maximum iterations per component.
+    tol : float
+        Convergence tolerance (change in w between iterations).
+    fun : str
+        Contrast function: 'logcosh' (default) or 'exp'.
+    random_state : int or None
+        Seed for weight initialisation.
+    """
+
+    def __init__(
+        self,
+        n_components: int | None = None,
+        max_iter: int = 200,
+        tol: float = 1e-4,
+        fun: str = "logcosh",
+        random_state: int | None = None,
+    ):
+        if fun not in {"logcosh", "exp"}:
+            raise ValueError("fun must be 'logcosh' or 'exp'.")
+        self.n_components = n_components
+        self.max_iter = max_iter
+        self.tol = tol
+        self.fun = fun
+        self.random_state = random_state
+
+        self.components_ = None    # (n_components, n_features) unmixing matrix
+        self.mixing_ = None        # pseudo-inverse of components_
+        self.mean_ = None          # for centering
+        self.whitening_ = None     # whitening matrix
+
+    # ------------------------------------------------------------------
+    # Contrast functions (g) and their derivatives (g')
+    # ------------------------------------------------------------------
+
+    def _g_and_gprime(self, u: np.ndarray):
+        """Return g(u) and g'(u) element-wise for the chosen contrast."""
+        if self.fun == "logcosh":
+            g = np.tanh(u)
+            g_prime = 1.0 - g ** 2
+        else:  # exp
+            exp_u = np.exp(-0.5 * u ** 2)
+            g = u * exp_u
+            g_prime = (1.0 - u ** 2) * exp_u
+        return g, g_prime
+
+    # ------------------------------------------------------------------
+    # Whitening
+    # ------------------------------------------------------------------
+
+    def _whiten(self, X: np.ndarray):
+        """
+        Whiten data: zero-mean, unit variance, uncorrelated.
+
+        Returns
+        -------
+        X_white : ndarray (n_features, n_samples)  — note transposed
+        W_white : whitening matrix
+        """
+        # X is (n_samples, n_features); work in (n_features, n_samples) form
+        X_c = X.T                       # (d, n)
+        cov = np.cov(X_c, rowvar=True)  # (d, d)
+        eigvals, eigvecs = np.linalg.eigh(cov)
+        # Guard against near-zero eigenvalues
+        eigvals = np.maximum(eigvals, 1e-10)
+        W_white = eigvecs @ np.diag(1.0 / np.sqrt(eigvals)) @ eigvecs.T
+        X_white = W_white @ X_c         # (d, n)
+        return X_white, W_white
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def fit(self, X: np.ndarray) -> "FastICA":
+        """
+        Fit ICA on X.
+
+        Parameters
+        ----------
+        X : ndarray of shape (n_samples, n_features)
+        """
+        rng = np.random.default_rng(self.random_state)
+        n_samples, n_features = X.shape
+        n_components = self.n_components or min(n_samples, n_features)
+
+        self.mean_ = X.mean(axis=0)
+        X_centered = X - self.mean_
+
+        X_white, self.whitening_ = self._whiten(X_centered)  # (d, n)
+
+        W = np.zeros((n_components, n_features))  # unmixing in white space
+
+        for p in range(n_components):
+            w = rng.standard_normal(n_features)
+            w /= np.linalg.norm(w) + 1e-12
+
+            for _ in range(self.max_iter):
+                u = w @ X_white                     # (n,)
+                g_u, g_prime_u = self._g_and_gprime(u)
+
+                # Fixed-point update
+                w_new = (X_white * g_u).mean(axis=1) - g_prime_u.mean() * w
+
+                # Deflation: subtract projections onto previous components
+                for j in range(p):
+                    w_new -= (w_new @ W[j]) * W[j]
+
+                w_new /= np.linalg.norm(w_new) + 1e-12
+
+                if abs(abs(w_new @ w) - 1.0) < self.tol:
+                    w = w_new
+                    break
+                w = w_new
+
+            W[p] = w
+
+        # Recover components in original (non-whitened) space
+        self.components_ = W @ self.whitening_   # (n_components, n_features)
+        self.mixing_ = np.linalg.pinv(self.components_)
+        return self
+
+    def transform(self, X: np.ndarray) -> np.ndarray:
+        """
+        Recover independent components from X.
+
+        Returns
+        -------
+        S : ndarray of shape (n_samples, n_components)
+        """
+        X_centered = X - self.mean_
+        return X_centered @ self.components_.T
+
+    def fit_transform(self, X: np.ndarray) -> np.ndarray:
+        """Fit and return independent components."""
+        self.fit(X)
+        return self.transform(X)