scratchkit 0.2.0__py3-none-any.whl

mlscratch/supervised/linear_regression.py

@@ -0,0 +1,197 @@
+"""
+Linear Regression
+=================
+
+Ordinary Least Squares and mini-batch gradient descent implementations for
+linear regression.
+
+The module uses a clear, sklearn-compatible interface with explicit math.
+"""
+
+from __future__ import annotations
+
+from typing import Callable
+
+import numpy as np
+from numpy.typing import ArrayLike, NDArray
+
+FloatArray = NDArray[np.float64]
+
+
+def _validate_regression_inputs(
+    X: ArrayLike, y: ArrayLike,
+) -> tuple[FloatArray, FloatArray]:
+    X_arr = np.asarray(X, dtype=float)
+    y_arr = np.asarray(y, dtype=float).flatten()
+    if X_arr.ndim != 2:
+        raise ValueError("X must be a 2D array of shape (n_samples, n_features).")
+    if X_arr.shape[0] != y_arr.shape[0]:
+        raise ValueError(
+            f"X has {X_arr.shape[0]} samples but y has {y_arr.shape[0]}."
+        )
+    return X_arr, y_arr
+
+
+class OrdinaryLeastSquares:
+    """Ordinary least squares regression using the normal equations.
+
+    Parameters
+    ----------
+    add_intercept : bool, default=True
+        If True, the model fits an intercept term by prepending a column of ones
+        to the design matrix.
+
+    Attributes
+    ----------
+    coef_ : FloatArray
+        Estimated regression coefficients for each feature.
+    intercept_ : float
+        Estimated bias term.
+    residuals_ : FloatArray
+        Residual values after fitting.
+    """
+
+    def __init__(self, add_intercept: bool = True) -> None:
+        self.add_intercept = add_intercept
+        self.coef_: FloatArray | None = None
+        self.intercept_: float | None = None
+        self.residuals_: FloatArray | None = None
+
+    def fit(self, X: ArrayLike, y: ArrayLike) -> "OrdinaryLeastSquares":
+        r"""Fit the linear regression model.
+
+        The closed-form least-squares solution is computed via a numerically
+        stable least-squares solver:
+
+        .. math::
+            \hat{\beta} = \operatorname{argmin}_\beta \|X \beta - y\|_2^2
+
+        Returns
+        -------
+        self : OrdinaryLeastSquares
+        """
+        X_arr, y_arr = _validate_regression_inputs(X, y)
+        if self.add_intercept:
+            X_arr = np.column_stack([np.ones(X_arr.shape[0]), X_arr])
+
+        solution, residuals, *_ = np.linalg.lstsq(X_arr, y_arr, rcond=None)
+        if self.add_intercept:
+            self.intercept_ = float(solution[0])
+            self.coef_ = solution[1:].astype(np.float64)
+            y_pred = X_arr[:, 1:] @ self.coef_ + self.intercept_
+        else:
+            self.intercept_ = 0.0
+            self.coef_ = solution.astype(np.float64)
+            y_pred = X_arr @ self.coef_
+
+        self.residuals_ = y_arr - y_pred
+        return self
+
+    def predict(self, X: ArrayLike) -> FloatArray:
+        """Predict target values for new data."""
+        if self.coef_ is None or self.intercept_ is None:
+            raise RuntimeError("Call fit() before predict().")
+        X_arr = np.asarray(X, dtype=float)
+        if X_arr.ndim != 2:
+            raise ValueError("X must be a 2D array.")
+        result = X_arr @ self.coef_ + self.intercept_
+        return result.astype(np.float64)
+
+    def score(self, X: ArrayLike, y: ArrayLike) -> float:
+        """Return the coefficient of determination R² on the given data."""
+        _, y_arr = _validate_regression_inputs(X, y)
+        y_pred = self.predict(X)
+        ss_res = np.sum((y_arr - y_pred) ** 2)
+        ss_tot = np.sum((y_arr - np.mean(y_arr)) ** 2)
+        return float(1.0 - ss_res / ss_tot) if ss_tot > 0 else 0.0
+
+
+class GradientDescentRegressor:
+    r"""Linear regression using mini-batch gradient descent.
+
+    The squared error loss is:
+
+    .. math::
+        L(W, b) = \frac{1}{n} \sum_{i=1}^n (y_i - X_i W - b)^2
+
+    The gradient with respect to the weights is:
+
+    .. math::
+        \frac{\partial L}{\partial W} = -\frac{2}{n} X^\top (y - XW - b)
+    """
+
+    def __init__(
+        self,
+        learning_rate: float = 0.01,
+        n_epochs: int = 1000,
+        batch_size: int = 32,
+        random_state: int | None = None,
+        verbose: bool = False,
+    ) -> None:
+        self.learning_rate = learning_rate
+        self.n_epochs = n_epochs
+        self.batch_size = batch_size
+        self.random_state = random_state
+        self.verbose = verbose
+        self.coef_: FloatArray | None = None
+        self.intercept_: float | None = None
+        self.loss_history_: list[float] = []
+
+    def fit(self, X: ArrayLike, y: ArrayLike) -> "GradientDescentRegressor":
+        """Fit the model using mini-batch gradient descent."""
+        X_arr, y_arr = _validate_regression_inputs(X, y)
+        rng = np.random.default_rng(self.random_state)
+        n_samples, n_features = X_arr.shape
+        self.coef_ = np.zeros(n_features, dtype=np.float64)
+        self.intercept_ = 0.0
+        self.loss_history_ = []
+        learning_rate = self.learning_rate
+        prev_loss = float("inf")
+
+        for epoch in range(self.n_epochs):
+            indices = rng.permutation(n_samples)
+            X_shuffled = X_arr[indices]
+            y_shuffled = y_arr[indices]
+            coef_before = self.coef_.copy()
+            intercept_before = self.intercept_
+
+            for start in range(0, n_samples, self.batch_size):
+                end = start + self.batch_size
+                X_batch = X_shuffled[start:end]
+                y_batch = y_shuffled[start:end]
+                y_pred = X_batch @ self.coef_ + self.intercept_
+                errors = y_pred - y_batch
+                grad_w = (2.0 / X_batch.shape[0]) * (X_batch.T @ errors)
+                grad_b = (2.0 / X_batch.shape[0]) * np.sum(errors)
+                self.coef_ -= learning_rate * grad_w
+                self.intercept_ -= learning_rate * grad_b
+
+            loss = np.mean((X_arr @ self.coef_ + self.intercept_ - y_arr) ** 2)
+            if loss > prev_loss + 1e-12:
+                self.coef_ = coef_before
+                self.intercept_ = intercept_before
+                learning_rate *= 0.5
+                loss = prev_loss
+
+            self.loss_history_.append(float(loss))
+            prev_loss = loss
+            if self.verbose and epoch % 100 == 0:
+                print(f"Epoch {epoch} loss={loss:.6f}")
+        return self
+
+    def predict(self, X: ArrayLike) -> FloatArray:
+        """Predict target values for new data."""
+        if self.coef_ is None or self.intercept_ is None:
+            raise RuntimeError("Call fit() before predict().")
+        X_arr = np.asarray(X, dtype=float)
+        if X_arr.ndim != 2:
+            raise ValueError("X must be a 2D array.")
+        return (X_arr @ self.coef_ + self.intercept_).astype(np.float64)
+
+    def score(self, X: ArrayLike, y: ArrayLike) -> float:
+        """Return the coefficient of determination R² on the given data."""
+        _, y_arr = _validate_regression_inputs(X, y)
+        y_pred = self.predict(X)
+        ss_res = np.sum((y_arr - y_pred) ** 2)
+        ss_tot = np.sum((y_arr - np.mean(y_arr)) ** 2)
+        return float(1.0 - ss_res / ss_tot) if ss_tot > 0 else 0.0

mlscratch/supervised/logistic_regression.py

@@ -0,0 +1,119 @@
+"""
+Logistic Regression
+===================
+
+A from-scratch binary classifier using gradient descent and a sigmoid link.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+from numpy.typing import ArrayLike, NDArray
+
+FloatArray = NDArray[np.float64]
+
+
+def _validate_classification_inputs(
+    X: ArrayLike, y: ArrayLike,
+) -> tuple[FloatArray, FloatArray]:
+    X_arr = np.asarray(X, dtype=float)
+    y_arr = np.asarray(y, dtype=float).flatten()
+    if X_arr.ndim != 2:
+        raise ValueError("X must be a 2D array of shape (n_samples, n_features).")
+    if X_arr.shape[0] != y_arr.shape[0]:
+        raise ValueError(
+            f"X has {X_arr.shape[0]} samples but y has {y_arr.shape[0]}."
+        )
+    if not np.all(np.isin(y_arr, [0.0, 1.0])):
+        raise ValueError("y must contain only binary labels 0 and 1.")
+    return X_arr, y_arr
+
+
+class LogisticRegression:
+    r"""Binary logistic regression using gradient descent.
+
+    The model is:
+
+    .. math::
+        p(y=1 \mid x) = \sigma(w^\top x + b),
+        \quad \sigma(z) = \frac{1}{1 + e^{-z}}
+
+    The loss is the binary cross-entropy:
+
+    .. math::
+        L = -\frac{1}{n} \sum_{i=1}^n 
+            \left[y_i \log \sigma(z_i) + (1-y_i) \log (1-\sigma(z_i))\right]
+    """
+
+    def __init__(
+        self,
+        learning_rate: float = 0.01,
+        n_epochs: int = 1000,
+        batch_size: int = 32,
+        random_state: int | None = None,
+        verbose: bool = False,
+    ) -> None:
+        self.learning_rate = learning_rate
+        self.n_epochs = n_epochs
+        self.batch_size = batch_size
+        self.random_state = random_state
+        self.verbose = verbose
+        self.weights_: FloatArray | None = None
+        self.bias_: float | None = None
+        self.loss_history_: list[float] = []
+
+    def fit(self, X: ArrayLike, y: ArrayLike) -> "LogisticRegression":
+        """Fit the logistic regression model to binary data."""
+        X_arr, y_arr = _validate_classification_inputs(X, y)
+        rng = np.random.default_rng(self.random_state)
+        n_samples, n_features = X_arr.shape
+        self.weights_ = np.zeros(n_features, dtype=np.float64)
+        self.bias_ = 0.0
+        self.loss_history_ = []
+
+        for epoch in range(self.n_epochs):
+            perm = rng.permutation(n_samples)
+            for start in range(0, n_samples, self.batch_size):
+                end = start + self.batch_size
+                X_batch = X_arr[perm[start:end]]
+                y_batch = y_arr[perm[start:end]]
+                z = X_batch @ self.weights_ + self.bias_
+                predictions = self._sigmoid(z)
+                errors = predictions - y_batch
+                grad_w = X_batch.T @ errors / X_batch.shape[0]
+                grad_b = np.mean(errors)
+                self.weights_ -= self.learning_rate * grad_w
+                self.bias_ -= self.learning_rate * grad_b
+
+            loss = self._binary_cross_entropy(y_arr, self.predict_proba(X_arr))
+            self.loss_history_.append(float(loss))
+            if self.verbose and epoch % 100 == 0:
+                print(f"Epoch {epoch} loss={loss:.6f}")
+        return self
+
+    def predict_proba(self, X: ArrayLike) -> FloatArray:
+        """Return probability estimates for the positive class."""
+        if self.weights_ is None or self.bias_ is None:
+            raise RuntimeError("Call fit() before predict_proba().")
+        X_arr = np.asarray(X, dtype=float)
+        if X_arr.ndim != 2:
+            raise ValueError("X must be a 2D array.")
+        return self._sigmoid(X_arr @ self.weights_ + self.bias_)
+
+    def predict(self, X: ArrayLike) -> NDArray[np.int64]:
+        """Return binary predictions for the input data."""
+        return (self.predict_proba(X) >= 0.5).astype(np.int64)
+
+    def score(self, X: ArrayLike, y: ArrayLike) -> float:
+        """Return classification accuracy on the given dataset."""
+        X_arr, y_arr = _validate_classification_inputs(X, y)
+        y_pred = self.predict(X_arr)
+        return float(np.mean(y_pred == y_arr))
+
+    def _sigmoid(self, z: FloatArray) -> FloatArray:
+        z = np.clip(z, -500.0, 500.0)
+        return 1.0 / (1.0 + np.exp(-z))
+
+    def _binary_cross_entropy(self, y_true: FloatArray, y_prob: FloatArray) -> float:
+        y_prob = np.clip(y_prob, 1e-12, 1.0 - 1e-12)
+        return float(-np.mean(y_true * np.log(y_prob) + (1.0 - y_true) * np.log(1.0 - y_prob)))

mlscratch/supervised/naive_bayes.py

@@ -0,0 +1,113 @@
+r"""
+Gaussian Naive Bayes Classifier
+================================
+
+A probabilistic classifier that assumes each class follows a Gaussian
+distribution and that features are conditionally independent given the class.
+
+The model computes class log-likelihoods as:
+
+.. math::
+    \log p(\mathbf{x}, y_k)
+    = \log \pi_k - \frac{1}{2} \sum_{j=1}^d \left[
+        \log(2\pi \sigma_{kj}^2)
+        + \frac{(x_j - \mu_{kj})^2}{\sigma_{kj}^2}
+    \right]
+
+Complexity
+----------
+- Training: O(n d)
+- Inference: O(n d)
+- Space:    O(K d)
+"""
+
+from __future__ import annotations
+
+import numpy as np
+from numpy.typing import ArrayLike, NDArray
+
+FloatArray = NDArray[np.float64]
+IntArray = NDArray[np.int64]
+
+
+def _validate_classification_inputs(
+    X: ArrayLike,
+    y: ArrayLike,
+) -> tuple[FloatArray, IntArray]:
+    X_arr = np.asarray(X, dtype=float)
+    y_arr = np.asarray(y, dtype=int).flatten()
+    if X_arr.ndim != 2:
+        raise ValueError("X must be a 2D array of shape (n_samples, n_features).")
+    if X_arr.shape[0] != y_arr.shape[0]:
+        raise ValueError(
+            f"X has {X_arr.shape[0]} samples but y has {y_arr.shape[0]}."
+        )
+    return X_arr, y_arr
+
+
+class GaussianNB:
+    """Gaussian Naive Bayes classifier.
+
+    Parameters
+    ----------
+    var_smoothing : float, default=1e-9
+        Portion of the largest variance of all features added to variances for
+        stability in the Gaussian likelihood denominator.
+    """
+
+    def __init__(self, var_smoothing: float = 1e-9) -> None:
+        self.var_smoothing = float(var_smoothing)
+        self.class_count_: IntArray | None = None
+        self.class_prior_: FloatArray | None = None
+        self.class_mean_: FloatArray | None = None
+        self.class_var_: FloatArray | None = None
+        self.classes_: IntArray | None = None
+        self.n_features_in_: int | None = None
+
+    def fit(self, X: ArrayLike, y: ArrayLike) -> "GaussianNB":
+        X_arr, y_arr = _validate_classification_inputs(X, y)
+        self.n_features_in_ = X_arr.shape[1]
+        self.classes_, counts = np.unique(y_arr, return_counts=True)
+        self.class_count_ = counts.astype(np.int64)
+        self.class_prior_ = counts.astype(np.float64) / float(y_arr.size)
+
+        means = []
+        variances = []
+        for clazz in self.classes_:
+            X_class = X_arr[y_arr == clazz]
+            means.append(X_class.mean(axis=0))
+            variances.append(X_class.var(axis=0) + self.var_smoothing)
+
+        self.class_mean_ = np.vstack(means)
+        self.class_var_ = np.vstack(variances)
+        return self
+
+    def predict(self, X: ArrayLike) -> IntArray:
+        if self.class_prior_ is None or self.class_mean_ is None or self.class_var_ is None:
+            raise RuntimeError("Call fit() before predict().")
+        X_arr = np.asarray(X, dtype=float)
+        if X_arr.ndim != 2:
+            raise ValueError("X must be a 2D array.")
+        log_likelihood = self._joint_log_likelihood(X_arr)
+        argmax = np.argmax(log_likelihood, axis=1)
+        return self.classes_[argmax]
+
+    def score(self, X: ArrayLike, y: ArrayLike) -> float:
+        X_arr, y_arr = _validate_classification_inputs(X, y)
+        return float(np.mean(self.predict(X_arr) == y_arr))
+
+    def _joint_log_likelihood(self, X: FloatArray) -> FloatArray:
+        n_samples, n_features = X.shape
+        if self.class_mean_ is None or self.class_var_ is None or self.class_prior_ is None:
+            raise RuntimeError("Classifier must be fitted before computing likelihoods.")
+
+        joint = np.empty((n_samples, self.classes_.size), dtype=np.float64)
+        for idx, (prior, mean, var) in enumerate(
+            zip(self.class_prior_, self.class_mean_, self.class_var_)
+        ):
+            log_prior = np.log(prior)
+            log_det = -0.5 * np.sum(np.log(2.0 * np.pi * var))
+            diff = X - mean
+            exp_term = -0.5 * np.sum((diff ** 2) / var, axis=1)
+            joint[:, idx] = log_prior + log_det + exp_term
+        return joint