scratchkit 0.2.0__py3-none-any.whl

mlscratch/__init__.py

@@ -0,0 +1,56 @@
+"""
+mlscratch
+=========
+Pure-NumPy from-scratch implementations of ML / AI / RL / Bayesian algorithms.
+No PyTorch. No TensorFlow. No scikit-learn. Just numpy and the maths.
+
+Sub-packages
+------------
+mlscratch.supervised        Supervised learning algorithms
+mlscratch.unsupervised      Unsupervised learning algorithms
+mlscratch.bayesian          Bayesian methods
+mlscratch.reinforcement     Reinforcement learning algorithms
+mlscratch.neural            Neural network architectures
+mlscratch.metrics           Classification & regression evaluation metrics
+mlscratch.preprocessing     Scalers, encoders, polynomial features, train_test_split
+
+Quick-start
+-----------
+>>> from mlscratch.unsupervised import KMeans
+>>> from mlscratch.supervised import LinearRegression, RandomForestClassifier
+>>> from mlscratch.bayesian import GaussianNB
+>>> from mlscratch.reinforcement import QLearning
+>>> from mlscratch.neural import MultiLayerPerceptron
+>>> from mlscratch.metrics import accuracy_score
+>>> from mlscratch.preprocessing import StandardScaler, train_test_split
+
+Install
+-------
+    pip install scratchkit              # core (numpy only); import name is "mlscratch"
+    pip install "scratchkit[dev]"       # + pytest, ruff, black, mypy
+    pip install "scratchkit[docs]"      # + mkdocs
+    pip install "scratchkit[all]"       # everything
+
+Links
+-----
+GitHub      : https://github.com/Mattral/ML-AI-Algorithms-from-scratch
+Issues      : https://github.com/Mattral/ML-AI-Algorithms-from-scratch/issues
+Changelog   : https://github.com/Mattral/ML-AI-Algorithms-from-scratch/blob/main/CHANGELOG.md
+"""
+
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    # The importable package is "mlscratch", but the PyPI distribution it
+    # ships under is "scratchkit" (the name "mlscratch" was already taken
+    # by an unrelated project) — look up metadata under the distribution
+    # name, the same way `import bs4` resolves "beautifulsoup4" metadata.
+    __version__: str = version("scratchkit")
+except PackageNotFoundError:
+    # Package is not installed (e.g. running directly from source tree)
+    __version__ = "0.0.0+dev"
+
+__author__  = "Mattral"
+__license__ = "Apache-2.0"
+
+__all__ = ["__version__", "__author__", "__license__"]

mlscratch/__main__.py

@@ -0,0 +1,118 @@
+"""
+mlscratch CLI
+=============
+Usage
+-----
+    python -m mlscratch              # same as 'info'
+    python -m mlscratch version      # print version
+    python -m mlscratch info         # version + sub-package summary
+    python -m mlscratch list         # list all available algorithm classes
+    python -m mlscratch list supervised
+    python -m mlscratch list unsupervised
+    python -m mlscratch list bayesian
+    python -m mlscratch list reinforcement
+    python -m mlscratch list neural
+    python -m mlscratch list metrics
+    python -m mlscratch list preprocessing
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+
+
+# ── helpers ───────────────────────────────────────────────────────────────────
+
+def _print_version() -> None:
+    import mlscratch
+    print(f"mlscratch {mlscratch.__version__}")
+
+
+def _print_info() -> None:
+    import mlscratch
+    import numpy as np
+
+    print(f"\nmlscratch {mlscratch.__version__}")
+    print(f"  numpy   : {np.__version__}")
+    print(f"  python  : {sys.version.split()[0]}")
+    print()
+
+    modules = {
+        "supervised":     "mlscratch.supervised",
+        "unsupervised":   "mlscratch.unsupervised",
+        "bayesian":       "mlscratch.bayesian",
+        "reinforcement":  "mlscratch.reinforcement",
+        "neural":         "mlscratch.neural",
+        "metrics":        "mlscratch.metrics",
+        "preprocessing":  "mlscratch.preprocessing",
+    }
+
+    for name, mod_path in modules.items():
+        try:
+            import importlib
+            mod = importlib.import_module(mod_path)
+            n = len(getattr(mod, "__all__", []))
+            print(f"  {name:<18} {n} public symbol(s)")
+        except ImportError:
+            print(f"  {name:<18} not yet installed")
+
+    print()
+
+
+def _list_algorithms(subpackage: str | None = None) -> None:
+    import importlib
+
+    targets = (
+        {"supervised", "unsupervised", "bayesian", "reinforcement", "neural", "metrics", "preprocessing"}
+        if subpackage is None
+        else {subpackage}
+    )
+
+    for name in sorted(targets):
+        mod_path = f"mlscratch.{name}"
+        try:
+            mod = importlib.import_module(mod_path)
+            symbols = getattr(mod, "__all__", [])
+            print(f"\n[{name}]")
+            for s in sorted(symbols):
+                print(f"  {s}")
+        except ImportError:
+            print(f"\n[{name}]  — not available (sub-package not installed)")
+
+
+# ── entry point ───────────────────────────────────────────────────────────────
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="mlscratch",
+        description="mlscratch — ML algorithms from scratch",
+    )
+    subparsers = parser.add_subparsers(dest="command")
+
+    subparsers.add_parser("version", help="Print version and exit")
+    subparsers.add_parser("info",    help="Print version, numpy, and sub-package summary")
+
+    list_parser = subparsers.add_parser("list", help="List available algorithms")
+    list_parser.add_argument(
+        "subpackage",
+        nargs="?",
+        choices=["supervised", "unsupervised", "bayesian", "reinforcement", "neural", "metrics", "preprocessing"],
+        default=None,
+        help="Restrict listing to one sub-package",
+    )
+
+    args = parser.parse_args(argv)
+
+    if args.command in (None, "info"):
+        _print_info()
+    elif args.command == "version":
+        _print_version()
+    elif args.command == "list":
+        _list_algorithms(args.subpackage)
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

mlscratch/bayesian/__init__.py

@@ -0,0 +1,53 @@
+"""
+mlscratch.bayesian
+==================
+From-scratch implementations of Bayesian learning algorithms.
+Drop these files alongside existing code in src/mlscratch/bayesian/.
+
+Algorithms
+----------
+GaussianNB                  – Gaussian Naive Bayes
+MultinomialNB               – Multinomial Naive Bayes
+BernoulliNB                 – Bernoulli Naive Bayes
+BayesianLinearRegression    – Conjugate Gaussian prior regression
+GaussianProcessRegressor    – GP Regression (RBF, Matern52, Linear, Periodic)
+RBFKernel                   – RBF / Squared-Exponential kernel
+Matern52Kernel              – Matern 5/2 kernel
+LinearKernel                – Linear kernel
+PeriodicKernel              – Periodic kernel
+HiddenMarkovModel           – Discrete HMM (forward-backward, Viterbi, Baum-Welch)
+BayesianNeuralNetwork       – BNN via mean-field variational inference
+BayesianNetwork             – Discrete DAG (variable elimination, sampling)
+KalmanFilter                – Linear Kalman Filter + RTS Smoother
+"""
+
+from .naive_bayes import GaussianNB, MultinomialNB, BernoulliNB  # noqa: F401
+from .bayesian_linear_regression import BayesianLinearRegression  # noqa: F401
+from .gaussian_process import (                                    # noqa: F401
+    GaussianProcessRegressor,
+    RBFKernel,
+    Matern52Kernel,
+    LinearKernel,
+    PeriodicKernel,
+)
+from .hmm import HiddenMarkovModel                                 # noqa: F401
+from .bayesian_nn import BayesianNeuralNetwork, BayesianLayer      # noqa: F401
+from .bayesian_network import BayesianNetwork                      # noqa: F401
+from .kalman_filter import KalmanFilter                            # noqa: F401
+
+__all__ = [
+    "GaussianNB",
+    "MultinomialNB",
+    "BernoulliNB",
+    "BayesianLinearRegression",
+    "GaussianProcessRegressor",
+    "RBFKernel",
+    "Matern52Kernel",
+    "LinearKernel",
+    "PeriodicKernel",
+    "HiddenMarkovModel",
+    "BayesianNeuralNetwork",
+    "BayesianLayer",
+    "BayesianNetwork",
+    "KalmanFilter",
+]

mlscratch/bayesian/bayesian_linear_regression.py

@@ -0,0 +1,171 @@
+"""
+Bayesian Linear Regression
+===========================
+Treats the model weights as a distribution, not a point estimate.
+
+Model
+-----
+    y = X w + ε,   ε ~ N(0, β⁻¹)
+    w ~ N(0, α⁻¹ I)
+
+With a Gaussian prior over weights and Gaussian noise, the posterior over
+weights is also Gaussian (conjugate prior):
+
+    p(w | X, y) = N(w | m_N, S_N)
+
+    S_N = (α I + β X^T X)^{-1}
+    m_N = β S_N X^T y
+
+Predictions are also Gaussian:
+
+    p(y* | x*, X, y) = N(y* | m_N^T x*, σ_N²(x*))
+    σ_N²(x*) = β⁻¹ + x*^T S_N x*
+
+Parameters α (weight precision) and β (noise precision) can be fixed or
+estimated via type-II maximum likelihood (evidence approximation).
+
+Reference: Bishop, PRML, Chapter 3.
+Only numpy is used.
+"""
+
+import numpy as np
+
+
+class BayesianLinearRegression:
+    """
+    Bayesian Linear Regression with conjugate Gaussian prior.
+
+    Parameters
+    ----------
+    alpha : float
+        Prior precision over weights (1 / prior variance).
+    beta : float
+        Noise precision (1 / noise variance).
+    fit_intercept : bool
+        If True, prepend a column of ones to X.
+    optimize_hyperparams : bool
+        If True, estimate alpha and beta via evidence maximisation
+        (iterative re-estimation).  Ignored if False.
+    max_iter : int
+        Maximum iterations for hyperparameter optimisation.
+    tol : float
+        Convergence tolerance for hyperparameter optimisation.
+    """
+
+    def __init__(
+        self,
+        alpha: float = 1.0,
+        beta: float = 1.0,
+        fit_intercept: bool = True,
+        optimize_hyperparams: bool = False,
+        max_iter: int = 300,
+        tol: float = 1e-5,
+    ):
+        self.alpha = alpha
+        self.beta = beta
+        self.fit_intercept = fit_intercept
+        self.optimize_hyperparams = optimize_hyperparams
+        self.max_iter = max_iter
+        self.tol = tol
+
+        self.m_N_ = None    # posterior mean  (n_features,)
+        self.S_N_ = None    # posterior cov   (n_features, n_features)
+
+    # ------------------------------------------------------------------
+    # Helpers
+    # ------------------------------------------------------------------
+
+    def _add_bias(self, X: np.ndarray) -> np.ndarray:
+        if self.fit_intercept:
+            return np.column_stack([np.ones(len(X)), X])
+        return X
+
+    def _compute_posterior(
+        self, X: np.ndarray, y: np.ndarray
+    ) -> tuple:
+        """Return (m_N, S_N) given current alpha and beta."""
+        n_features = X.shape[1]
+        S_N_inv = self.alpha * np.eye(n_features) + self.beta * X.T @ X
+        S_N = np.linalg.inv(S_N_inv)
+        m_N = self.beta * S_N @ X.T @ y
+        return m_N, S_N
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def fit(self, X: np.ndarray, y: np.ndarray) -> "BayesianLinearRegression":
+        """
+        Compute posterior distribution over weights.
+
+        Parameters
+        ----------
+        X : ndarray of shape (n_samples, n_features)
+        y : ndarray of shape (n_samples,)
+        """
+        X_ = self._add_bias(X)
+        n_samples, n_features = X_.shape
+
+        if self.optimize_hyperparams:
+            # Evidence approximation (Bishop PRML §3.5.2)
+            alpha, beta = self.alpha, self.beta
+            for _ in range(self.max_iter):
+                m_N, S_N = self._compute_posterior(X_, y)
+                # Eigenvalues of β X^T X
+                eigvals = np.linalg.eigvalsh(beta * X_.T @ X_)
+                gamma = np.sum(eigvals / (alpha + eigvals))
+
+                alpha_new = gamma / (m_N @ m_N)
+                residuals = y - X_ @ m_N
+                beta_new = (n_samples - gamma) / (residuals @ residuals)
+
+                alpha_new = max(alpha_new, 1e-10)
+                beta_new = max(beta_new, 1e-10)
+
+                if abs(alpha_new - alpha) < self.tol and abs(beta_new - beta) < self.tol:
+                    alpha, beta = alpha_new, beta_new
+                    break
+                alpha, beta = alpha_new, beta_new
+
+            self.alpha, self.beta = alpha, beta
+
+        self.m_N_, self.S_N_ = self._compute_posterior(X_, y)
+        return self
+
+    def predict(self, X: np.ndarray, return_std: bool = False):
+        """
+        Predictive mean (and optionally std) for new inputs.
+
+        Parameters
+        ----------
+        X : ndarray of shape (n_samples, n_features)
+        return_std : bool
+            If True, also return the predictive standard deviation.
+
+        Returns
+        -------
+        y_mean : ndarray of shape (n_samples,)
+        y_std  : ndarray of shape (n_samples,)  [only if return_std=True]
+        """
+        X_ = self._add_bias(X)
+        y_mean = X_ @ self.m_N_
+
+        if not return_std:
+            return y_mean
+
+        # Predictive variance: β⁻¹ + x^T S_N x
+        var = (1.0 / self.beta) + np.einsum("ij,jk,ik->i", X_, self.S_N_, X_)
+        return y_mean, np.sqrt(np.maximum(var, 0.0))
+
+    @property
+    def coef_(self) -> np.ndarray:
+        """Posterior mean weights (excluding bias if fit_intercept=True)."""
+        if self.fit_intercept:
+            return self.m_N_[1:]
+        return self.m_N_
+
+    @property
+    def intercept_(self) -> float:
+        if self.fit_intercept:
+            return float(self.m_N_[0])
+        return 0.0

mlscratch/bayesian/bayesian_network.py

@@ -0,0 +1,248 @@
+"""
+Bayesian Network
+=================
+A Directed Acyclic Graph (DAG) where each node represents a discrete random
+variable and each node stores a Conditional Probability Table (CPT).
+
+Supports:
+- Manual CPT specification
+- Exact inference via Variable Elimination
+- Ancestral sampling
+
+Notation
+--------
+Each variable is a string name.  Observations are dicts {name: value}.
+CPTs are given as numpy arrays indexed in the order (var, *parents).
+
+Only numpy and Python stdlib are used.
+"""
+
+import numpy as np
+from itertools import product
+
+
+class BayesianNetwork:
+    """
+    Discrete Bayesian Network.
+
+    Usage
+    -----
+    >>> bn = BayesianNetwork()
+    >>> bn.add_variable('Rain',     2)
+    >>> bn.add_variable('Sprinkler',2, parents=['Rain'])
+    >>> bn.add_variable('Wet',      2, parents=['Rain','Sprinkler'])
+    >>> bn.set_cpt('Rain',      np.array([0.8, 0.2]))
+    >>> bn.set_cpt('Sprinkler', np.array([[0.6,0.4],[0.99,0.01]]))
+    >>> bn.set_cpt('Wet',       np.array([[[0.99,0.01],[0.1,0.9]],
+    ...                                    [[0.1,0.9],[0.01,0.99]]]))
+    >>> bn.query('Wet', evidence={'Rain':1})
+    """
+
+    def __init__(self):
+        self._parents: dict[str, list] = {}   # name → parent names
+        self._cpt: dict[str, np.ndarray] = {}  # name → CPT array
+        self._domain: dict[str, int] = {}      # name → domain size
+        self._order: list[str] = []            # topological insertion order
+
+    # ------------------------------------------------------------------
+    # Graph construction
+    # ------------------------------------------------------------------
+
+    def add_variable(
+        self,
+        name: str,
+        domain_size: int,
+        parents: list | None = None,
+    ) -> None:
+        """
+        Register a variable.
+
+        Parameters
+        ----------
+        name : str
+        domain_size : int
+            Number of possible values (0 … domain_size-1).
+        parents : list of str or None
+            Names of parent variables (must already be added).
+        """
+        self._domain[name] = domain_size
+        self._parents[name] = parents or []
+        self._order.append(name)
+
+    def set_cpt(self, name: str, cpt: np.ndarray) -> None:
+        """
+        Set the CPT for a variable.
+
+        The array shape must be:
+        (domain_size,)                  for root nodes (no parents)
+        (*parent_domain_sizes, domain_size)  for nodes with parents
+        """
+        self._cpt[name] = np.array(cpt, dtype=float)
+
+    # ------------------------------------------------------------------
+    # Topological order (Kahn's algorithm)
+    # ------------------------------------------------------------------
+
+    def _topological_sort(self) -> list:
+        in_degree = {n: len(self._parents[n]) for n in self._order}
+        queue = [n for n in self._order if in_degree[n] == 0]
+        result = []
+        while queue:
+            node = queue.pop(0)
+            result.append(node)
+            for child in self._order:
+                if node in self._parents[child]:
+                    in_degree[child] -= 1
+                    if in_degree[child] == 0:
+                        queue.append(child)
+        return result
+
+    # ------------------------------------------------------------------
+    # Ancestral sampling
+    # ------------------------------------------------------------------
+
+    def sample(self, n_samples: int = 1, random_state=None) -> list:
+        """
+        Generate samples by ancestral sampling.
+
+        Returns
+        -------
+        samples : list of dicts {variable_name: value}
+        """
+        rng = np.random.default_rng(random_state)
+        order = self._topological_sort()
+        results = []
+
+        for _ in range(n_samples):
+            assignment = {}
+            for var in order:
+                parents = self._parents[var]
+                cpt = self._cpt[var]
+                if not parents:
+                    probs = cpt
+                else:
+                    idx = tuple(assignment[p] for p in parents)
+                    probs = cpt[idx]
+                assignment[var] = int(rng.choice(len(probs), p=probs))
+            results.append(assignment)
+
+        return results if n_samples > 1 else results[0]
+
+    # ------------------------------------------------------------------
+    # Variable Elimination (exact inference)
+    # ------------------------------------------------------------------
+
+    def query(
+        self,
+        query_var: str,
+        evidence: dict | None = None,
+    ) -> np.ndarray:
+        """
+        Compute P(query_var | evidence) via variable elimination.
+
+        Parameters
+        ----------
+        query_var : str
+        evidence : dict {var_name: observed_value} or None
+
+        Returns
+        -------
+        proba : ndarray of shape (domain_size_of_query_var,)
+        """
+        evidence = evidence or {}
+
+        # Build initial factors from CPTs
+        # A factor maps a tuple of variable names to an ndarray
+        factors: list[tuple[tuple, np.ndarray]] = []
+
+        for var in self._order:
+            cpt = self._cpt[var].copy()
+            scope = tuple(self._parents[var] + [var])
+            # Reduce observed variables
+            reduced_scope = []
+            reduced_cpt = cpt
+            for i, v in enumerate(scope):
+                if v in evidence:
+                    # Index into that axis
+                    sl = [slice(None)] * len(scope)
+                    sl[i] = evidence[v]
+                    reduced_cpt = reduced_cpt[tuple(sl)]
+                else:
+                    reduced_scope.append(v)
+            factors.append((tuple(reduced_scope), reduced_cpt))
+
+        # Determine elimination order: all non-query, non-evidence variables
+        to_eliminate = [
+            v for v in self._order
+            if v != query_var and v not in evidence
+        ]
+
+        for var in to_eliminate:
+            # Collect factors that involve `var`
+            relevant = [(s, f) for s, f in factors if var in s]
+            remaining = [(s, f) for s, f in factors if var not in s]
+
+            # Multiply relevant factors
+            product_scope, product_factor = self._factor_product(relevant)
+
+            # Sum out `var`
+            var_idx = list(product_scope).index(var)
+            summed = np.sum(product_factor, axis=var_idx)
+            new_scope = tuple(s for s in product_scope if s != var)
+
+            remaining.append((new_scope, summed))
+            factors = remaining
+
+        # Multiply remaining factors
+        if not factors:
+            return np.ones(self._domain[query_var]) / self._domain[query_var]
+
+        final_scope, final_factor = self._factor_product(factors)
+
+        # Sum out everything except query_var
+        while len(final_scope) > 1:
+            for i, v in enumerate(final_scope):
+                if v != query_var:
+                    final_factor = np.sum(final_factor, axis=i)
+                    final_scope = tuple(s for j, s in enumerate(final_scope) if j != i)
+                    break
+
+        result = final_factor.ravel()
+        total = result.sum()
+        return result / total if total > 0 else result
+
+    def _factor_product(
+        self, factors: list[tuple[tuple, np.ndarray]]
+    ) -> tuple[tuple, np.ndarray]:
+        """Multiply a list of (scope, array) factors together."""
+        if not factors:
+            return ((), np.array(1.0))
+
+        # Compute union scope (maintaining order)
+        union_scope = []
+        for scope, _ in factors:
+            for v in scope:
+                if v not in union_scope:
+                    union_scope.append(v)
+        union_scope = tuple(union_scope)
+
+        # Build shape
+        shape = tuple(self._domain[v] for v in union_scope)
+        result = np.ones(shape)
+
+        for scope, factor in factors:
+            # Expand factor axes to match union_scope
+            expand_axes = [union_scope.index(v) for v in scope]
+            expanded = np.ones(shape)
+            # Use np.einsum-style axis alignment
+            source_shape = [self._domain[v] for v in scope]
+            reshaped = factor.reshape(source_shape)
+
+            # Map each axis of scope into union_scope
+            new_shape = [1] * len(union_scope)
+            for i, ax in enumerate(expand_axes):
+                new_shape[ax] = source_shape[i]
+            expanded = reshaped.reshape(new_shape)
+            result = result * expanded
+
+        return union_scope, result