patentml 0.1.0__py3-none-any.whl

patentml/__init__.py

@@ -0,0 +1,169 @@
+"""
+patentml — Machine Learning from Expired Patents
+All algorithms derived from expired US patents. Zero dependencies. Pure Python stdlib.
+
+EXPIRED PATENT SOURCES (selected key ones):
+  US5970487  Mitsubishi (1997)  GA hardware machine
+  US6912587  AT&T       (2001)  Constraint-weighted GA fitness
+  US6477444  Fuji Xerox (2000)  Genetic programming tree evolution
+  US6212427  Kennedy    (1999)  Particle swarm optimisation
+  US6484115  Storn      (1999)  Differential evolution
+  US7047169  Univ. IL   (2002)  EDA / Bayesian optimisation
+  US7219040  GE         (2002)  Simulated annealing + constraints
+  US5835901  Lockheed   (1997)  Neuroevolution [142 cites]
+  US7162461  SAS        (2005)  Hybrid NN activation search
+  US6128606  AT&T       (1997)  Modular NN + backprop [53 cites]
+  US7747070  Microsoft  (2005)  CNN on GPU [91 cites]
+  US6963862  Texas A&M  (2001)  Recurrent network training
+  US7447669  Nanyang    (2004)  Ant Colony Optimisation
+  US6161130  Microsoft  (1998)  Online classifier [896 cites]
+  US6327581  Microsoft  (1998)  SVM-SMO [173 cites]
+  US7421415  Siemens    (2005)  AdaBoost + feature selection
+  US6816847  Microsoft  (1999)  Decision tree + ensembles [72 cites]
+  US5613012  SmartTouch (1995)  Voting ensemble [1182 cites]
+  US6012058  Microsoft  (1998)  Scalable K-means
+  US6985172  SW Research(2002)  Q-learning / TD + Actor-Critic A2C / PPO-lite [145 cites]
+  US6981040  Utopy      (2000)  Bandit selection [919 cites]
+  US7076102  Philips    (2002)  HMM event learning [116 cites]
+  US6529891  Microsoft  (1998)  PCA / Bayesian model selection [124 cites]
+  US6807536  Microsoft  (2001)  Randomised SVD / low-rank approximation [170 cites]
+  US6263337  Microsoft  (1998)  Scalable EM / Gaussian mixture clustering [142 cites]
+  US6192360  Microsoft  (1998)  Text classifier TF-IDF + Naive Bayes [364 cites]
+  US6496816  Microsoft  (1998)  Collaborative filtering + Bayesian mixture [128 cites]
+  US6374251  Microsoft  (1998)  KNN + BallTree scalable search [187 cites]
+  US6049797  Lucent     (1998)  DBSCAN + OPTICS density clustering [154 cites]
+  US7031530  Lockheed   (2001)  Gradient Boosting Machine (GBM) [188 cites]
+  US6795794  U.Illinois (2002)  Kalman Filter + Extended KF [127 cites]
+  US6931384  Microsoft  (2001)  Gaussian Process regression + classification [258 cites]
+  US6216066  GE         (1998)  Spectral clustering via Laplacian eigenmap [148 cites]
+  US6317707  AT&T       (1998)  Mean Shift + Kernel Density Estimation [269 cites]
+  US7225343  Columbia U (2003)  Isolation Forest + One-Class SVM [159 cites]
+  US6490698  Microsoft  (1999)  Bayesian Network + Naive Bayes [102 cites]
+  US6421467  Texas Tech (1999)  Vector Quantisation LBG + Product Quantiser [101 cites]
+  US6556983  Microsoft  (2000)  Word Embeddings PMI + Word2Vec SGNS [645 cites]
+"""
+
+__version__ = "0.1.0"
+
+from .genetic_algorithm   import GeneticAlgorithm, Chromosome, make_population
+from .genetic_programming import GeneticProgramming, rand_tree, Var, Const, BinOp, UnaryOp
+from .grammar_gp import (GrammarGP, GrammaticalEvolution, GNode,
+                          rand_grammar_tree, make_math_grammar, MATH_GRAMMAR,
+                          subtree_crossover, subtree_mutation, point_mutation)
+from .linear_gp import (LinearGP, Instruction, Program,
+                         rand_program, effective_program, effective_instructions,
+                         linear_crossover, micro_mutation, macro_mutation,
+                         to_expr_string, execute, OPERATIONS, FAST_OPS)
+from .particle_swarm      import ParticleSwarmOptimiser, DifferentialEvolution, CMAES
+from .online_classifier   import OnlineBayesClassifier
+from .scalable_kmeans     import ScalableKMeans, HierarchicalKMeans
+from .mini_neural_net     import MiniNeuralNet
+from .deep_networks       import Conv1D, SimpleRNN, GRUCell, AntColonyOptimiser
+from .svm_smo             import SVM, linear_kernel, rbf_kernel, poly_kernel
+from .adaboost            import AdaBoost, DecisionStump
+from .q_learning          import QLearner, SARSALearner, FunctionApproxQLearner
+from .multi_armed_bandit  import EpsilonGreedy, UCB1, ThompsonSampling, EXP3, LinUCB
+from .bayesian_optimiser  import GaussianEDA, BayesianOptimiser
+from .simulated_annealing import SimulatedAnnealing, MultiObjectiveSA
+from .decision_tree       import DecisionTree, RandomForest
+from .optimisers          import SGD, RMSProp, Adam, AdamW, LRScheduler, numerical_gradient
+from .hidden_markov       import HiddenMarkovModel
+from .neuroevolution      import EvoNet, WeightEvolution, HybridNNEvolver, ACTIVATIONS
+from .ensemble            import VotingEnsemble, StackingEnsemble, BaggingEnsemble, WeightedEnsemble
+from .knn                 import KNNClassifier, KNNRegressor, FastKNN, BallTree
+from .dbscan              import DBSCAN, OPTICS
+from .gradient_boosting   import GBMRegressor, GBMClassifier
+from .kalman_filter       import KalmanFilter, ExtendedKalmanFilter
+from .gaussian_process    import GaussianProcessRegressor, GaussianProcessClassifier, rbf_kernel as gp_rbf, matern32_kernel, periodic_kernel
+from .spectral_clustering import SpectralClustering, rbf_affinity, knn_affinity
+from .mean_shift          import MeanShift, KernelDensityEstimator, estimate_bandwidth
+from .isolation_forest    import IsolationForest, OneClassSVM
+from .bayesian_network    import BayesianNetwork, NaiveBayes, CPT
+from .vector_quantisation import LBG, AdaptiveVQ, ProductQuantiser
+from .actor_critic        import A2C, PPOLite, LinearActor, LinearCritic
+from .word_embeddings     import Word2Vec, PMIEmbeddings, build_cooccurrence, tokenise
+from .svd_reducer         import randomised_svd, pca as randomised_pca, transform as svd_transform
+from .em_clustering       import EMClustering, GaussianComponent
+from .text_classifier     import TextPipeline, TFIDFVectoriser, NaiveBayesClassifier
+from .collaborative_filter import MemoryCF, BayesianCF
+from .feature_engineering import (
+    PCA, StandardScaler, MinMaxScaler,
+    variance_threshold, correlation_filter,
+    mi_feature_ranking, forward_feature_selection,
+)
+
+__all__ = [
+    # Evolutionary / Global Optimisation
+    "GeneticAlgorithm", "Chromosome", "make_population",
+    "GeneticProgramming", "rand_tree", "Var", "Const", "BinOp", "UnaryOp",
+    # Grammar-Guided GP / Grammatical Evolution
+    "GrammarGP", "GrammaticalEvolution", "GNode",
+    "rand_grammar_tree", "make_math_grammar", "MATH_GRAMMAR",
+    "subtree_crossover", "subtree_mutation", "point_mutation",
+    # Linear GP
+    "LinearGP", "Instruction", "Program",
+    "rand_program", "effective_program", "effective_instructions",
+    "linear_crossover", "micro_mutation", "macro_mutation",
+    "to_expr_string", "execute", "OPERATIONS", "FAST_OPS",
+    "ParticleSwarmOptimiser", "DifferentialEvolution", "CMAES",
+    "GaussianEDA", "BayesianOptimiser",
+    "SimulatedAnnealing", "MultiObjectiveSA",
+    "AntColonyOptimiser",
+    # Neural Networks
+    "MiniNeuralNet",
+    "Conv1D", "SimpleRNN", "GRUCell",
+    "EvoNet", "WeightEvolution", "HybridNNEvolver", "ACTIVATIONS",
+    # Gradient Optimisers
+    "SGD", "RMSProp", "Adam", "AdamW", "LRScheduler", "numerical_gradient",
+    # Classifiers
+    "OnlineBayesClassifier",
+    "SVM", "linear_kernel", "rbf_kernel", "poly_kernel",
+    "AdaBoost", "DecisionStump",
+    "DecisionTree", "RandomForest",
+    # Ensembles
+    "VotingEnsemble", "StackingEnsemble", "BaggingEnsemble", "WeightedEnsemble",
+    # Clustering
+    "ScalableKMeans", "HierarchicalKMeans",
+    # Reinforcement Learning / Bandits
+    "QLearner", "SARSALearner", "FunctionApproxQLearner",
+    "EpsilonGreedy", "UCB1", "ThompsonSampling", "EXP3", "LinUCB",
+    # Sequential Models
+    "HiddenMarkovModel",
+    # Nearest Neighbours
+    "KNNClassifier", "KNNRegressor", "FastKNN", "BallTree",
+    # Density Clustering
+    "DBSCAN", "OPTICS",
+    # Gradient Boosting
+    "GBMRegressor", "GBMClassifier",
+    # Sequential / State Estimation
+    "KalmanFilter", "ExtendedKalmanFilter",
+    # Gaussian Processes
+    "GaussianProcessRegressor", "GaussianProcessClassifier",
+    "gp_rbf", "matern32_kernel", "periodic_kernel",
+    # Graph / Spectral Clustering
+    "SpectralClustering", "rbf_affinity", "knn_affinity",
+    # Density Estimation / Mode Finding
+    "MeanShift", "KernelDensityEstimator", "estimate_bandwidth",
+    # Anomaly Detection
+    "IsolationForest", "OneClassSVM",
+    # Probabilistic Graphical Models
+    "BayesianNetwork", "NaiveBayes", "CPT",
+    # Vector Quantisation
+    "LBG", "AdaptiveVQ", "ProductQuantiser",
+    # Policy Gradient RL
+    "A2C", "PPOLite", "LinearActor", "LinearCritic",
+    # Word Embeddings / NLP
+    "Word2Vec", "PMIEmbeddings", "build_cooccurrence", "tokenise",
+    # Dimensionality Reduction
+    "randomised_svd", "randomised_pca", "svd_transform",
+    # Clustering (additional)
+    "EMClustering", "GaussianComponent",
+    # Text / NLP
+    "TextPipeline", "TFIDFVectoriser", "NaiveBayesClassifier",
+    # Recommender Systems
+    "MemoryCF", "BayesianCF",
+    # Feature Engineering
+    "PCA", "StandardScaler", "MinMaxScaler",
+    "variance_threshold", "correlation_filter",
+    "mi_feature_ranking", "forward_feature_selection",
+]

patentml/actor_critic.py

@@ -0,0 +1,304 @@
+"""
+Actor-Critic and Advantage Actor-Critic (A2C) reinforcement learning.
+
+Patent basis: US6985172 (Southwest Research Institute, filed 2002, expired 2022)
+  "Q-learning with linear reward shaping" — 145 forward citations.
+  Covers concurrent value-function and policy optimisation where a critic
+  baseline reduces variance in policy gradient estimates.
+"""
+
+import math
+import random
+from typing import List, Tuple, Optional, Callable, Dict
+
+
+def _softmax(logits: List[float]) -> List[float]:
+    m = max(logits)
+    exps = [math.exp(x - m) for x in logits]
+    total = sum(exps)
+    return [e / total for e in exps]
+
+def _log_softmax(logits: List[float]) -> List[float]:
+    probs = _softmax(logits)
+    return [math.log(max(p, 1e-300)) for p in probs]
+
+def _relu(x: float) -> float:
+    return max(0.0, x)
+
+def _drelu(x: float) -> float:
+    return 1.0 if x > 0 else 0.0
+
+
+# ── Simple linear actor/critic ─────────────────────────────────────────────────
+
+class LinearActor:
+    """
+    Linear policy: pi(a|s) = softmax(W_a @ s + b_a).
+    """
+
+    def __init__(self, n_states: int, n_actions: int, seed: int = 42):
+        rng = random.Random(seed)
+        scale = 0.1
+        self.W = [[rng.gauss(0, scale) for _ in range(n_states)] for _ in range(n_actions)]
+        self.b = [0.0] * n_actions
+        self.n_states  = n_states
+        self.n_actions = n_actions
+
+    def logits(self, state: List[float]) -> List[float]:
+        return [sum(self.W[a][s] * state[s] for s in range(self.n_states)) + self.b[a]
+                for a in range(self.n_actions)]
+
+    def probs(self, state: List[float]) -> List[float]:
+        return _softmax(self.logits(state))
+
+    def select_action(self, state: List[float], rng: random.Random) -> int:
+        probs = self.probs(state)
+        r = rng.random()
+        cumulative = 0.0
+        for a, p in enumerate(probs):
+            cumulative += p
+            if r <= cumulative:
+                return a
+        return len(probs) - 1
+
+    def update(self, state: List[float], action: int, advantage: float, lr: float) -> None:
+        """Policy gradient update: theta += lr * advantage * grad log pi(a|s)."""
+        probs = self.probs(state)
+        for a in range(self.n_actions):
+            grad = (1.0 - probs[a]) if a == action else -probs[a]
+            for s in range(self.n_states):
+                self.W[a][s] += lr * advantage * grad * state[s]
+            self.b[a] += lr * advantage * grad
+
+
+class LinearCritic:
+    """
+    Linear value function: V(s) = w_v @ s + b_v.
+    Trained by TD(0): V(s) ≈ r + gamma * V(s').
+    """
+
+    def __init__(self, n_states: int, seed: int = 42):
+        rng = random.Random(seed)
+        self.w = [rng.gauss(0, 0.1) for _ in range(n_states)]
+        self.b = 0.0
+        self.n_states = n_states
+
+    def value(self, state: List[float]) -> float:
+        return sum(self.w[i] * state[i] for i in range(self.n_states)) + self.b
+
+    def update(self, state: List[float], target: float, lr: float) -> float:
+        """MSE gradient step. Returns TD error."""
+        v    = self.value(state)
+        td   = target - v
+        for i in range(self.n_states):
+            self.w[i] += lr * td * state[i]
+        self.b += lr * td
+        return td
+
+
+# ── Advantage Actor-Critic (A2C) ───────────────────────────────────────────────
+
+class A2C:
+    """
+    Advantage Actor-Critic (A2C).
+
+    Critic estimates V(s).  Advantage A(s,a) = r + gamma*V(s') - V(s).
+    Actor is updated with policy gradient weighted by advantage.
+    Advantage reduces variance compared to pure REINFORCE.
+
+    Can work with any environment exposing (state, reward, done, info) = step(action).
+    """
+
+    def __init__(
+        self,
+        n_states:      int,
+        n_actions:     int,
+        lr_actor:      float = 0.01,
+        lr_critic:     float = 0.05,
+        gamma:         float = 0.99,
+        entropy_coef:  float = 0.01,
+        seed:          int   = 42,
+    ):
+        self.actor     = LinearActor(n_states, n_actions, seed=seed)
+        self.critic    = LinearCritic(n_states, seed=seed + 1)
+        self.lr_actor  = lr_actor
+        self.lr_critic = lr_critic
+        self.gamma     = gamma
+        self.entropy   = entropy_coef
+        self._rng      = random.Random(seed)
+        self.episode_rewards: List[float] = []
+
+    def select_action(self, state: List[float]) -> int:
+        return self.actor.select_action(state, self._rng)
+
+    def update(
+        self,
+        state:      List[float],
+        action:     int,
+        reward:     float,
+        next_state: List[float],
+        done:       bool,
+    ) -> float:
+        """Single-step TD update. Returns advantage."""
+        v_next    = 0.0 if done else self.critic.value(next_state)
+        td_target = reward + self.gamma * v_next
+        advantage = self.critic.update(state, td_target, self.lr_critic)
+        # Entropy regularisation: add small gradient toward uniform policy
+        probs = self.actor.probs(state)
+        for a in range(self.actor.n_actions):
+            entropy_grad = -math.log(max(probs[a], 1e-9)) - 1
+            self.actor.b[a] += self.lr_actor * self.entropy * entropy_grad
+        # Policy gradient
+        self.actor.update(state, action, advantage, self.lr_actor)
+        return advantage
+
+    def train(
+        self,
+        env_fn: Callable,
+        n_episodes: int = 500,
+        max_steps:  int = 200,
+    ) -> List[float]:
+        """
+        Train for n_episodes.
+
+        env_fn() must return an object with:
+          .reset() -> state (list of floats)
+          .step(action) -> (next_state, reward, done)
+        """
+        self.episode_rewards = []
+        for ep in range(n_episodes):
+            env   = env_fn()
+            state = env.reset()
+            total = 0.0
+            for _ in range(max_steps):
+                action = self.select_action(state)
+                next_state, reward, done = env.step(action)
+                self.update(state, action, reward, next_state, done)
+                total += reward
+                state  = next_state
+                if done: break
+            self.episode_rewards.append(total)
+        return self.episode_rewards
+
+
+# ── PPO-lite (clip-based policy optimisation) ──────────────────────────────────
+
+class PPOLite:
+    """
+    Proximal Policy Optimisation (lite version) — clipped surrogate objective.
+
+    Collects a batch of transitions, computes advantages, then does k_epochs
+    of gradient updates with clipping to prevent too-large policy updates.
+    """
+
+    def __init__(
+        self,
+        n_states:  int,
+        n_actions: int,
+        lr:        float = 0.003,
+        gamma:     float = 0.99,
+        clip_eps:  float = 0.2,
+        k_epochs:  int   = 4,
+        seed:      int   = 42,
+    ):
+        self.actor  = LinearActor(n_states, n_actions, seed=seed)
+        self.critic = LinearCritic(n_states, seed=seed + 1)
+        self.lr       = lr
+        self.gamma    = gamma
+        self.clip_eps = clip_eps
+        self.k_epochs = k_epochs
+        self._rng     = random.Random(seed)
+        # Replay buffer
+        self._states:      List[List[float]] = []
+        self._actions:     List[int]         = []
+        self._rewards:     List[float]       = []
+        self._dones:       List[bool]        = []
+        self._old_logprobs: List[float]      = []
+
+    def select_action(self, state: List[float]) -> Tuple[int, float]:
+        probs    = self.actor.probs(state)
+        action   = self.actor.select_action(state, self._rng)
+        log_prob = math.log(max(probs[action], 1e-300))
+        return action, log_prob
+
+    def store(self, state, action, reward, done, log_prob):
+        self._states.append(state)
+        self._actions.append(action)
+        self._rewards.append(reward)
+        self._dones.append(done)
+        self._old_logprobs.append(log_prob)
+
+    def update(self) -> None:
+        """Compute returns, advantages, then run k_epochs of PPO clip update."""
+        n = len(self._rewards)
+        if n == 0: return
+        # Compute discounted returns
+        returns = [0.0] * n
+        running = 0.0
+        for i in reversed(range(n)):
+            if self._dones[i]: running = 0.0
+            running = self._rewards[i] + self.gamma * running
+            returns[i] = running
+
+        for _ in range(self.k_epochs):
+            for i in range(n):
+                state  = self._states[i]
+                action = self._actions[i]
+                ret    = returns[i]
+                v      = self.critic.value(state)
+                adv    = ret - v
+                # Normalise advantage
+                adv /= (abs(adv) + 1.0)
+                # Critic update
+                self.critic.update(state, ret, self.lr)
+                # Actor: clipped surrogate
+                new_probs   = self.actor.probs(state)
+                new_logprob = math.log(max(new_probs[action], 1e-300))
+                old_logprob = self._old_logprobs[i]
+                ratio       = math.exp(new_logprob - old_logprob)
+                ratio_clip  = max(1 - self.clip_eps, min(1 + self.clip_eps, ratio))
+                # Gradient from min(ratio*adv, clipped*adv)
+                if adv >= 0:
+                    effective_ratio = min(ratio, 1 + self.clip_eps)
+                else:
+                    effective_ratio = max(ratio, 1 - self.clip_eps)
+                self.actor.update(state, action, adv * effective_ratio / (ratio + 1e-9), self.lr)
+
+        # Clear buffer
+        self._states.clear(); self._actions.clear(); self._rewards.clear()
+        self._dones.clear(); self._old_logprobs.clear()
+
+
+if __name__ == "__main__":
+    # Simple environment: pole balancing proxy via CartPole-like reward signal
+    class GridWalk:
+        """1D grid: state=[pos], actions=[left,right], goal at pos=5."""
+        def __init__(self): self.pos = 2
+        def reset(self): self.pos = 2; return [self.pos / 10.0]
+        def step(self, action):
+            self.pos += 1 if action == 1 else -1
+            self.pos = max(0, min(9, self.pos))
+            done = self.pos == 5
+            reward = 1.0 if done else -0.01
+            return [self.pos / 10.0], reward, done
+
+    a2c = A2C(n_states=1, n_actions=2, lr_actor=0.05, lr_critic=0.1, gamma=0.99, seed=0)
+    rewards = a2c.train(GridWalk, n_episodes=300, max_steps=50)
+    last50 = sum(rewards[-50:]) / 50
+    print(f"A2C GridWalk: last-50 mean reward = {last50:.3f} (expect > 0.8)")
+
+    ppo = PPOLite(n_states=1, n_actions=2, lr=0.05, gamma=0.99, seed=1)
+    rng = random.Random(42)
+    ep_rewards = []
+    for ep in range(200):
+        env = GridWalk(); state = env.reset(); total = 0.0
+        for _ in range(30):
+            action, lp = ppo.select_action(state)
+            ns, r, done = env.step(action)
+            ppo.store(state, action, r, done, lp)
+            total += r; state = ns
+            if done: break
+        ep_rewards.append(total)
+        if (ep + 1) % 10 == 0: ppo.update()
+    last50_ppo = sum(ep_rewards[-50:]) / 50
+    print(f"PPO-lite GridWalk: last-50 mean reward = {last50_ppo:.3f} (expect > 0.8)")

patentml/adaboost.py

@@ -0,0 +1,168 @@
+"""
+AdaBoost — Adaptive Boosting with Feature Selection
+Public Domain Implementation
+
+Derived from expired patents:
+  US7421415 (Siemens Corporate Research, filed 2005)
+  "Methods and systems for 3D object detection using learning"
+  "using a learning procedure for feature selection based on boosting"
+Expired 2025. Public domain.
+
+Also draws from:
+  US6816847 (Microsoft, filed 1999, 72 cites) — ensemble of classifiers
+  "training a classifier comprises training one of a bayesian classifier,
+   a support vector machine, a neural net classifier, and a decision tree"
+
+AdaBoost: Freund & Schapire 1997 — theoretical basis, Siemens patent
+covers the application to feature selection in high-dimensional spaces.
+"""
+import math
+import random
+from typing import Callable, List, Optional, Tuple
+
+
+class DecisionStump:
+    """
+    Weak learner: single-feature threshold classifier.
+    US7421415: "a weak learning algorithm applied at each boosting round"
+    """
+    def __init__(self):
+        self.feature_idx: int = 0
+        self.threshold:   float = 0.0
+        self.polarity:    int = 1    # 1 or -1
+        self.alpha:       float = 0.0
+
+    def fit(self, X: List[List[float]], y: List[int],
+            weights: List[float]) -> float:
+        """Find best feature+threshold minimising weighted error."""
+        n_features = len(X[0])
+        best_error = float("inf")
+
+        for fi in range(n_features):
+            vals = sorted(set(x[fi] for x in X))
+            thresholds = [(vals[i] + vals[i+1]) / 2
+                         for i in range(len(vals) - 1)]
+            if not thresholds:
+                thresholds = [vals[0]]
+
+            for thresh in thresholds:
+                for polarity in (1, -1):
+                    preds = [polarity if x[fi] <= thresh else -polarity
+                             for x in X]
+                    error = sum(w for w, p, yi in zip(weights, preds, y)
+                               if p != yi)
+                    if error < best_error:
+                        best_error = error
+                        self.feature_idx = fi
+                        self.threshold   = thresh
+                        self.polarity    = polarity
+
+        return best_error
+
+    def predict(self, X: List[List[float]]) -> List[int]:
+        return [self.polarity if x[self.feature_idx] <= self.threshold
+                else -self.polarity for x in X]
+
+
+class AdaBoost:
+    """
+    Adaptive Boosting with feature selection.
+
+    US7421415: "for each boosting round t:
+      - apply weak learning algorithm to find feature with minimum error
+      - compute alpha_t = 0.5 * ln((1-e_t)/e_t)
+      - update weights: w_{t+1} = w_t * exp(-alpha_t * y_i * h_t(x_i))
+      - normalise weights"
+
+    The ensemble is: H(x) = sign(sum_t alpha_t * h_t(x))
+    """
+
+    def __init__(
+        self,
+        n_estimators: int = 50,
+        random_seed: Optional[int] = None,
+    ):
+        self.T = n_estimators
+        self.stumps: List[DecisionStump] = []
+        if random_seed:
+            random.seed(random_seed)
+
+    def fit(self, X: List[List[float]], y: List[int]) -> "AdaBoost":
+        """
+        US7421415 boosting loop:
+        "for each boosting round, train weak learner on weighted data"
+        """
+        n = len(X)
+        weights = [1.0 / n] * n
+
+        self.stumps = []
+        self.feature_importance = [0.0] * len(X[0])
+
+        for t in range(self.T):
+            stump = DecisionStump()
+            error = stump.fit(X, y, weights)
+
+            # Clip error to avoid log(0)
+            error = max(1e-10, min(1 - 1e-10, error))
+
+            # Alpha: confidence of this stump
+            stump.alpha = 0.5 * math.log((1 - error) / error)
+
+            # Track feature importance (US7421415 feature selection)
+            self.feature_importance[stump.feature_idx] += abs(stump.alpha)
+
+            # Update weights: higher weight for misclassified examples
+            preds = stump.predict(X)
+            new_weights = [
+                w * math.exp(-stump.alpha * yi * pi)
+                for w, yi, pi in zip(weights, y, preds)
+            ]
+            total = sum(new_weights)
+            weights = [w / total for w in new_weights]
+
+            self.stumps.append(stump)
+
+        # Normalise feature importance
+        total_imp = sum(self.feature_importance) or 1.0
+        self.feature_importance = [f / total_imp for f in self.feature_importance]
+
+        return self
+
+    def predict_score(self, X: List[List[float]]) -> List[float]:
+        """Return raw ensemble scores (positive = class +1)."""
+        scores = [0.0] * len(X)
+        for stump in self.stumps:
+            preds = stump.predict(X)
+            for i, p in enumerate(preds):
+                scores[i] += stump.alpha * p
+        return scores
+
+    def predict(self, X: List[List[float]]) -> List[int]:
+        return [1 if s >= 0 else -1 for s in self.predict_score(X)]
+
+    def top_features(self, n: int = 5) -> List[Tuple[int, float]]:
+        """Return (feature_index, importance) sorted by importance."""
+        ranked = sorted(enumerate(self.feature_importance),
+                       key=lambda x: -x[1])
+        return ranked[:n]
+
+
+if __name__ == "__main__":
+    random.seed(42)
+
+    # Generate linearly separable data with irrelevant features
+    X = [[x1, x2, random.random(), random.random()]
+         for x1 in [0.0, 0.5, 1.0, 1.5, 2.0]
+         for x2 in [0.0, 0.5, 1.0, 1.5, 2.0]]
+    y = [1 if x[0] + x[1] > 2.0 else -1 for x in X]
+
+    ab = AdaBoost(n_estimators=20, random_seed=42)
+    ab.fit(X, y)
+    preds = ab.predict(X)
+    acc = sum(1 for p, yi in zip(preds, y) if p == yi) / len(y)
+
+    print("AdaBoost with feature selection (US7421415, Siemens, expired 2025):")
+    print(f"  Accuracy: {acc:.1%}")
+    print(f"  Feature importance: {[round(f, 3) for f in ab.feature_importance]}")
+    print(f"  Top features: {ab.top_features(2)}")
+    print(f"  (Features 0+1 should dominate; 2+3 are noise)")