chimeraboost 0.5.2__py3-none-any.whl

chimeraboost/__init__.py

@@ -0,0 +1,23 @@
+"""ChimeraBoost: a CatBoost-inspired gradient boosting library in pure Python.
+
+Key ingredients borrowed from CatBoost:
+  * Ordered target statistics for categorical features (anti-leakage encoding)
+  * Oblivious / symmetric trees (fast, strongly regularized -> good defaults)
+  * Histogram-based quantized splitting (numba accelerated)
+
+Public API:
+  >>> from chimeraboost import ChimeraBoostRegressor, ChimeraBoostClassifier
+  >>> model = ChimeraBoostClassifier().fit(X, y, cat_features=[0, 3])
+  >>> proba = model.predict_proba(X_test)
+"""
+
+from .sklearn_api import (
+    ChimeraBoostRegressor,
+    ChimeraBoostClassifier,
+)
+
+__all__ = [
+    "ChimeraBoostRegressor",
+    "ChimeraBoostClassifier",
+]
+__version__ = "0.4.0"

chimeraboost/binning.py

@@ -0,0 +1,69 @@
+"""Quantization of numeric features into integer bins.
+
+Borders are learned once on the training data (quantile based). Every feature
+is mapped to a small integer bin index, which is what the tree builder consumes.
+NaNs are routed to a dedicated bin so a split can isolate missing values, the
+way CatBoost/LightGBM do.
+
+Bin layout per feature:
+    real values -> 0 .. n_borders        (via searchsorted on borders)
+    NaN         -> n_borders + 1          (the highest bin, "missing")
+The histogram width for a feature is therefore (n_borders + 2).
+"""
+
+import numpy as np
+
+BIN_DTYPE = np.uint16
+
+
+def _feature_borders(col, max_bins):
+    """Quantile borders for one numeric column, ignoring NaNs."""
+    finite = col[np.isfinite(col)]
+    if finite.size == 0:
+        return np.array([], dtype=np.float64)
+    uniq = np.unique(finite)
+    if uniq.size <= max_bins:
+        # Few distinct values: put a border between each pair.
+        return ((uniq[:-1] + uniq[1:]) / 2.0).astype(np.float64)
+    qs = np.linspace(0.0, 1.0, max_bins + 1)[1:-1]
+    borders = np.quantile(finite, qs)
+    return np.unique(borders).astype(np.float64)
+
+
+class Binner:
+    """Learns per-feature borders and maps a float matrix to bins."""
+
+    def __init__(self, max_bins=128):
+        self.max_bins = int(max_bins)
+        self.borders_ = None       # list of np.ndarray, one per feature
+        self.n_bins_ = None        # np.ndarray int, width per feature
+
+    def fit(self, X):
+        """Learn quantile borders for each column from training data."""
+        X = np.asarray(X, dtype=np.float64)
+        n_features = X.shape[1]
+        self.borders_ = [
+            _feature_borders(X[:, f], self.max_bins) for f in range(n_features)
+        ]
+        # +1 for the searchsorted upper bucket, +1 for the NaN bucket.
+        self.n_bins_ = np.array(
+            [len(b) + 2 for b in self.borders_], dtype=np.int64
+        )
+        return self
+
+    def transform(self, X):
+        """Map a float matrix to integer bin indices; NaNs go to the top bin."""
+        X = np.asarray(X, dtype=np.float64)
+        n_samples, n_features = X.shape
+        out = np.empty((n_samples, n_features), dtype=BIN_DTYPE)
+        for f in range(n_features):
+            col = X[:, f]
+            borders = self.borders_[f]
+            nan_bin = len(borders) + 1
+            binned = np.searchsorted(borders, col, side="right").astype(BIN_DTYPE)
+            binned[~np.isfinite(col)] = nan_bin
+            out[:, f] = binned
+        return out
+
+    def fit_transform(self, X):
+        return self.fit(X).transform(X)

chimeraboost/booster.py

@@ -0,0 +1,408 @@
+"""The gradient boosting core: builds the full additive model.
+
+Two boosters share the same machinery (FeaturePreprocessor, oblivious trees):
+  * GradientBoosting     -> scalar output (regression, binary classification)
+  * MulticlassBoosting   -> K simultaneous outputs (softmax multiclass)
+"""
+
+import time
+import numpy as np
+
+from .losses import LOSSES, MultiSoftmax
+from .preprocessing import FeaturePreprocessor
+from .tree import build_oblivious_tree
+
+
+def _apply_thread_count(thread_count):
+    """Set numba's thread pool size. None / -1 means use all detected cores.
+
+    Returns the effective thread count so callers can record it.
+    """
+    import numba
+    max_threads = numba.config.NUMBA_NUM_THREADS
+    if thread_count is None or thread_count < 0:
+        n = max_threads
+    else:
+        n = max(1, min(int(thread_count), max_threads))
+    numba.set_num_threads(n)
+    return n
+
+
+def _auto_learning_rate(n_samples, iterations, early_stopping):
+    """Pick a default learning rate when the user did not specify one.
+
+    With early stopping, we default to 0.05 (down from 0.1). This forces the 
+    model to take smaller steps and build a larger, smoother ensemble. We trade 
+    a bit of our massive speed advantage for better test generalization.
+    Without early stopping, the rate scales inversely with the iteration budget.
+    """
+    if early_stopping:
+        return 0.1
+    lr = 20.0 / max(iterations, 1)
+    return float(np.clip(lr, 0.03, 0.2))
+
+
+class _BaseBooster:
+    """Shared machinery for the scalar and multiclass boosters.
+
+    Holds the common hyperparameters and the helpers both subclasses use:
+    histogram-buffer allocation, column subsampling, row subsampling, feature
+    preprocessing, and split-gain feature importances. Subclasses implement
+    `fit` and `predict_raw`.
+    """
+
+    def __init__(self, iterations=500, learning_rate=None, depth=6,
+                 l2_leaf_reg=3.0, max_bins=128, subsample=1.0,
+                 colsample=1.0, cat_smoothing=1.0, early_stopping_rounds=None,
+                 min_child_weight=1.0, thread_count=None, random_state=None,
+                 verbose=False, ordered_boosting=True):
+        self.iterations = int(iterations)
+        self.learning_rate = learning_rate
+        self.depth = int(depth)
+        self.l2_leaf_reg = float(l2_leaf_reg)
+        self.max_bins = int(max_bins)
+        self.subsample = float(subsample)
+        self.colsample = float(colsample)
+        self.cat_smoothing = float(cat_smoothing)
+        self.early_stopping_rounds = early_stopping_rounds
+        self.min_child_weight = float(min_child_weight)
+        self.thread_count = thread_count
+        self.random_state = random_state
+        self.verbose = verbose
+        self.ordered_boosting = bool(ordered_boosting)
+
+    def _alloc_hist_buffers(self, n_features, n_bins):
+        """Allocate reusable histogram buffers once per fit.
+
+        Shape (n_features, 2**depth, max_bins). Reused for every tree and level
+        via _build_histograms_into, which zeroes the active slice each call.
+        This avoids reallocating these (potentially large) arrays thousands of
+        times over a long boosting run.
+        """
+        max_leaves = 1 << self.depth
+        max_bins = int(n_bins.max()) if len(n_bins) else 1
+        hg = np.zeros((n_features, max_leaves, max_bins))
+        hh = np.zeros((n_features, max_leaves, max_bins))
+        return (hg, hh)
+
+    def _feature_mask(self, n_cols, rng):
+        """0/1 mask selecting a random subset of columns for one tree."""
+        if self.colsample >= 1.0:
+            return None
+        k = max(1, int(round(self.colsample * n_cols)))
+        mask = np.zeros(n_cols, dtype=np.int64)
+        mask[rng.choice(n_cols, size=k, replace=False)] = 1
+        return mask
+
+    def _new_preprocessor(self):
+        """Build a FeaturePreprocessor configured from this booster's params."""
+        return FeaturePreprocessor(self.max_bins, self.cat_smoothing,
+                                   self.random_state)
+
+    def _maybe_subsample(self, grad, hess, rng):
+        """Stochastic row subsampling: zero out the gradient/hessian of rows not
+        in this tree's sample. Zeroed rows contribute nothing to the histograms
+        but are still routed to leaves, as in standard stochastic GBDT."""
+        if self.subsample >= 1.0:
+            return grad, hess
+        mask = rng.random(grad.shape[0]) < self.subsample
+        return np.where(mask, grad, 0.0), np.where(mask, hess, 0.0)
+
+    def _accumulate_importance(self, tree):
+        """Add this tree's per-split gains to the running importance totals,
+        mapped from internal columns back to original input features."""
+        for f, g in zip(tree.splits_feat, tree.gains):
+            orig = self.prep_.feature_map_[f]
+            self._importance[orig] += g
+
+    @property
+    def feature_importances_(self):
+        """Total split gain per ORIGINAL input column, normalized to sum 1."""
+        imp = self._importance.copy()
+        s = imp.sum()
+        return imp / s if s > 0 else imp
+
+
+class GradientBoosting(_BaseBooster):
+    """Scalar booster: regression and binary classification."""
+
+    def __init__(self, loss="RMSE", loss_kwargs=None, **kw):
+        super().__init__(**kw)
+        self.loss_name = loss
+        self.loss_kwargs = loss_kwargs or {}
+
+    def fit(self, X, y, cat_features=None, eval_set=None, sample_weight=None):
+        """Fit the additive model. Optionally pass `cat_features` (column indices
+        to target-encode) and `eval_set=(X_val, y_val)` for early stopping.
+        `sample_weight` is a 1-D array of per-sample weights; None means uniform.
+        Weights are normalized to mean 1 internally so the gradient scale stays
+        comparable to the no-weight case."""
+        X = (np.asarray(X, dtype=object) if cat_features
+             else np.asarray(X, dtype=np.float64))
+        y = np.asarray(y, dtype=np.float64)
+        n_samples = X.shape[0]
+
+        # Normalize weights to mean=1. np.ones(n) stays np.ones(n), so
+        # sample_weight=np.ones(n) is bitwise-equivalent to sample_weight=None
+        # for all losses except MAE/Quantile (which use a different quantile
+        # algorithm when weights are present).
+        w = None
+        if sample_weight is not None:
+            w = np.asarray(sample_weight, dtype=np.float64)
+            w = w * (n_samples / w.sum())
+
+        self.n_threads_ = _apply_thread_count(self.thread_count)
+        self.loss_ = LOSSES[self.loss_name](**self.loss_kwargs)
+        _es = self.early_stopping_rounds is not None and eval_set is not None
+        self.lr_ = (self.learning_rate if self.learning_rate is not None
+                    else _auto_learning_rate(n_samples, self.iterations, _es))
+
+        self.prep_ = self._new_preprocessor()
+        X_binned = self.prep_.fit_transform(X, [y], cat_features)
+        n_bins = self.prep_.n_bins_
+        hist_buffers = self._alloc_hist_buffers(X_binned.shape[1], n_bins)
+        self._importance = np.zeros(self.prep_.n_input_features_)
+
+        Xv_binned = yv = Fv = None
+        if eval_set is not None:
+            Xv, yv = eval_set
+            Xv = (np.asarray(Xv, dtype=object) if cat_features
+                  else np.asarray(Xv, dtype=np.float64))
+            yv = np.asarray(yv, dtype=np.float64)
+            Xv_binned = self.prep_.transform(Xv)
+
+        self.init_ = self.loss_.init(y, w)
+        F = np.full(n_samples, self.init_, dtype=np.float64)
+        if yv is not None:
+            Fv = np.full(len(yv), self.init_)
+
+        rng = np.random.default_rng(self.random_state)
+        self.trees_ = []
+        self.train_history_, self.valid_history_ = [], []
+        best_score, best_iter = np.inf, 0
+        t0 = time.time()
+
+        for m in range(self.iterations):
+            grad, hess = self.loss_.grad_hess(y, F)
+            if w is not None:
+                grad = grad * w
+                hess = hess * w
+            g, h = self._maybe_subsample(grad, hess, rng)
+            fmask = self._feature_mask(X_binned.shape[1], rng)
+            tree = build_oblivious_tree(X_binned, g, h, n_bins, self.depth,
+                                        self.l2_leaf_reg, self.lr_,
+                                        feature_mask=fmask,
+                                        min_child_weight=self.min_child_weight,
+                                        hist_buffers=hist_buffers)
+            # A depth-0 tree found no legal split; subsequent rounds on the same
+            # gradients would too, so stop rather than append empty trees.
+            if tree.depth == 0:
+                if self.verbose:
+                    print(f"No further splits at iteration {m}; stopping.")
+                break
+            if getattr(self.loss_, "adjusts_leaves", False):
+                self._correct_leaves(tree, X_binned, y - F, w)
+            self.trees_.append(tree)
+            self._accumulate_importance(tree)
+            if self.ordered_boosting and not getattr(self.loss_, "adjusts_leaves", False):
+                # Leave-one-out leaf step: each row's update uses its leaf's
+                # gradient/hessian totals with that row's own contribution
+                # removed, reducing the self-reinforcement of plain boosting.
+                # tree.values keeps the standard Newton values for inference;
+                # only the training F uses this corrected update. Subsampled-out
+                # rows (g=h=0) fall back to the standard leaf value.
+                leaf = tree.apply(X_binned)
+                n_lv = tree.values.shape[0]
+                leaf_G = np.bincount(leaf, weights=g, minlength=n_lv)
+                leaf_H = np.bincount(leaf, weights=h, minlength=n_lv)
+                F += -self.lr_ * (leaf_G[leaf] - g) / (
+                    np.maximum(leaf_H[leaf] - h, 0.0) + self.l2_leaf_reg)
+            else:
+                F += tree.predict(X_binned)
+            self.train_history_.append(self.loss_.eval(y, F, w))
+
+            if Fv is not None:
+                Fv += tree.predict(Xv_binned)
+                val = self.loss_.eval(yv, Fv)   # validation is always unweighted
+                self.valid_history_.append(val)
+                if val < best_score - 1e-9:
+                    best_score, best_iter = val, m
+                elif (self.early_stopping_rounds and
+                      m - best_iter >= self.early_stopping_rounds):
+                    if self.verbose:
+                        print(f"Early stop at {m} (best {best_iter}, "
+                              f"val {best_score:.5f})")
+                    self.trees_ = self.trees_[: best_iter + 1]
+                    break
+
+            if self.verbose and (m % max(1, self.iterations // 10) == 0):
+                msg = f"[{m}] train {self.train_history_[-1]:.5f}"
+                if Fv is not None:
+                    msg += f"  val {self.valid_history_[-1]:.5f}"
+                print(msg)
+
+        self.fit_time_ = time.time() - t0
+        self.best_iteration_ = len(self.trees_)
+        return self
+
+    def _correct_leaves(self, tree, X_binned, residuals, sample_weight=None):
+        """Override Newton leaf values with the loss-appropriate residual
+        statistic (median for MAE, alpha-quantile for Quantile). The tree
+        structure was chosen by the gradient; this fixes the step size."""
+        leaf = tree.apply(X_binned)
+        n_leaves = tree.values.shape[0]
+        for l in range(n_leaves):
+            mask = leaf == l
+            r = residuals[mask]
+            w = sample_weight[mask] if sample_weight is not None else None
+            tree.values[l] = self.lr_ * self.loss_.leaf_value(r, w)
+
+    def predict_raw(self, X):
+        """Return raw additive scores (pre-link): the regression prediction, or
+        the log-odds for binary classification."""
+        X = (np.asarray(X, dtype=object) if self.prep_.cat_features_
+             else np.asarray(X, dtype=np.float64))
+        X_binned = self.prep_.transform(X)
+        F = np.full(X_binned.shape[0], self.init_, dtype=np.float64)
+        for tree in self.trees_:
+            F += tree.predict(X_binned)
+        return F
+
+    def staged_predict_raw(self, X):
+        """Yield the cumulative raw prediction after each tree (1..n_trees)."""
+        X = (np.asarray(X, dtype=object) if self.prep_.cat_features_
+             else np.asarray(X, dtype=np.float64))
+        X_binned = self.prep_.transform(X)
+        F = np.full(X_binned.shape[0], self.init_, dtype=np.float64)
+        for tree in self.trees_:
+            F += tree.predict(X_binned)
+            yield F.copy()
+
+
+class MulticlassBoosting(_BaseBooster):
+    """Softmax multiclass booster: fits K trees per round (one per class)."""
+
+    def fit(self, X, y, cat_features=None, eval_set=None, sample_weight=None):
+        """Fit K trees per boosting round (one per class) under softmax loss.
+        Same `cat_features` / `eval_set` / `sample_weight` semantics as the
+        scalar booster."""
+        X = (np.asarray(X, dtype=object) if cat_features
+             else np.asarray(X, dtype=np.float64))
+        y = np.asarray(y)
+        self.classes_ = np.unique(y)
+        K = self.classes_.size
+        self.n_classes_ = K
+        y_idx = np.searchsorted(self.classes_, y)
+        Y = np.eye(K)[y_idx]                      # one-hot (n, K)
+        n_samples = X.shape[0]
+
+        w = None
+        if sample_weight is not None:
+            w = np.asarray(sample_weight, dtype=np.float64)
+            w = w * (n_samples / w.sum())
+
+        self.n_threads_ = _apply_thread_count(self.thread_count)
+        self.loss_ = MultiSoftmax(K)
+        _es = self.early_stopping_rounds is not None and eval_set is not None
+        self.lr_ = (self.learning_rate if self.learning_rate is not None
+                    else _auto_learning_rate(n_samples, self.iterations, _es))
+
+        # One ordered-TS target per class (CatBoost-style per-class statistics).
+        self.prep_ = self._new_preprocessor()
+        X_binned = self.prep_.fit_transform(X, [Y[:, k] for k in range(K)],
+                                            cat_features)
+        n_bins = self.prep_.n_bins_
+        hist_buffers = self._alloc_hist_buffers(X_binned.shape[1], n_bins)
+        self._importance = np.zeros(self.prep_.n_input_features_)
+
+        Xv_binned = Yv = Fv = yv_idx = None
+        if eval_set is not None:
+            Xv, yv = eval_set
+            Xv = (np.asarray(Xv, dtype=object) if cat_features
+                  else np.asarray(Xv, dtype=np.float64))
+            yv_idx = np.searchsorted(self.classes_, np.asarray(yv))
+            Yv = np.eye(K)[yv_idx]
+            Xv_binned = self.prep_.transform(Xv)
+
+        self.init_ = self.loss_.init(Y, w)         # (K,)
+        F = np.tile(self.init_, (n_samples, 1))    # (n, K)
+        if Yv is not None:
+            Fv = np.tile(self.init_, (len(yv_idx), 1))
+
+        rng = np.random.default_rng(self.random_state)
+        self.trees_ = []                           # list of rounds; each = K trees
+        self.train_history_, self.valid_history_ = [], []
+        best_score, best_iter = np.inf, 0
+        t0 = time.time()
+
+        for m in range(self.iterations):
+            grad, hess = self.loss_.grad_hess(Y, F)   # (n, K) each
+            if w is not None:
+                grad = grad * w[:, None]
+                hess = hess * w[:, None]
+            fmask = self._feature_mask(X_binned.shape[1], rng)
+            round_trees = []
+            for k in range(K):
+                g, h = self._maybe_subsample(np.ascontiguousarray(grad[:, k]),
+                                             np.ascontiguousarray(hess[:, k]), rng)
+                tree = build_oblivious_tree(X_binned, g, h, n_bins, self.depth,
+                                            self.l2_leaf_reg, self.lr_,
+                                            feature_mask=fmask,
+                                            min_child_weight=self.min_child_weight,
+                                            hist_buffers=hist_buffers)
+                round_trees.append(tree)
+                self._accumulate_importance(tree)
+                if self.ordered_boosting and tree.depth > 0:
+                    leaf = tree.apply(X_binned)
+                    n_lv = tree.values.shape[0]
+                    leaf_G = np.bincount(leaf, weights=g, minlength=n_lv)
+                    leaf_H = np.bincount(leaf, weights=h, minlength=n_lv)
+                    F[:, k] += -self.lr_ * (leaf_G[leaf] - g) / (
+                        np.maximum(leaf_H[leaf] - h, 0.0) + self.l2_leaf_reg)
+                else:
+                    F[:, k] += tree.predict(X_binned)
+            # Stop only if EVERY class exhausted its splits this round; if even
+            # one class is still learning, the round was productive.
+            if all(t.depth == 0 for t in round_trees):
+                if self.verbose:
+                    print(f"No further splits for any class at iteration {m}; "
+                          f"stopping.")
+                break
+            self.trees_.append(round_trees)
+            self.train_history_.append(self.loss_.eval(Y, F, w))
+
+            if Fv is not None:
+                for k in range(K):
+                    Fv[:, k] += round_trees[k].predict(Xv_binned)
+                val = self.loss_.eval(Yv, Fv)   # validation is always unweighted
+                self.valid_history_.append(val)
+                if val < best_score - 1e-9:
+                    best_score, best_iter = val, m
+                elif (self.early_stopping_rounds and
+                      m - best_iter >= self.early_stopping_rounds):
+                    if self.verbose:
+                        print(f"Early stop at {m} (best {best_iter})")
+                    self.trees_ = self.trees_[: best_iter + 1]
+                    break
+
+            if self.verbose and (m % max(1, self.iterations // 10) == 0):
+                msg = f"[{m}] train {self.train_history_[-1]:.5f}"
+                if Fv is not None:
+                    msg += f"  val {self.valid_history_[-1]:.5f}"
+                print(msg)
+
+        self.fit_time_ = time.time() - t0
+        self.best_iteration_ = len(self.trees_)
+        return self
+
+    def predict_raw(self, X):
+        """Return the (n_samples, n_classes) matrix of raw per-class scores
+        (pre-softmax)."""
+        X = (np.asarray(X, dtype=object) if self.prep_.cat_features_
+             else np.asarray(X, dtype=np.float64))
+        X_binned = self.prep_.transform(X)
+        F = np.tile(self.init_, (X_binned.shape[0], 1))
+        for round_trees in self.trees_:
+            for k in range(self.n_classes_):
+                F[:, k] += round_trees[k].predict(X_binned)
+        return F