tide-binning 0.6.0__py3-none-any.whl

tide/__init__.py

@@ -0,0 +1,10 @@
+"""TIDE - Time Invariant Discretization Engine.
+
+Time-stable binning of mixed-type features for credit scorecards.
+"""
+
+from ._estimator import TIDE
+
+__version__ = "0.6.0"
+
+__all__ = ["TIDE", "__version__"]

tide/_bins.py

@@ -0,0 +1,178 @@
+"""The bins format: validation, row assignment and labels.
+
+A bins dict looks like::
+
+    {"trend": "pos",
+     "bins": {0: {(-np.inf, 0.34): "cont", "special_category": "cat"},
+              1: {(0.34, 0.65): "cont"},
+              2: {(0.65, np.inf): "cont"},
+              3: {None: "missing"}}}
+
+Continuous intervals are ``(a, b]`` for ``trend="pos"`` and ``[a, b)`` for
+``trend="neg"``. Infinite bounds are always inclusive. ``None`` is the key for
+NaN / None / pd.NA / NaT.
+"""
+
+from __future__ import annotations
+
+import numbers
+from dataclasses import dataclass
+
+import numpy as np
+
+from ._parse import ParsedColumn, normalize_key
+
+__all__ = ["CompiledBins", "bin_label", "compile_bins", "special_keys", "validate_bins"]
+
+ATOM_TYPES = ("cont", "cat", "missing")
+
+
+def validate_bins(bins: dict, feature: str) -> dict:
+    """Check a user bins dict and return a normalized copy."""
+    where = f"user_bins[{feature!r}]"
+    if not isinstance(bins, dict) or "bins" not in bins or not isinstance(bins["bins"], dict):
+        raise ValueError(f"{where} must be a dict with a 'bins' dict")
+    if not bins["bins"]:
+        raise ValueError(f"{where} has no bins")
+    out: dict = {}
+    has_cont = False
+    for idx, atoms in bins["bins"].items():
+        if not isinstance(idx, (int, np.integer)) or isinstance(idx, bool):
+            raise ValueError(f"{where}: bin index {idx!r} must be an int")
+        if not isinstance(atoms, dict) or not atoms:
+            raise ValueError(f"{where}: bin {idx} must be a non-empty dict of atom -> type")
+        norm = {}
+        for atom, kind in atoms.items():
+            if kind not in ATOM_TYPES:
+                raise ValueError(f"{where}: bin {idx} has atom type {kind!r}; use {ATOM_TYPES}")
+            if kind == "cont":
+                ok = (
+                    isinstance(atom, tuple)
+                    and len(atom) == 2
+                    and all(isinstance(v, numbers.Real) and not isinstance(v, bool) for v in atom)
+                    and not np.isnan(atom[0])
+                    and not np.isnan(atom[1])
+                )
+                if not ok or not atom[0] < atom[1]:
+                    raise ValueError(
+                        f"{where}: bin {idx} has invalid interval {atom!r}; use (a, b) with a < b"
+                    )
+                has_cont = True
+                norm[(float(atom[0]), float(atom[1]))] = "cont"
+            else:
+                norm[normalize_key(atom)] = kind
+        out[int(idx)] = norm
+    trend = bins.get("trend")
+    if has_cont and trend not in ("pos", "neg"):
+        raise ValueError(f"{where} has continuous intervals, so 'trend' must be 'pos' or 'neg'")
+    result = {"trend": trend if trend in ("pos", "neg") else "pos", "bins": out}
+    compile_bins(result, feature)  # checks overlaps and duplicate keys
+    return result
+
+
+def special_keys(bins: dict) -> tuple[list, list]:
+    """Numeric keys typed 'cat' / 'missing' in a bins dict (e.g. -999)."""
+    cat, miss = [], []
+    for atoms in bins["bins"].values():
+        for atom, kind in atoms.items():
+            if kind != "cont" and isinstance(atom, numbers.Real) and not isinstance(atom, bool):
+                (cat if kind == "cat" else miss).append(atom)
+    return cat, miss
+
+
+@dataclass
+class CompiledBins:
+    """A bins dict in a form ready for vectorized assignment."""
+
+    trend: str
+    bin_ids: list  # bin index (dict key) per position
+    a: np.ndarray  # interval lower bounds, sorted
+    b: np.ndarray  # interval upper bounds
+    g: np.ndarray  # position of the bin owning each interval
+    key_map: dict  # cat / missing key -> position
+
+    def assign_values(self, x: np.ndarray) -> np.ndarray:
+        """Bin position of each continuous value, or -1 if it falls in no interval."""
+        if not self.a.size:
+            return np.full(x.shape[0], -1, dtype=np.int64)
+        if self.trend == "pos":  # (a, b]
+            i = np.searchsorted(self.b, x, side="left")
+            valid = i < self.b.size
+            i = np.minimum(i, self.b.size - 1)
+            ok = valid & ((x > self.a[i]) | np.isneginf(self.a[i]))
+        else:  # [a, b)
+            i = np.searchsorted(self.a, x, side="right") - 1
+            valid = i >= 0
+            i = np.maximum(i, 0)
+            ok = valid & ((x < self.b[i]) | np.isposinf(self.b[i]))
+        return np.where(ok, self.g[i], -1)
+
+    def assign(self, parsed: ParsedColumn, cont_codes: np.ndarray | None = None) -> np.ndarray:
+        """Bin position per row, or -1 for values that fall in no bin."""
+        out = np.full(parsed.n, -1, dtype=np.int64)
+        if parsed.x_cont.size:
+            if cont_codes is None:
+                cont_codes = self.assign_values(parsed.x_cont)
+            out[parsed.cont_mask] = cont_codes
+        for codes, keys in (
+            (parsed.cat_codes, parsed.cat_keys),
+            (parsed.miss_codes, parsed.miss_keys),
+        ):
+            if not keys:
+                continue
+            lut = np.array([self.key_map.get(k, -1) for k in keys] + [-1], dtype=np.int64)
+            m = codes >= 0
+            out[m] = lut[codes[m]]
+        return out
+
+
+def compile_bins(bins: dict, feature: str = "") -> CompiledBins:
+    """Compile a bins dict; raises on overlapping intervals or duplicate keys."""
+    bin_ids = sorted(bins["bins"])
+    pos_of = {b: i for i, b in enumerate(bin_ids)}
+    intervals, key_map = [], {}
+    for b, atoms in bins["bins"].items():
+        for atom, kind in atoms.items():
+            if kind == "cont":
+                intervals.append((atom[0], atom[1], pos_of[b]))
+            else:
+                if atom in key_map:
+                    raise ValueError(f"{feature}: key {atom!r} appears in more than one bin")
+                key_map[atom] = pos_of[b]
+    intervals.sort()
+    a = np.array([t[0] for t in intervals], dtype=np.float64)
+    bb = np.array([t[1] for t in intervals], dtype=np.float64)
+    g = np.array([t[2] for t in intervals], dtype=np.int64)
+    if a.size > 1 and np.any(a[1:] < bb[:-1]):
+        raise ValueError(f"{feature}: continuous intervals overlap")
+    return CompiledBins(bins.get("trend", "pos"), bin_ids, a, bb, g, key_map)
+
+
+def _fmt(v: float, decimals: int = 4) -> str:
+    if np.isinf(v):
+        return "inf" if v > 0 else "-inf"
+    if float(v).is_integer():
+        return str(int(v))
+    return f"{v:.{decimals}f}"
+
+
+def bin_label(atoms: dict, trend: str, decimals: int = 4) -> str:
+    """Display label of a bin, e.g. ``(-inf,0.3400]; special_category``.
+
+    Integer bounds print without decimals (``(18,25]``); other bounds print with exactly
+    ``decimals`` decimal places, trailing zeros kept. Only the label is rounded: the bins
+    themselves keep the full values.
+    """
+    parts = []
+    for atom, kind in atoms.items():
+        if kind == "cont":
+            a, b = atom
+            if trend == "pos":
+                right = ")" if np.isinf(b) else "]"
+                parts.append(f"({_fmt(a, decimals)},{_fmt(b, decimals)}{right}")
+            else:
+                left = "(" if np.isinf(a) else "["
+                parts.append(f"{left}{_fmt(a, decimals)},{_fmt(b, decimals)})")
+        else:
+            parts.append("NaN" if atom is None else str(atom))
+    return "; ".join(parts)

tide/_categorical.py

@@ -0,0 +1,200 @@
+"""Merging strategies for categorical bins: ChiMerge and the TIDE stability merge."""
+
+from __future__ import annotations
+
+import numpy as np
+from scipy.stats import chi2
+
+from ._greedy import EPS
+
+__all__ = ["chi2_pvalues", "chi_merge", "tide_merge"]
+
+
+def chi2_pvalues(a0, a1, b0, b1) -> np.ndarray:
+    """Vectorized p-values of the 2x2 chi-squared test with Yates' correction.
+
+    Matches ``scipy.stats.chi2_contingency([[a0, a1], [b0, b1]]).pvalue``. As in v0.5,
+    tables with two or more zero cells are given p = 1, unless the two bins are
+    perfectly separated (``[[0, +], [+, 0]]`` or ``[[+, 0], [0, +]]``).
+    """
+    obs = np.stack(
+        [np.asarray(a0, float), np.asarray(a1, float), np.asarray(b0, float), np.asarray(b1, float)]
+    )
+    zeros = (obs == 0).sum(axis=0)
+    diag = ((obs[0] == 0) & (obs[1] > 0) & (obs[2] > 0) & (obs[3] == 0)) | (
+        (obs[0] > 0) & (obs[1] == 0) & (obs[2] == 0) & (obs[3] > 0)
+    )
+    force_one = (zeros >= 2) & ~diag
+
+    ra, rb = obs[0] + obs[1], obs[2] + obs[3]
+    c0, c1 = obs[0] + obs[2], obs[1] + obs[3]
+    tot = ra + rb
+    with np.errstate(divide="ignore", invalid="ignore"):
+        exp = np.stack([ra * c0, ra * c1, rb * c0, rb * c1]) / tot
+        diff = exp - obs
+        adj = obs + np.sign(diff) * np.minimum(0.5, np.abs(diff))
+        stat = ((adj - exp) ** 2 / exp).sum(axis=0)
+    p = chi2.sf(stat, 1)
+    p = np.where(force_one, 1.0, p)
+    return np.atleast_1d(p)
+
+
+def chi_merge(n0: np.ndarray, n1: np.ndarray, alpha: float) -> list[list[int]]:
+    """ChiMerge (Kerber, 1992) over categories, as in v0.5.
+
+    Any two bins may merge. The pair with the highest p-value merges while that
+    p-value is above ``alpha``. The merged bin moves to the end of the bin list,
+    and ties resolve to the pair first seen (v0.5 order).
+
+    Returns the groups as lists of category indices, in v0.5 output order.
+    """
+    k = n0.shape[0]
+    if k < 2:
+        return [[i] for i in range(k)]
+    members: dict[int, list[int]] = {i: [i] for i in range(k)}
+    c0 = np.asarray(n0, float).copy()
+    c1 = np.asarray(n1, float).copy()
+    order = list(range(k))  # slot ids in bin-list order
+
+    pv = np.full((k, k), -np.inf)
+    rank = np.full((k, k), np.iinfo(np.int64).max, dtype=np.int64)
+    iu, ju = np.triu_indices(k, 1)
+    pv[iu, ju] = chi2_pvalues(c0[iu], c1[iu], c0[ju], c1[ju])
+    rank[iu, ju] = np.arange(iu.size)
+    next_rank = iu.size
+
+    while len(order) > 1:
+        best = pv.max()
+        if best <= alpha:
+            break
+        cand = pv == best
+        r = np.where(cand, rank, np.iinfo(np.int64).max)
+        i, j = np.unravel_index(np.argmin(r), r.shape)
+        i, j = int(i), int(j)
+        # merge slot j into slot i; the merged bin goes to the end of the list
+        members[i] = members[i] + members.pop(j)
+        c0[i] += c0[j]
+        c1[i] += c1[j]
+        order = [s for s in order if s not in (i, j)] + [i]
+        for s in (i, j):
+            pv[s, :] = -np.inf
+            pv[:, s] = -np.inf
+        others = np.array(order[:-1], dtype=np.int64)
+        if others.size:
+            p_new = chi2_pvalues(
+                c0[others], c1[others], np.full(others.size, c0[i]), np.full(others.size, c1[i])
+            )
+            lo = np.minimum(others, i)
+            hi = np.maximum(others, i)
+            pv[lo, hi] = p_new
+            rank[lo, hi] = next_rank + np.arange(others.size)
+            next_rank += others.size
+
+    return [sorted(members[s]) for s in order]
+
+
+def tide_merge(
+    counts: np.ndarray,
+    events: np.ndarray,
+    n_cont: int,
+    trend: str,
+    min_sample_rate: float,
+    min_class_obs: int,
+    min_er_delta: float,
+) -> list[list[int]]:
+    """Merge atomic bins (continuous bins, categories, missing) until all constraints hold.
+
+    Atoms ``0..n_cont-1`` are the continuous bins, in axis order. Violations are fixed
+    in this priority order:
+
+    0. monotonic event rate of the bins with continuous intervals, in every period;
+    1. no crossings of event-rate curves between any two bins (v0.5);
+    2. minimum event-rate gap ``min_er_delta`` between any two bins (v0.5);
+    3. bin size ``>= min_sample_rate`` in every period (v0.5);
+    4. ``>= min_class_obs`` events and non-events in every period (v0.5).
+
+    For 1-4, the offending bin (the smallest by rows, on ties) merges with the bin
+    closest by squared event-rate distance. Two bins with continuous intervals may merge
+    only if they are adjacent, so every bin keeps one contiguous interval.
+
+    Returns the groups as lists of atom indices.
+    """
+    pos = trend == "pos"
+    c = counts.astype(np.float64).copy()
+    e = events.astype(np.float64).copy()
+    groups: list[list[int]] = [[a] for a in range(c.shape[0])]
+    lo = np.array([a if a < n_cont else -1 for a in range(c.shape[0])], dtype=np.int64)
+    hi = lo.copy()
+
+    def merge(t: int, g: int) -> None:
+        nonlocal c, e, lo, hi
+        groups[t] = groups[t] + groups[g]
+        c[t] += c[g]
+        e[t] += e[g]
+        if lo[g] >= 0:
+            lo[t] = lo[g] if lo[t] < 0 else min(lo[t], lo[g])
+            hi[t] = max(hi[t], hi[g])
+        del groups[g]
+        c = np.delete(c, g, axis=0)
+        e = np.delete(e, g, axis=0)
+        lo = np.delete(lo, g)
+        hi = np.delete(hi, g)
+
+    while len(groups) > 1:
+        er = e / (c + EPS)
+        tot = c.sum(axis=1)
+        has_cont = lo >= 0
+
+        # 0. monotonicity of continuous bins
+        ci = np.flatnonzero(has_cont)
+        if ci.size > 1:
+            ci = ci[np.argsort(lo[ci])]
+            d = er[ci[1:]] - er[ci[:-1]]
+            viol = (d < 0).any(axis=1) if pos else (d > 0).any(axis=1)
+            if viol.any():
+                cand = np.flatnonzero(viol)
+                pair_tot = tot[ci[cand]] + tot[ci[cand + 1]]
+                q = cand[np.argmin(pair_tot)]
+                t, g = int(ci[q]), int(ci[q + 1])
+                merge(t, g)
+                continue
+
+        sgn = (er[:, None, :] > er[None, :, :]).astype(np.int8) - (
+            er[:, None, :] < er[None, :, :]
+        ).astype(np.int8)
+        crossings = (sgn[:, :, :-1] != sgn[:, :, 1:]).sum(axis=2).sum(axis=1)
+        dist = np.abs(er[:, None, :] - er[None, :, :]).min(axis=2)
+        np.fill_diagonal(dist, np.inf)
+        delta = dist.min(axis=1)
+        sample_rate = (c / c.sum(axis=0)).min(axis=1)
+        ev_min = e.min(axis=1)
+        nev_min = (c - e).min(axis=1)
+
+        if crossings.max() > 0:
+            bad = crossings == crossings.max()
+        elif delta.min() < min_er_delta:
+            bad = delta == delta.min()
+        elif sample_rate.min() < min_sample_rate:
+            bad = sample_rate == sample_rate.min()
+        elif ev_min.min() < min_class_obs:
+            bad = ev_min == ev_min.min()
+        elif nev_min.min() < min_class_obs:
+            bad = nev_min == nev_min.min()
+        else:
+            break
+        cand = np.flatnonzero(bad)
+        t = int(cand[np.argmin(tot[cand])])
+
+        allowed = np.ones(len(groups), dtype=bool)
+        allowed[t] = False
+        if has_cont[t]:
+            adjacent = (hi[t] + 1 == lo) | (hi + 1 == lo[t])
+            allowed &= ~has_cont | adjacent
+        if not allowed.any():  # pragma: no cover - cannot happen with >= 2 groups
+            break
+        sq = ((er[t] - er) ** 2).sum(axis=1)
+        sq[~allowed] = np.inf
+        g = int(np.argmin(sq))
+        merge(t, g)
+
+    return groups