compileml 0.1.0__py3-none-any.whl

compileml/__init__.py

@@ -0,0 +1,17 @@
+"""CompileML — compile tree-ensemble models into deterministic decision artifacts.
+
+The package splits into two halves with different dependency contracts:
+
+- ``compileml.compile`` / ``compileml.artifact`` / ``compileml.bands`` /
+  ``compileml.validate`` / ``compileml.export`` — the *compile side*, which may
+  use numpy and scikit-learn.
+- ``compileml.runtime`` — the *decision side*, which imports only the Python
+  standard library. A compiled artifact can be scored, banded, calibrated, and
+  explained with nothing but this subpackage (or a copy of it).
+"""
+
+from compileml.runtime import decide, load_artifact, verify_artifact
+
+__version__ = "0.1.0.dev0"
+
+__all__ = ["decide", "load_artifact", "verify_artifact", "__version__"]

compileml/artifact/__init__.py

@@ -0,0 +1,7 @@
+"""Artifact assembly: build, calibrate, hash, save, recalibrate."""
+
+from compileml.artifact.build import build_artifact, save_artifact
+from compileml.artifact.calibration import fit_isotonic_table
+from compileml.artifact.recalibrate import recalibrate_artifact
+
+__all__ = ["build_artifact", "fit_isotonic_table", "recalibrate_artifact", "save_artifact"]

compileml/artifact/build.py

@@ -0,0 +1,241 @@
+"""Assemble, hash, and save CompileML decision artifacts (spec §3, §9)."""
+
+from __future__ import annotations
+
+import json
+import warnings
+from os import PathLike
+
+import numpy as np
+
+from compileml import __version__
+from compileml.artifact.calibration import fit_isotonic_table
+from compileml.compile.extract import extract_trees, score_float
+from compileml.compile.quantize import (
+    max_depth,
+    quantization_error_bound,
+    quantize_model,
+    rha,
+)
+from compileml.runtime.io import ARTIFACT_TYPE, SCHEMA_VERSION, canonical_hash, validate_structure
+from compileml.runtime.score import score_micro
+
+
+def _band_edges_int(band_edges, scale: int) -> list[int]:
+    """Convert float band edges to strictly increasing fixed-point integers."""
+    edges_int = [rha(float(e) * scale) for e in band_edges]
+    collisions = [
+        (band_edges[i], band_edges[i + 1])
+        for i in range(len(edges_int) - 1)
+        if edges_int[i + 1] <= edges_int[i]
+    ]
+    if collisions:
+        raise ValueError(
+            f"band edges collide after fixed-point conversion at scale={scale}: "
+            f"{collisions}. Use fewer bands or a larger scale."
+        )
+    return edges_int
+
+
+def build_artifact(
+    model,
+    feature_names,
+    baseline,
+    band_edges,
+    *,
+    band_labels=None,
+    calibration_latent=None,
+    calibration_y=None,
+    calibration: dict | None = None,
+    calibration_mode: str = "linear_int",
+    reasons: dict | None = None,
+    display_names: dict | None = None,
+    feature_meta: list | None = None,
+    missing_policy: str = "baseline",
+    metadata: dict | None = None,
+    scale: int = 1000,
+    micro_scale: int = 1_000_000,
+    top_k: int = 5,
+    threshold_decimals: int | None = None,
+    X_sample=None,
+) -> dict:
+    """Compile a fitted model into a complete, hashed decision artifact.
+
+    Args:
+        model: Fitted sklearn GBM, XGBoost, or LightGBM model whose raw
+            output is a probability-like latent in [0, 1] (distilled
+            whiteboxes always satisfy this; classifiers emitting log-odds
+            margins must be distilled first via ``train_whitebox``).
+        feature_names: Feature order the model was trained on.
+        baseline: Reference row (typically imputer medians): imputation
+            values under missing_policy="baseline" and the attribution
+            reference point.
+        band_edges: Float latent band edges (from compileml.bands builders),
+            converted here to the fixed-point ladder.
+        band_labels: Labels per band; defaults to G01..Gnn.
+        calibration_latent: Latent sample used to fit the isotonic PD table
+            (alternatively pass a prebuilt ``calibration`` block).
+        calibration_y: Binary outcomes aligned with ``calibration_latent``.
+        reasons: Reason dictionary mapping feature name -> {code, negative,
+            positive, suppress}. **User-supplied content**: without an entry
+            a feature falls back to generic messages that are not suitable
+            for consumer-facing notices. Coverage below 100% warns and is
+            recorded in metadata (spec §7.6).
+        missing_policy: "baseline" (impute at decision time) or "reject".
+        X_sample: Optional sample rows; enables the measured quantization
+            report and the latent-range check.
+
+    Returns:
+        The artifact dict, hashed and structurally validated.
+    """
+    if missing_policy not in ("baseline", "reject"):
+        raise ValueError("missing_policy must be 'baseline' or 'reject'")
+    if micro_scale % scale != 0:
+        raise ValueError("micro_scale must be an integer multiple of scale")
+
+    # Accept a BandSpec (from compileml.bands builders) in place of raw edges.
+    bands_meta: dict = {}
+    if hasattr(band_edges, "edges") and hasattr(band_edges, "labels"):
+        spec = band_edges
+        if band_labels is None:
+            band_labels = list(spec.labels)
+        bands_meta = dict(getattr(spec, "metadata", {}) or {})
+        band_edges = list(spec.edges)
+
+    names = [str(n) for n in feature_names]
+    base_vals = [float(b) for b in np.asarray(baseline, dtype=float).reshape(-1)]
+    if len(names) != len(base_vals):
+        raise ValueError("feature_names and baseline must have the same length")
+
+    # --- extract + quantize -------------------------------------------------
+    extracted = extract_trees(model)
+    if extracted.n_features and extracted.n_features != len(names):
+        raise ValueError(f"model expects {extracted.n_features} features, got {len(names)} names")
+    if threshold_decimals is not None:
+        # Quantize split thresholds for decimal-arithmetic targets (spec §11).
+        # Every runtime — Python included — then compares identical values;
+        # the quantization report below measures the routing cost.
+        for tree in extracted.trees:
+            tree["threshold"] = [round(t, int(threshold_decimals)) for t in tree["threshold"]]
+    model_int = quantize_model(extracted, micro_scale=micro_scale)
+    depth = max_depth(model_int)
+    if depth > 2:
+        warnings.warn(
+            f"whitebox depth is {depth} (> 2): attribution will carry a nonzero "
+            "residual and exact_attribution will be False (spec §7.3).",
+            stacklevel=2,
+        )
+
+    # --- quantization + latent-range report ---------------------------------
+    quant_report = {"error_bound": quantization_error_bound(model_int)}
+    if threshold_decimals is not None:
+        quant_report["threshold_decimals"] = int(threshold_decimals)
+    if X_sample is not None:
+        X_arr = np.asarray(X_sample, dtype=float)
+        float_scores = np.array([score_float(extracted, [float(v) for v in row]) for row in X_arr])
+        int_scores = (
+            np.array([score_micro(model_int, [float(v) for v in row]) for row in X_arr])
+            / micro_scale
+        )
+        quant_report["measured_max_error"] = float(np.max(np.abs(float_scores - int_scores)))
+        outside = float(np.mean((float_scores < 0.0) | (float_scores > 1.0)))
+        quant_report["share_outside_unit_interval"] = outside
+        if outside > 0.01:
+            warnings.warn(
+                f"{outside:.1%} of sample latents fall outside [0, 1] before clamping. "
+                "The artifact contract expects probability-like latents; distill "
+                "margin-space models first (train_whitebox).",
+                stacklevel=2,
+            )
+
+    # --- calibration ---------------------------------------------------------
+    if calibration is None and calibration_latent is not None:
+        if calibration_y is None:
+            raise ValueError("calibration_y is required when calibration_latent is given")
+        calibration = fit_isotonic_table(
+            calibration_latent, calibration_y, micro_scale=micro_scale, mode=calibration_mode
+        )
+
+    # --- bands ----------------------------------------------------------------
+    edges_int = _band_edges_int(band_edges, scale)
+    n_bands = len(edges_int) - 1
+    labels = (
+        [str(label) for label in band_labels]
+        if band_labels is not None
+        else [f"G{i + 1:02d}" for i in range(n_bands)]
+    )
+    if len(labels) != n_bands:
+        raise ValueError(f"expected {n_bands} band labels, got {len(labels)}")
+
+    # --- reason coverage (spec §7.6: user-supplied content) -------------------
+    reasons = dict(reasons or {})
+    covered = [n for n in names if n in reasons]
+    uncovered = [n for n in names if n not in reasons]
+    coverage = len(covered) / len(names) if names else 1.0
+    if uncovered:
+        warnings.warn(
+            f"reason dictionary covers {len(covered)}/{len(names)} features "
+            f"({coverage:.0%}). Uncovered features fall back to generic messages "
+            f"unsuitable for consumer-facing notices: {uncovered}",
+            stacklevel=2,
+        )
+
+    artifact = {
+        "artifact_type": ARTIFACT_TYPE,
+        "schema_version": SCHEMA_VERSION,
+        "scale": int(scale),
+        "model": model_int,
+        "calibration": calibration,
+        "bands": {
+            "edges_int": edges_int,
+            "labels": labels,
+            "boundary": "left_closed_right_open",
+        },
+        "features": {
+            "names": names,
+            "baseline": base_vals,
+            "missing_policy": missing_policy,
+            "display_names": {str(k): str(v) for k, v in (display_names or {}).items()},
+            "meta": _plain(feature_meta or []),
+        },
+        "reasons": _plain(reasons),
+        "runtime": {
+            "attribution": "pairwise_interaction_int",
+            "top_k": int(top_k),
+            "whitebox_max_depth": depth,
+            "exact_attribution": depth <= 2,
+        },
+        "metadata": {
+            **_plain(metadata or {}),
+            "compileml_version": __version__,
+            "model_family": extracted.family,
+            "n_trees": len(model_int["trees"]),
+            "reason_coverage": round(coverage, 4),
+            "quantization": _plain(quant_report),
+            **({"bands": _plain(bands_meta)} if bands_meta else {}),
+        },
+    }
+    artifact["artifact_hash"] = canonical_hash(artifact)
+    validate_structure(artifact)
+    return artifact
+
+
+def save_artifact(artifact: dict, path: str | PathLike, *, indent: int | None = 2) -> None:
+    """Write an artifact to JSON (hash already embedded; loaders verify it)."""
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(artifact, f, indent=indent, ensure_ascii=False)
+
+
+def _plain(value):
+    """Recursively convert numpy scalars/arrays to JSON-native Python types."""
+    if isinstance(value, np.ndarray):
+        return [_plain(v) for v in value.tolist()]
+    if isinstance(value, np.floating):
+        return float(value)
+    if isinstance(value, np.integer):
+        return int(value)
+    if isinstance(value, dict):
+        return {str(k): _plain(v) for k, v in value.items()}
+    if isinstance(value, (list, tuple)):
+        return [_plain(v) for v in value]
+    return value

compileml/artifact/calibration.py

@@ -0,0 +1,52 @@
+"""Fit an isotonic calibration and freeze it as an integer table (spec §6)."""
+
+from __future__ import annotations
+
+import numpy as np
+
+from compileml.compile.quantize import rha
+from compileml.runtime.calibrate import PD_SCALE
+
+
+def fit_isotonic_table(
+    latent,
+    y,
+    *,
+    micro_scale: int = 1_000_000,
+    mode: str = "linear_int",
+) -> dict:
+    """Isotonic PD calibration frozen into integer thresholds.
+
+    ``latent`` is the (clipped) model latent on a calibration sample; ``y``
+    the binary outcomes. Returns the artifact's ``calibration`` block:
+    strictly increasing ``f_micro``, non-decreasing ``pd_ppm``.
+    """
+    from sklearn.isotonic import IsotonicRegression
+
+    if mode not in ("linear_int", "step"):
+        raise ValueError("mode must be 'linear_int' or 'step'")
+
+    x = np.clip(np.asarray(latent, dtype=float).reshape(-1), 0.0, 1.0)
+    yy = np.asarray(y, dtype=float).reshape(-1)
+    if x.shape[0] != yy.shape[0]:
+        raise ValueError("latent and y must have the same length")
+    if x.shape[0] == 0:
+        raise ValueError("latent must not be empty")
+
+    iso = IsotonicRegression(y_min=0.0, y_max=1.0, out_of_bounds="clip")
+    iso.fit(x, yy)
+
+    f_micro: list[int] = []
+    pd_ppm: list[int] = []
+    for xf, yf in zip(iso.X_thresholds_, iso.y_thresholds_):
+        f = rha(float(xf) * micro_scale)
+        p = min(max(rha(float(yf) * PD_SCALE), 0), PD_SCALE)
+        if f_micro and f == f_micro[-1]:
+            # Thresholds that collide after rounding: keep the larger PD
+            # (isotonic y is non-decreasing, so this is the later one).
+            pd_ppm[-1] = max(pd_ppm[-1], p)
+        else:
+            f_micro.append(f)
+            pd_ppm.append(p)
+
+    return {"mode": mode, "f_micro": f_micro, "pd_ppm": pd_ppm}

compileml/artifact/recalibrate.py

@@ -0,0 +1,98 @@
+"""Zero-churn artifact recalibration — the retraining story (spec §6, §9).
+
+When the portfolio drifts, PDs go stale but the model and the band edges
+need not move. ``recalibrate_artifact`` refits the isotonic PD table and
+refreshes per-band bad-rate metadata on fresh outcomes while keeping the
+model and the fixed-point ladder byte-identical. The result is a new
+hashed artifact that records its predecessor's hash — a provenance chain:
+
+    artifact_v1 --(fresh outcomes)--> artifact_v2
+    same model, same edges, same band assignments, new PDs, new hash
+
+Because band assignment depends only on the model and edges, **no account
+changes band** as a result of recalibration. That is the zero-churn
+guarantee, and it is testable.
+"""
+
+from __future__ import annotations
+
+import copy
+
+import numpy as np
+
+from compileml.artifact.calibration import fit_isotonic_table
+from compileml.compile.quantize import rha
+from compileml.runtime.bands import band_index
+from compileml.runtime.io import canonical_hash
+
+
+def recalibrate_artifact(
+    artifact: dict,
+    latent,
+    y,
+    *,
+    mode: str | None = None,
+    prior_strength: float = 0.0,
+    prior_pi0: float | None = None,
+) -> dict:
+    """Refit calibration on fresh outcomes; model and band edges unchanged.
+
+    Args:
+        artifact: An existing decision artifact.
+        latent: Fresh latent scores in [0, 1] (clipped), e.g. the artifact's
+            own ``latent_micro / micro_scale`` on the new sample.
+        y: Fresh binary outcomes aligned with ``latent``.
+        mode: Calibration mode for the new table; defaults to the old one.
+        prior_strength: Optional shrinkage weight pulling per-band empirical
+            bad rates toward a prior for the band metadata refresh — small
+            bands get stabilized rates.
+        prior_pi0: The prior rate (defaults to the global bad rate).
+
+    Returns:
+        A new artifact dict with updated ``calibration``, refreshed band
+        bad-rate metadata, ``metadata.recalibrated_from`` set to the old
+        hash, and a new ``artifact_hash``.
+    """
+    F = np.clip(np.asarray(latent, dtype=float).reshape(-1), 0.0, 1.0)
+    y_arr = np.asarray(y, dtype=float).reshape(-1)
+    if F.shape[0] != y_arr.shape[0]:
+        raise ValueError("latent and y must have the same length")
+
+    old_hash = artifact.get("artifact_hash")
+    micro_scale = int(artifact["model"]["micro_scale"])
+    scale = int(artifact["scale"])
+    ratio = micro_scale // scale
+
+    new = copy.deepcopy(artifact)
+
+    # --- refit the decision-time PD table --------------------------------
+    old_mode = (artifact.get("calibration") or {}).get("mode", "linear_int")
+    new["calibration"] = fit_isotonic_table(
+        F, y_arr, micro_scale=micro_scale, mode=mode or old_mode
+    )
+
+    # --- refresh per-band bad-rate metadata via the deployed integer path -
+    edges_int = new["bands"]["edges_int"]
+    n_bands = len(edges_int) - 1
+    latent_int = [rha(float(v) * micro_scale) // ratio for v in F]
+    idx = np.array([band_index(v, edges_int) for v in latent_int])
+    counts = np.bincount(idx, minlength=n_bands)
+    bad = np.bincount(idx, weights=y_arr, minlength=n_bands)
+    global_pd = float(np.mean(y_arr))
+    rate = np.where(counts > 0, bad / np.maximum(counts, 1), global_pd)
+    if prior_strength > 0.0:
+        pi0 = global_pd if prior_pi0 is None else float(prior_pi0)
+        rate = (rate * counts + prior_strength * pi0) / (counts + prior_strength)
+
+    new["metadata"] = dict(new.get("metadata") or {})
+    new["metadata"]["recalibration"] = {
+        "recalibrated_from": old_hash,
+        "n_observations": int(F.shape[0]),
+        "global_bad_rate": global_pd,
+        "band_counts": [int(c) for c in counts],
+        "band_bad_rate": [float(r) for r in rate],
+        "prior_strength": float(prior_strength),
+    }
+
+    new["artifact_hash"] = canonical_hash(new)
+    return new

compileml/bands/__init__.py

@@ -0,0 +1,15 @@
+"""Risk band construction: plain quantile, monotone-quantile, and
+search-and-certify builders. All return a :class:`BandSpec` consumed by
+``compileml.artifact.build_artifact``.
+"""
+
+from compileml.bands.builders import BandSpec, monotone_quantile_bands, quantile_bands
+from compileml.bands.certified import governance_bands, semantic_bands
+
+__all__ = [
+    "BandSpec",
+    "governance_bands",
+    "monotone_quantile_bands",
+    "quantile_bands",
+    "semantic_bands",
+]

compileml/bands/builders.py

@@ -0,0 +1,123 @@
+"""Risk band construction from latent scores.
+
+Builders return a :class:`BandSpec` — float edges, labels, and metadata —
+which ``compileml.artifact.build_artifact`` freezes into the fixed-point
+integer ladder. Note: builders deliberately record **no timestamps**;
+identical inputs must yield identical artifacts (and identical hashes).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+import numpy as np
+from sklearn.isotonic import IsotonicRegression
+
+
+@dataclass
+class BandSpec:
+    """Float-space band definition produced by the builders."""
+
+    edges: list[float]  # n_bands + 1, strictly increasing
+    labels: list[str]
+    metadata: dict = field(default_factory=dict)
+
+    @property
+    def n_bands(self) -> int:
+        return len(self.edges) - 1
+
+
+def _quantile_edges(latent: np.ndarray, n_bands: int, method: str) -> np.ndarray:
+    if method == "quantile":
+        edges = np.quantile(latent, np.linspace(0.0, 1.0, n_bands + 1))
+        edges[0] = float(np.min(latent))
+        edges[-1] = float(np.max(latent)) + 1e-12
+    elif method == "equal_width":
+        edges = np.linspace(float(np.min(latent)), float(np.max(latent)), n_bands + 1)
+        edges[-1] += 1e-12
+    else:
+        raise ValueError("method must be 'quantile' or 'equal_width'")
+    # enforce strict monotonicity on degenerate distributions
+    for i in range(1, len(edges)):
+        if edges[i] <= edges[i - 1]:
+            edges[i] = edges[i - 1] + 1e-12
+    return edges
+
+
+def _labels(n: int) -> list[str]:
+    return [f"G{i + 1:02d}" for i in range(n)]
+
+
+def quantile_bands(latent, n_bands: int = 10, *, method: str = "quantile") -> BandSpec:
+    """Plain quantile (or equal-width) bands; no outcome data required."""
+    x = np.asarray(latent, dtype=float).reshape(-1)
+    edges = _quantile_edges(x, n_bands, method)
+    counts = np.bincount(np.clip(np.digitize(x, edges) - 1, 0, n_bands - 1), minlength=n_bands)
+    return BandSpec(
+        edges=[float(e) for e in edges],
+        labels=_labels(n_bands),
+        metadata={"method": method, "counts": [int(c) for c in counts]},
+    )
+
+
+def monotone_quantile_bands(
+    latent,
+    y,
+    n_bands: int = 10,
+    *,
+    allow_merge: bool = False,
+    merge_eps: float = 0.005,
+) -> BandSpec:
+    """Quantile bands with empirical bad rates and isotonic-smoothed semantics.
+
+    Fixed-K quantile edges by default. With ``allow_merge=True``, adjacent
+    bands whose empirical bad rates invert by more than ``merge_eps`` are
+    merged until no material violation remains — trading band count for
+    guaranteed-monotone empirical semantics.
+    """
+    F = np.asarray(latent, dtype=float).reshape(-1)
+    y_arr = np.asarray(y, dtype=float).reshape(-1)
+    if F.shape[0] != y_arr.shape[0]:
+        raise ValueError("latent and y must have the same length")
+
+    edges = _quantile_edges(F, n_bands, "quantile")
+
+    def stats(cur_edges):
+        idx = np.clip(np.digitize(F, cur_edges) - 1, 0, len(cur_edges) - 2)
+        k = len(cur_edges) - 1
+        counts = np.bincount(idx, minlength=k)
+        bad = np.bincount(idx, weights=y_arr, minlength=k)
+        rate = np.where(counts > 0, bad / np.maximum(counts, 1), float(np.mean(y_arr)))
+        return counts.astype(int), rate
+
+    merges: list[dict] = []
+    counts, emp_rate = stats(edges)
+    if allow_merge:
+        while len(emp_rate) > 1:
+            violations = emp_rate[:-1] - emp_rate[1:]
+            worst = float(np.max(violations))
+            if worst <= merge_eps:
+                break
+            i = int(np.argmax(violations))
+            merges.append({"merge_idx": i, "violation": worst})
+            edges = np.delete(edges, i + 1)
+            counts, emp_rate = stats(edges)
+
+    smoothed = IsotonicRegression(increasing=True, out_of_bounds="clip").fit_transform(
+        np.arange(len(emp_rate)), emp_rate
+    )
+
+    k = len(edges) - 1
+    return BandSpec(
+        edges=[float(e) for e in edges],
+        labels=_labels(k),
+        metadata={
+            "method": "monotone_quantile",
+            "counts": [int(c) for c in counts],
+            "empirical_bad_rate": [float(v) for v in emp_rate],
+            "smoothed_bad_rate": [float(v) for v in smoothed],
+            "allow_merge": bool(allow_merge),
+            "merge_eps": float(merge_eps),
+            "merges": merges,
+        },
+    )