closed-loop-default-detection 0.1.0__py3-none-any.whl

cldd/diagnostics.py

@@ -0,0 +1,107 @@
+"""Deployable positivity/overlap diagnostics — the observable frontier signal.
+
+Lesson from the hackathon harness (FABLE.md): the operating frontier was measured
+against *planted* truth (declined-cohort ECE), which a real lender can never
+compute — so the frontier, as measured, is not deployable knowledge. What a
+lender CAN compute is how pathological the selection itself looks:
+
+* **propensity separability** — fit P(funded | features) and take its in-sample
+  AUC. Near 0.5 the funded and unfunded pools overlap (IPW has support to work
+  with); near 1.0 the policy is deterministic in the features and positivity is
+  gone by construction.
+* **IPW weight degeneracy** — the effective sample size of the 1/propensity
+  weights actually used by the reweight lever, as a fraction of the funded count.
+  When a few rows carry all the weight, the "corrected" estimate is resting on
+  almost no data.
+* **unreachable region** — the share of unfunded rows whose propensity sits at
+  or below the clip floor: the part of the declined population the funded sample
+  cannot speak for at all.
+
+None of these need a single declined-row label. The harness validates them
+against the hidden truth (see ``test_diagnostics`` and the frontier artifacts):
+inside the measured frontier they look healthy, beyond it they deteriorate — so
+a real deployment can use them as an abstention trigger where this harness used
+planted ground truth.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import numpy as np
+from sklearn.metrics import roc_auc_score
+
+from . import config, model_pd
+
+
+@dataclass(frozen=True)
+class PositivityDiagnostics:
+    """Observable-only overlap health for one cohort's funding selection."""
+
+    #: In-sample AUC of P(funded | features). ~0.5 = overlapping pools,
+    #: ~1.0 = selection deterministic in the features (positivity broken).
+    propensity_auc: float
+    #: Effective sample size of the clipped 1/propensity weights on funded rows,
+    #: divided by the funded count. 1.0 = uniform weights; small = degenerate.
+    ess_ratio: float
+    #: Share of unfunded rows with raw propensity <= the clip floor — the region
+    #: the funded sample cannot represent.
+    unfunded_below_floor: float
+    #: Observable abstention signal: True when any component crosses its
+    #: configured threshold (calibrated on this harness — see config).
+    flagged: bool
+
+
+def positivity_diagnostics(
+    X,
+    funded,
+    *,
+    random_state: int | None = None,
+    clip: tuple[float, float] = (0.05, 0.95),
+    auc_max: float | None = None,
+    ess_min: float | None = None,
+    below_floor_max: float | None = None,
+) -> PositivityDiagnostics:
+    """Compute observable positivity diagnostics for a funding selection.
+
+    Mirrors :func:`cldd.model_pd.selection_adjusted_weights` (same classifier,
+    same clip) so the ESS describes the weights the IPW lever would actually use.
+    Thresholds default to the config values; pass overrides for sweeps.
+    """
+    X = np.asarray(X, dtype=float)
+    funded = np.asarray(funded).astype(bool)
+    rs = config.RANDOM_SEED if random_state is None else random_state
+    auc_max = config.DIAG_PROPENSITY_AUC_MAX if auc_max is None else auc_max
+    ess_min = config.DIAG_ESS_RATIO_MIN if ess_min is None else ess_min
+    below_floor_max = (
+        config.DIAG_UNFUNDED_BELOW_FLOOR_MAX if below_floor_max is None else below_floor_max
+    )
+
+    n_funded = int(funded.sum())
+    n_unfunded = int((~funded).sum())
+    if n_funded == 0 or n_unfunded == 0:
+        # Degenerate selection: nothing to compare against — abstain.
+        return PositivityDiagnostics(
+            propensity_auc=float("nan"),
+            ess_ratio=float("nan"),
+            unfunded_below_floor=float("nan"),
+            flagged=True,
+        )
+
+    clf = model_pd._new_classifier(rs)
+    clf.fit(X, funded.astype(int))
+    propensity = clf.predict_proba(X)[:, 1]
+
+    auc = float(roc_auc_score(funded.astype(int), propensity))
+    below_floor = float((propensity[~funded] <= clip[0]).mean())
+
+    w = 1.0 / np.clip(propensity[funded], clip[0], clip[1])
+    ess_ratio = float((w.sum() ** 2) / (w**2).sum() / n_funded)
+
+    flagged = (auc > auc_max) or (ess_ratio < ess_min) or (below_floor > below_floor_max)
+    return PositivityDiagnostics(
+        propensity_auc=auc,
+        ess_ratio=ess_ratio,
+        unfunded_below_floor=below_floor,
+        flagged=bool(flagged),
+    )

cldd/eval_default.py

@@ -0,0 +1,161 @@
+"""Measurement edge of the closed loop (the **measure** stage).
+
+Train the PD model on the *observed* (approved) rows only — exactly what a real
+lender has — then score it against the planted ground truth across three
+subpopulations: **all / approved / declined**. The declined subpopulation is the
+headline: it is the part of the applicant pool whose outcomes real data can never
+reveal, so its calibration error (ECE) is the signal the loop optimizes and the
+selection bias it exposes.
+
+The reusable pieces here (``fit_observed_model`` and ``score_pd_detection``) are
+shared with ``loop.py`` so the "measure" and "improve" stages score detection
+identically — mirroring the building blocks shared in
+``upstream-label-correction/evals/mislabel_detection.py``.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+import numpy as np
+from sklearn.metrics import brier_score_loss, roc_auc_score
+
+from . import config, model_pd
+
+
+@dataclass
+class SubgroupMetrics:
+    """Detection/calibration metrics on one subpopulation."""
+
+    n: int
+    base_rate: float          # true default fraction in the subgroup
+    mean_pd: float            # mean predicted PD (vs base_rate => bias direction)
+    auc: float
+    brier: float
+    ece: float
+    f1: float
+
+
+@dataclass
+class PdDetectionResult:
+    """Result of scoring one model on one cohort (local analog of EvalResult)."""
+
+    name: str
+    passed: bool
+    score: float              # headline: declined-subpopulation ECE (lower better)
+    threshold: float
+    details: dict = field(default_factory=dict)
+
+
+def expected_calibration_error(y_true: np.ndarray, p: np.ndarray, n_bins: int = 10) -> float:
+    """Standard ECE: mean over equal-width PD bins of ``|accuracy - confidence|``."""
+    y_true = np.asarray(y_true, dtype=float)
+    p = np.asarray(p, dtype=float)
+    if len(p) == 0:
+        return float("nan")
+    bins = np.linspace(0.0, 1.0, n_bins + 1)
+    bin_idx = np.digitize(p, bins[1:-1])  # 0..n_bins-1
+    ece = 0.0
+    for b in range(n_bins):
+        m = bin_idx == b
+        count = int(m.sum())
+        if count == 0:
+            continue
+        ece += (count / len(p)) * abs(float(y_true[m].mean()) - float(p[m].mean()))
+    return float(ece)
+
+
+def _f1_at_threshold(y_true: np.ndarray, p: np.ndarray, threshold: float) -> float:
+    pred = (np.asarray(p) >= threshold).astype(int)
+    yt = np.asarray(y_true).astype(int)
+    tp = int(((pred == 1) & (yt == 1)).sum())
+    fp = int(((pred == 1) & (yt == 0)).sum())
+    fn = int(((pred == 0) & (yt == 1)).sum())
+    precision = tp / (tp + fp) if (tp + fp) else 0.0
+    recall = tp / (tp + fn) if (tp + fn) else 0.0
+    return float(2 * precision * recall / (precision + recall)) if (precision + recall) else 0.0
+
+
+def _subgroup_metrics(y_true: np.ndarray, p: np.ndarray, threshold: float) -> SubgroupMetrics:
+    y_true = np.asarray(y_true, dtype=int)
+    p = np.asarray(p, dtype=float)
+    n = len(y_true)
+    if n == 0:
+        return SubgroupMetrics(0, float("nan"), float("nan"), float("nan"), float("nan"), float("nan"), float("nan"))
+    try:
+        auc = float(roc_auc_score(y_true, p)) if len(np.unique(y_true)) > 1 else float("nan")
+    except ValueError:
+        auc = float("nan")
+    return SubgroupMetrics(
+        n=n,
+        base_rate=float(y_true.mean()),
+        mean_pd=float(p.mean()),
+        auc=auc,
+        brier=float(brier_score_loss(y_true, p)) if len(np.unique(y_true)) > 1 else float("nan"),
+        ece=expected_calibration_error(y_true, p),
+        f1=_f1_at_threshold(y_true, p, threshold),
+    )
+
+
+def fit_observed_model(cohort: dict, sample_weight=None, random_state: int | None = None) -> model_pd.CalibratedPDModel:
+    """Fit the PD model on the cohort's **approved** rows only (selective labels).
+
+    ``sample_weight``, when given, must already be aligned to the approved rows
+    (length == ``cohort['approved'].sum()``) — the loop passes the approved slice
+    of the IPW weights here.
+    """
+    X = cohort["features"].to_numpy(dtype=float)
+    approved = cohort["approved"]
+    return model_pd.train_pd_model(
+        X[approved],
+        cohort["true_default"][approved],
+        sample_weight=sample_weight,
+        random_state=random_state,
+    )
+
+
+def score_pd_detection(
+    model: model_pd.CalibratedPDModel,
+    cohort: dict,
+    threshold: float | None = None,
+    funded=None,
+) -> dict:
+    """Score ``model`` on the full cohort vs planted truth, split by subpopulation.
+
+    Returns ``{'all', 'approved', 'declined'}`` -> :class:`SubgroupMetrics`, plus
+    the raw ``predictions``. ``funded`` overrides the subgrouping mask (default:
+    the cohort's prior-policy ``approved``) — the exploration lever and the
+    feedback loop pass the mask of rows that were *actually* funded, so
+    ``'declined'`` is exactly the never-labeled (out-of-training) population.
+    """
+    threshold = config.POLICY_PD_THRESHOLD if threshold is None else threshold
+    X = cohort["features"].to_numpy(dtype=float)
+    p = model_pd.predict_pd(model, X)
+    y = cohort["true_default"]
+    approved = cohort["approved"] if funded is None else np.asarray(funded, dtype=bool)
+    declined = ~approved
+    return {
+        "all": _subgroup_metrics(y, p, threshold),
+        "approved": _subgroup_metrics(y[approved], p[approved], threshold),
+        "declined": _subgroup_metrics(y[declined], p[declined], threshold),
+        "predictions": p,
+    }
+
+
+def evaluate_naive(cohort: dict, threshold: float | None = None, target_ece: float | None = None) -> PdDetectionResult:
+    """Convenience: train-on-observed (no correction) and score, as a PASS/FAIL result.
+
+    PASS when declined-subpopulation ECE <= ``target_ece``.
+    """
+    threshold = config.POLICY_PD_THRESHOLD if threshold is None else threshold
+    target_ece = config.TARGET_DECLINED_ECE if target_ece is None else target_ece
+    model = fit_observed_model(cohort)
+    scored = score_pd_detection(model, cohort, threshold)
+    declined_ece = scored["declined"].ece
+    return PdDetectionResult(
+        name="pd_default_detection",
+        passed=declined_ece <= target_ece,
+        score=declined_ece,
+        threshold=threshold,
+        details={"all": scored["all"], "approved": scored["approved"], "declined": scored["declined"]},
+    )

cldd/feedback.py

@@ -0,0 +1,191 @@
+"""Closing the loop for real: the model's own decisions create its training data.
+
+``SelectiveLabelsLoop`` measures a *static* question — how well levers correct
+selection imposed by a fixed prior policy. But the lesson of the hackathon
+harness (FABLE.md) is dynamic: once a PD model is deployed, **its own approvals
+decide which outcomes the next model generation gets to learn from**. That
+feedback has a structural property no severity knob reaches: a deterministic
+model policy (fund the lowest-PD fraction) makes selection a *function* of the
+features, so the funded and declined pools stop overlapping **by construction**
+— positivity is not merely strained, it is gone, and every observational
+correction (IPW, retrain) is undefined on the declined side.
+
+``FeedbackLoop`` simulates that regime:
+
+* **Generation 0** funds via the prior underwriter policy at the configured
+  ``selection_severity`` (exactly the world the static loop measures) and trains
+  the first model on those labels.
+* **Generation t >= 1** draws a *fresh* applicant cohort, funds the
+  ``approval_rate`` fraction the previous generation's model scores safest,
+  observes outcomes for funded rows only, and trains the next model on them.
+* With ``exploration_rate=eps > 0``, each generation additionally funds a random
+  ``eps``-fraction of its declines (dedicated RNG stream) and trains with exact
+  labeled-propensity weights — the identification-buying lever from the static
+  loop, now acting as a *stabilizer* of the feedback dynamics.
+
+Per generation we record the planted-truth calibration on the declined (never
+labeled) side, the *observable* funded-book performance, and the observable
+positivity diagnostics — so the harness can show whether the loop's blind spot
+grows while everything a real lender watches still looks healthy.
+
+Design choice: each generation trains on its own cohort's funded rows only (no
+label accumulation across generations). This isolates the selection mechanism —
+any drift is attributable to *who got funded*, not to a growing training set.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+import numpy as np
+
+from . import config, eval_default, model_pd
+from .diagnostics import PositivityDiagnostics, positivity_diagnostics
+from .loop import LeverMetrics, make_generator
+
+
+@dataclass
+class GenerationResult:
+    """One generation of the deployed-model feedback simulation."""
+
+    generation: int
+    #: ``"prior"`` (generation 0) or ``"model"`` (the previous generation's model
+    #: made the funding decisions).
+    policy: str
+    funded_rate: float
+    #: Observable portfolio performance: realized default rate of the funded book
+    #: (including explored loans — they were funded too).
+    funded_default_rate: float
+    #: Planted-truth metrics for this generation's model on ITS OWN blind spot —
+    #: the rows its training never saw. ``declined_*`` = unfunded rows.
+    metrics: LeverMetrics
+    #: Observable positivity diagnostics of the *policy* selection (before
+    #: exploration), i.e. what monitoring would see.
+    diagnostics: PositivityDiagnostics
+    n_explored: int = 0
+    explored_defaults: int = 0
+
+
+@dataclass
+class FeedbackResult:
+    """Full trajectory of a feedback run."""
+
+    generations: list[GenerationResult] = field(default_factory=list)
+    selection_severity: float = 0.0
+    exploration_rate: float = 0.0
+    generator: str = "scm"
+
+    @property
+    def declined_ece_trajectory(self) -> list[float]:
+        return [g.metrics.declined_ece for g in self.generations]
+
+
+class FeedbackLoop:
+    """Simulate model-in-the-loop selective labels across deployment generations."""
+
+    def __init__(
+        self,
+        *,
+        selection_severity: float = 0.4,
+        n_generations: int = 6,
+        exploration_rate: float = 0.0,
+        generator: str = "scm",
+        n_applicants: int = config.DEFAULT_N_APPLICANTS,
+        approval_rate: float = config.DEFAULT_APPROVAL_RATE,
+        seed: int = config.RANDOM_SEED,
+        policy_threshold: float = config.POLICY_PD_THRESHOLD,
+    ) -> None:
+        if not 0.0 <= exploration_rate < 1.0:
+            raise ValueError(f"exploration_rate must be in [0, 1); got {exploration_rate!r}")
+        if n_generations < 1:
+            raise ValueError(f"n_generations must be >= 1; got {n_generations!r}")
+        self.selection_severity = selection_severity
+        self.n_generations = n_generations
+        self.exploration_rate = exploration_rate
+        self.generator = generator
+        self.n_applicants = n_applicants
+        self.approval_rate = approval_rate
+        self.seed = seed
+        self.policy_threshold = policy_threshold
+
+    # ------------------------------------------------------------------ #
+
+    def _model_policy(self, model: model_pd.CalibratedPDModel, X: np.ndarray) -> np.ndarray:
+        """Deterministic deployed policy: fund exactly the ``approval_rate``
+        fraction with the lowest predicted PD.
+
+        Rank-based top-k rather than a quantile cutoff: the isotonic calibrator
+        makes predictions stepwise-constant, so ``p <= quantile(p, rate)`` can
+        fund far more than ``rate`` through ties (observed: 84% at rate 0.6).
+        ``argsort(kind="stable")`` breaks ties by row order — deterministic.
+        """
+        p = model_pd.predict_pd(model, X)
+        k = int(round(self.approval_rate * len(p)))
+        funded = np.zeros(len(p), dtype=bool)
+        funded[np.argsort(p, kind="stable")[:k]] = True
+        return funded
+
+    def _explore(self, policy_funded: np.ndarray, generation: int) -> np.ndarray:
+        if self.exploration_rate <= 0.0:
+            return np.zeros_like(policy_funded)
+        rng = np.random.default_rng([self.seed, generation, config.EXPLORE_STREAM_FEEDBACK])
+        return (~policy_funded) & (rng.random(policy_funded.shape[0]) < self.exploration_rate)
+
+    # ------------------------------------------------------------------ #
+
+    def run(self) -> FeedbackResult:
+        result = FeedbackResult(
+            selection_severity=self.selection_severity,
+            exploration_rate=self.exploration_rate,
+            generator=self.generator,
+        )
+        model: model_pd.CalibratedPDModel | None = None
+
+        for generation in range(self.n_generations):
+            cohort = make_generator(
+                self.generator,
+                severity=self.selection_severity,
+                seed=self.seed + generation,
+                n_applicants=self.n_applicants,
+                approval_rate=self.approval_rate,
+            ).generate_cohort()
+            X = cohort["features"].to_numpy(dtype=float)
+            y = cohort["true_default"]
+
+            if model is None:
+                policy_funded = cohort["approved"]
+                policy = "prior"
+            else:
+                policy_funded = self._model_policy(model, X)
+                policy = "model"
+
+            explored = self._explore(policy_funded, generation)
+            funded = policy_funded | explored
+
+            # Diagnose the POLICY selection (pre-exploration): that is the
+            # selection a deployment would be monitoring.
+            diag = positivity_diagnostics(X, policy_funded, random_state=self.seed + generation)
+
+            if self.exploration_rate > 0.0:
+                weights = np.where(explored, 1.0 / self.exploration_rate, 1.0)[funded]
+            else:
+                weights = None
+            model = model_pd.train_pd_model(
+                X[funded], y[funded], sample_weight=weights, random_state=self.seed + generation
+            )
+
+            scored = eval_default.score_pd_detection(model, cohort, self.policy_threshold, funded=funded)
+            result.generations.append(
+                GenerationResult(
+                    generation=generation,
+                    policy=policy,
+                    funded_rate=float(funded.mean()),
+                    funded_default_rate=float(y[funded].mean()),
+                    metrics=LeverMetrics.from_scored(scored),
+                    diagnostics=diag,
+                    n_explored=int(explored.sum()),
+                    explored_defaults=int(y[explored].sum()),
+                )
+            )
+
+        return result