closed-loop-default-detection 0.1.0__py3-none-any.whl

cldd/__init__.py

@@ -0,0 +1,96 @@
+"""Closed-loop default-rate detection — a CLUE-style harness for selective labels.
+
+generate (``synthetic``) -> measure (``eval_default``) -> improve/frontier
+(``loop``). See README.md for the CLUE mapping and how to run.
+"""
+
+from __future__ import annotations
+
+from importlib.metadata import PackageNotFoundError, version as _pkg_version
+
+try:  # pyproject `version` is the single source of truth (read from installed metadata)
+    __version__ = _pkg_version("closed-loop-default-detection")
+except PackageNotFoundError:  # running from a source tree with no install
+    __version__ = "0.1.0"
+
+from .correctors import (
+    CorrectionOutcome,
+    Corrector,
+    CorrectorContext,
+    DisjointRetrainCorrector,
+    ExplorationCorrector,
+    IPWReweightCorrector,
+    NaiveCorrector,
+)
+from .counterfactual import (
+    CounterfactualResult,
+    GComputationEstimator,
+    generate_queries,
+    run_counterfactual_eval,
+)
+from .diagnostics import PositivityDiagnostics, positivity_diagnostics
+from .reject_inference import (
+    AugmentationCorrector,
+    FuzzyAugmentationCorrector,
+    ParcellingCorrector,
+    ReclassificationCorrector,
+    RejectInferenceCorrector,
+)
+from .eval_default import PdDetectionResult, fit_observed_model, score_pd_detection
+from .model_pd import CalibratedPDClassifier, CalibratedPDModel
+from .feedback import FeedbackLoop, FeedbackResult, GenerationResult
+from .loop import LeverMetrics, LoopResult, RoundResult, SelectiveLabelsLoop
+from .scm import (
+    BANK_FEED_COLUMNS,
+    FEATURE_COLUMNS,
+    INTERVENABLE_FEATURES,
+    InterventionResult,
+    SCMState,
+    StructuralBorrowerGenerator,
+    dag_children,
+    dag_parents,
+)
+from .synthetic import SyntheticBorrowerGenerator
+
+__all__ = [
+    "__version__",
+    "SyntheticBorrowerGenerator",
+    "StructuralBorrowerGenerator",
+    "SCMState",
+    "InterventionResult",
+    "INTERVENABLE_FEATURES",
+    "FEATURE_COLUMNS",
+    "BANK_FEED_COLUMNS",
+    "dag_children",
+    "dag_parents",
+    "SelectiveLabelsLoop",
+    "LoopResult",
+    "RoundResult",
+    "LeverMetrics",
+    "Corrector",
+    "CorrectorContext",
+    "CorrectionOutcome",
+    "NaiveCorrector",
+    "IPWReweightCorrector",
+    "DisjointRetrainCorrector",
+    "ExplorationCorrector",
+    "RejectInferenceCorrector",
+    "ReclassificationCorrector",
+    "AugmentationCorrector",
+    "FuzzyAugmentationCorrector",
+    "ParcellingCorrector",
+    "FeedbackLoop",
+    "FeedbackResult",
+    "GenerationResult",
+    "PositivityDiagnostics",
+    "positivity_diagnostics",
+    "PdDetectionResult",
+    "fit_observed_model",
+    "score_pd_detection",
+    "CalibratedPDModel",
+    "CalibratedPDClassifier",
+    "CounterfactualResult",
+    "GComputationEstimator",
+    "run_counterfactual_eval",
+    "generate_queries",
+]

cldd/config.py

@@ -0,0 +1,94 @@
+"""Central configuration: paths, seeds, loan economics, and the frontier grid.
+
+Everything here is either a *fact* (loan terms, the SMB challenge's ~17% base
+default rate) or a single-source-of-truth knob for the closed loop. Per-model
+hyperparameters live in the individual modules so this file stays scannable.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+# --------------------------------------------------------------------------- #
+# Paths
+# --------------------------------------------------------------------------- #
+
+ROOT = Path(__file__).resolve().parents[2]  # repo root
+ARTIFACTS_DIR = ROOT / "artifacts"          # gitignored; frontier CSV + plots
+
+# --------------------------------------------------------------------------- #
+# Reproducibility
+# --------------------------------------------------------------------------- #
+
+RANDOM_SEED = 42
+
+# Disjoint train-cohort offset for the retrain lever. Large enough that train
+# seeds (RANDOM_SEED + TRAIN_SEED_OFFSET + iteration) never collide with measure
+# seeds (RANDOM_SEED + iteration). This is the no-leakage discipline ported from
+# upstream-label-correction/clue/loop.py.
+TRAIN_SEED_OFFSET = 1000
+
+# --------------------------------------------------------------------------- #
+# Synthetic SMB loan economics + portfolio shape (mirrors the challenge brief)
+# --------------------------------------------------------------------------- #
+
+TERM_DAYS = 60                  # daily ACH draws over a 60-day term
+APR = 0.35                      # annualized
+ORIGINATION_FEE_RATE = 0.03     # 3% of amount, collected up front
+
+TARGET_BASE_DEFAULT_RATE = 0.17  # the challenge's labeled pool ran ~17.4%
+DEFAULT_APPROVAL_RATE = 0.60     # fraction of applicants the prior policy funds
+
+# Decision threshold on PD used only for the *detection F1* diagnostic
+# (approve/decline economics are out of scope for this harness).
+POLICY_PD_THRESHOLD = 0.5
+
+# --------------------------------------------------------------------------- #
+# Closed-loop frontier search over selection severity
+# --------------------------------------------------------------------------- #
+
+START_SEVERITY = 0.0    # severity 0 == approval independent of true risk (MAR)
+SEVERITY_STEP = 0.2
+MAX_SEVERITY = 1.0      # severity 1 == approval tracks full latent risk
+MAX_ROUNDS = 8
+
+# A round "passes" when the corrected declined-subpopulation calibration error
+# (ECE) stays at or below this. Lower is better; the frontier is the highest
+# severity still passing.
+TARGET_DECLINED_ECE = 0.10
+
+DEFAULT_N_APPLICANTS = 4000
+
+# --------------------------------------------------------------------------- #
+# Observable positivity-diagnostic thresholds (cldd.diagnostics)
+# --------------------------------------------------------------------------- #
+# Calibrated on this harness's severity grids (flat + SCM, seeds {7, 42, 2026},
+# n=4000): each component individually separates every pass-severity cell
+# (<= 0.4) from every fail-severity cell (>= 0.6) in all 6 grid runs —
+# pass-side worst cases AUC 0.831 / ESS 0.915 / floor 0.0025 vs fail-side worst
+# cases 0.877 / 0.844 / 0.020. The flag detects the positivity-breakdown
+# REGIME, not per-cohort ECE (one flat seed missed the 0.10 ECE target at
+# severity 0.4 with healthy diagnostics). A *proposal* for real-data
+# monitoring, validated only in the two synthetic worlds.
+DIAG_PROPENSITY_AUC_MAX = 0.85
+DIAG_ESS_RATIO_MIN = 0.875
+DIAG_UNFUNDED_BELOW_FLOOR_MAX = 0.01
+
+# Exploration lever: dedicated RNG stream tags so the explore draw can never
+# collide with (or shift) a generator's PCG64 stream or another lever's draw.
+EXPLORE_STREAM_LOOP = 7919
+EXPLORE_STREAM_FEEDBACK = 104729
+
+# --------------------------------------------------------------------------- #
+# Reject-inference correctors (cldd.reject_inference)
+# --------------------------------------------------------------------------- #
+# RI levers are graded against planted truth on a HELD-OUT fold of the declined
+# population (the "recovery of this declined population's labels" claim). The
+# pseudo-label fold and the parcelling label draw get dedicated RNG stream tags,
+# same discipline as the exploration lever, so they can never shift a generator's
+# PCG64 stream or another lever's draw.
+RI_EVAL_FRACTION = 0.5        # fraction of declines held out for grading
+RI_SCORE_BANDS = 10           # quantile score bands for augmentation / parcelling
+RI_PARCEL_FACTOR = 1.5        # parcelling: bad-rate uplift over the accepted band rate
+RI_SPLIT_STREAM = 15485863    # declined train/eval split
+RI_PARCEL_STREAM = 32452843   # parcelling pseudo-label draw

cldd/correctors.py

@@ -0,0 +1,172 @@
+"""Pluggable correction levers for the closed loop.
+
+Each correction lever the loop can apply is a :class:`Corrector` subclass:
+"adding a lever is adding a class". A corrector takes a cohort plus a
+:class:`CorrectorContext` (the per-round scalars + a closure that mints the
+disjoint train cohort) and returns a :class:`CorrectionOutcome` (its
+:class:`LeverMetrics` plus a small ``info`` dict of lever-specific extras).
+
+The four shipped correctors are verbatim extractions of the lever bodies that
+used to live as ``_measure_*`` methods on ``SelectiveLabelsLoop`` — identical
+numpy calls, RNG seeds, sklearn calls and ordering — so the loop's numeric
+output is unchanged.
+
+This module imports only ``config``, ``eval_default``, ``model_pd`` and numpy;
+it must NOT import ``loop`` (loop re-exports ``LeverMetrics`` from here, so the
+reverse dependency would be circular).
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from collections.abc import Callable
+from dataclasses import dataclass, field
+
+import numpy as np
+
+from . import config, eval_default, model_pd
+
+
+@dataclass
+class LeverMetrics:
+    """Declined-subpopulation focus + the all-population calibration for one lever."""
+
+    declined_ece: float
+    declined_auc: float
+    declined_brier: float
+    declined_mean_pd: float
+    declined_base_rate: float
+    declined_f1: float
+    all_ece: float
+
+    @classmethod
+    def from_scored(cls, scored: dict) -> "LeverMetrics":
+        d = scored["declined"]
+        a = scored["all"]
+        return cls(
+            declined_ece=d.ece,
+            declined_auc=d.auc,
+            declined_brier=d.brier,
+            declined_mean_pd=d.mean_pd,
+            declined_base_rate=d.base_rate,
+            declined_f1=d.f1,
+            all_ece=a.ece,
+        )
+
+
+@dataclass(frozen=True)
+class CorrectorContext:
+    """Per-round scalars a corrector may need, plus the train-cohort closure.
+
+    ``make_train_cohort`` returns ``(train_cohort, train_seed)`` — the loop
+    supplies a closure over ``self._generate_train(severity, iteration)`` so the
+    no-leakage disjoint-train discipline stays inside the loop.
+    """
+
+    seed: int
+    policy_threshold: float
+    severity: float
+    iteration: int
+    exploration_rate: float
+    make_train_cohort: Callable[[], tuple[dict, int]]
+
+
+@dataclass
+class CorrectionOutcome:
+    """A corrector's metrics plus its lever-specific extras.
+
+    ``info`` is ``{}`` (naive/reweight), ``{"train_seed": int}`` (retrain), or
+    ``{"n_explored": int, "explored_defaults": int}`` (explore).
+    """
+
+    metrics: LeverMetrics
+    info: dict = field(default_factory=dict)
+
+
+class Corrector(ABC):
+    """A single pluggable correction lever.
+
+    ``name`` keys the lever in ``RoundResult.corrections``; ``control_priority``
+    decides which present corrector drives loop control (highest wins; ties go to
+    the first in the loop's corrector list).
+    """
+
+    name: str = ""
+    control_priority: int = 0
+
+    @abstractmethod
+    def apply(self, cohort: dict, ctx: CorrectorContext) -> CorrectionOutcome:
+        ...
+
+
+class NaiveCorrector(Corrector):
+    """The baseline detector: PD model fitted on approved rows only."""
+
+    name = "naive"
+    control_priority = 0
+
+    def apply(self, cohort: dict, ctx: CorrectorContext) -> CorrectionOutcome:
+        model = eval_default.fit_observed_model(cohort, random_state=ctx.seed)
+        metrics = LeverMetrics.from_scored(
+            eval_default.score_pd_detection(model, cohort, ctx.policy_threshold)
+        )
+        return CorrectionOutcome(metrics=metrics)
+
+
+class IPWReweightCorrector(Corrector):
+    """Inverse-propensity reweighting of the approved training rows."""
+
+    name = "reweight"
+    control_priority = 2
+
+    def apply(self, cohort: dict, ctx: CorrectorContext) -> CorrectionOutcome:
+        X = cohort["features"].to_numpy(dtype=float)
+        approved = cohort["approved"]
+        weights = model_pd.selection_adjusted_weights(X, approved, random_state=ctx.seed)
+        model = eval_default.fit_observed_model(
+            cohort, sample_weight=weights[approved], random_state=ctx.seed
+        )
+        metrics = LeverMetrics.from_scored(
+            eval_default.score_pd_detection(model, cohort, ctx.policy_threshold)
+        )
+        return CorrectionOutcome(metrics=metrics)
+
+
+class DisjointRetrainCorrector(Corrector):
+    """Refit on a disjoint train cohort, score on the held-out measure cohort."""
+
+    name = "retrain"
+    control_priority = 1
+
+    def apply(self, cohort: dict, ctx: CorrectorContext) -> CorrectionOutcome:
+        train_cohort, train_seed = ctx.make_train_cohort()
+        model = eval_default.fit_observed_model(train_cohort, random_state=train_seed)
+        scored = eval_default.score_pd_detection(model, cohort, ctx.policy_threshold)
+        metrics = LeverMetrics.from_scored(scored)
+        return CorrectionOutcome(metrics=metrics, info={"train_seed": train_seed})
+
+
+class ExplorationCorrector(Corrector):
+    """Buy labels on a random slice of the declines and train with exact weights."""
+
+    name = "explore"
+    control_priority = 3
+
+    def apply(self, cohort: dict, ctx: CorrectorContext) -> CorrectionOutcome:
+        rng = np.random.default_rng([ctx.seed, ctx.iteration, config.EXPLORE_STREAM_LOOP])
+        approved = cohort["approved"]
+        explored = (~approved) & (rng.random(approved.shape[0]) < ctx.exploration_rate)
+        funded = approved | explored
+
+        X = cohort["features"].to_numpy(dtype=float)
+        y = cohort["true_default"]
+        weights = np.where(explored, 1.0 / ctx.exploration_rate, 1.0)
+        model = model_pd.train_pd_model(
+            X[funded], y[funded], sample_weight=weights[funded], random_state=ctx.seed
+        )
+        scored = eval_default.score_pd_detection(model, cohort, ctx.policy_threshold, funded=funded)
+        metrics = LeverMetrics.from_scored(scored)
+        return CorrectionOutcome(
+            metrics=metrics,
+            info={"n_explored": int(explored.sum()), "explored_defaults": int(y[explored].sum())},
+        )