closed-loop-default-detection 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cldd/__init__.py +96 -0
- cldd/config.py +94 -0
- cldd/correctors.py +172 -0
- cldd/counterfactual.py +767 -0
- cldd/diagnostics.py +107 -0
- cldd/eval_default.py +161 -0
- cldd/feedback.py +191 -0
- cldd/fidelity.py +430 -0
- cldd/loop.py +379 -0
- cldd/model_pd.py +194 -0
- cldd/py.typed +1 -0
- cldd/reject_inference.py +264 -0
- cldd/scm.py +1010 -0
- cldd/synthetic.py +256 -0
- closed_loop_default_detection-0.1.0.dist-info/METADATA +218 -0
- closed_loop_default_detection-0.1.0.dist-info/RECORD +19 -0
- closed_loop_default_detection-0.1.0.dist-info/WHEEL +5 -0
- closed_loop_default_detection-0.1.0.dist-info/licenses/LICENSE +21 -0
- closed_loop_default_detection-0.1.0.dist-info/top_level.txt +1 -0
cldd/synthetic.py
ADDED
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
"""Synthetic SMB applicant cohort generator with planted default ground truth.
|
|
2
|
+
|
|
3
|
+
This is the **generate** stage of the closed loop and the analog of
|
|
4
|
+
``upstream-label-correction/core/synthetic.py``. Where that planted molecular
|
|
5
|
+
mislabels, this plants a true ``default_flag`` for *every* applicant and then
|
|
6
|
+
hides the labels through a prior-underwriter approval policy — exactly the
|
|
7
|
+
**selective-labels** structure of the Intuit SMB challenge.
|
|
8
|
+
|
|
9
|
+
Two ground-truth facts are planted and known only to the harness:
|
|
10
|
+
|
|
11
|
+
1. ``true_default`` — sampled for **all** applicants from a latent risk model
|
|
12
|
+
that is a known function of the observed features *plus a small unobserved
|
|
13
|
+
confounder*. This is the truth a real lender never sees for the declines.
|
|
14
|
+
2. ``approved`` — the prior policy funds the lowest-risk ``approval_rate``
|
|
15
|
+
fraction by a *selection score*. ``selection_severity`` controls how tightly
|
|
16
|
+
that score tracks true risk:
|
|
17
|
+
|
|
18
|
+
* ``severity = 0`` — approval is independent of true risk (selection at
|
|
19
|
+
random; declines and approvals share the same default distribution).
|
|
20
|
+
* ``severity = 1`` — approval tracks the full latent risk, *including the
|
|
21
|
+
unobserved confounder*, so the declined pool is the high-risk tail and part
|
|
22
|
+
of the selection is invisible to any observed-feature propensity model.
|
|
23
|
+
|
|
24
|
+
Severity is the corruption axis the loop escalates to find the model's operating
|
|
25
|
+
frontier — "selection regimes real data cannot probe". All randomness flows
|
|
26
|
+
through one seeded ``numpy.random.Generator`` for byte-identical reproducibility.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
from __future__ import annotations
|
|
30
|
+
|
|
31
|
+
import numpy as np
|
|
32
|
+
import pandas as pd
|
|
33
|
+
|
|
34
|
+
from . import config
|
|
35
|
+
|
|
36
|
+
#: Observed feature columns exposed to the PD model. A realistic subset of the
|
|
37
|
+
#: challenge's 44-column schema — enough to be credible, not exhaustive. Bank-feed
|
|
38
|
+
#: columns are NaN when the applicant did not link a feed (structural missingness
|
|
39
|
+
#: is itself signal).
|
|
40
|
+
FEATURE_COLUMNS = [
|
|
41
|
+
"stated_annual_revenue",
|
|
42
|
+
"requested_amount",
|
|
43
|
+
"vintage_years",
|
|
44
|
+
"aggregate_credit_utilization",
|
|
45
|
+
"existing_debt_obligations",
|
|
46
|
+
"recent_inquiries_count_6mo",
|
|
47
|
+
"prior_loans_count",
|
|
48
|
+
"prior_loans_default_count",
|
|
49
|
+
"has_linked_bank_feed",
|
|
50
|
+
"observed_monthly_revenue_avg_3mo", # bank-feed gated
|
|
51
|
+
"observed_overdraft_count_3mo", # bank-feed gated
|
|
52
|
+
"payroll_regularity_score", # bank-feed gated
|
|
53
|
+
]
|
|
54
|
+
|
|
55
|
+
#: Columns that are null exactly when ``has_linked_bank_feed`` is False.
|
|
56
|
+
BANK_FEED_COLUMNS = [
|
|
57
|
+
"observed_monthly_revenue_avg_3mo",
|
|
58
|
+
"observed_overdraft_count_3mo",
|
|
59
|
+
"payroll_regularity_score",
|
|
60
|
+
]
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _zscore(x: np.ndarray) -> np.ndarray:
|
|
64
|
+
"""NaN-aware standardization; missing entries contribute 0 to the latent risk."""
|
|
65
|
+
x = np.asarray(x, dtype=float)
|
|
66
|
+
mean = np.nanmean(x)
|
|
67
|
+
std = np.nanstd(x)
|
|
68
|
+
if not np.isfinite(std) or std == 0:
|
|
69
|
+
std = 1.0
|
|
70
|
+
z = (x - mean) / std
|
|
71
|
+
return np.nan_to_num(z, nan=0.0)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
class SyntheticBorrowerGenerator:
|
|
75
|
+
"""Generate a synthetic SMB applicant cohort with planted default ground truth."""
|
|
76
|
+
|
|
77
|
+
def __init__(
|
|
78
|
+
self,
|
|
79
|
+
n_applicants: int = config.DEFAULT_N_APPLICANTS,
|
|
80
|
+
selection_severity: float = 0.0,
|
|
81
|
+
approval_rate: float = config.DEFAULT_APPROVAL_RATE,
|
|
82
|
+
target_base_rate: float = config.TARGET_BASE_DEFAULT_RATE,
|
|
83
|
+
bank_feed_rate: float = 0.64,
|
|
84
|
+
unobserved_strength: float = 0.7,
|
|
85
|
+
seed: int = config.RANDOM_SEED,
|
|
86
|
+
) -> None:
|
|
87
|
+
if not 0.0 <= selection_severity <= 1.0:
|
|
88
|
+
raise ValueError(f"selection_severity must be in [0, 1]; got {selection_severity}")
|
|
89
|
+
if not 0.0 < approval_rate < 1.0:
|
|
90
|
+
raise ValueError(f"approval_rate must be in (0, 1); got {approval_rate}")
|
|
91
|
+
self.n_applicants = n_applicants
|
|
92
|
+
self.selection_severity = selection_severity
|
|
93
|
+
self.approval_rate = approval_rate
|
|
94
|
+
self.target_base_rate = target_base_rate
|
|
95
|
+
self.bank_feed_rate = bank_feed_rate
|
|
96
|
+
self.unobserved_strength = unobserved_strength
|
|
97
|
+
self.seed = seed
|
|
98
|
+
self.rng = np.random.Generator(np.random.PCG64(seed))
|
|
99
|
+
|
|
100
|
+
# ------------------------------------------------------------------ #
|
|
101
|
+
# Presets (mirror upstream's unit/benchmark)
|
|
102
|
+
# ------------------------------------------------------------------ #
|
|
103
|
+
|
|
104
|
+
@classmethod
|
|
105
|
+
def unit(cls, seed: int = config.RANDOM_SEED, **kwargs) -> "SyntheticBorrowerGenerator":
|
|
106
|
+
return cls(n_applicants=1500, seed=seed, **kwargs)
|
|
107
|
+
|
|
108
|
+
@classmethod
|
|
109
|
+
def benchmark(cls, seed: int = config.RANDOM_SEED, **kwargs) -> "SyntheticBorrowerGenerator":
|
|
110
|
+
return cls(n_applicants=8000, seed=seed, **kwargs)
|
|
111
|
+
|
|
112
|
+
# ------------------------------------------------------------------ #
|
|
113
|
+
# Public API
|
|
114
|
+
# ------------------------------------------------------------------ #
|
|
115
|
+
|
|
116
|
+
def generate_cohort(self) -> dict:
|
|
117
|
+
"""Return one cohort dict with features + planted ground truth.
|
|
118
|
+
|
|
119
|
+
Keys
|
|
120
|
+
----
|
|
121
|
+
features : pd.DataFrame
|
|
122
|
+
Observed model matrix (``FEATURE_COLUMNS``), NaN for unlinked feeds.
|
|
123
|
+
true_default : np.ndarray[int]
|
|
124
|
+
Planted default outcome for **all** applicants (the hidden truth).
|
|
125
|
+
approved : np.ndarray[bool]
|
|
126
|
+
Prior-policy funding mask; outcomes are observable only here.
|
|
127
|
+
prior_score : np.ndarray[float]
|
|
128
|
+
The selection score (lower = safer = funded first).
|
|
129
|
+
ground_truth : dict
|
|
130
|
+
severity / base_rate / approval_rate / seed / n_applicants.
|
|
131
|
+
"""
|
|
132
|
+
features = self._draw_features()
|
|
133
|
+
latent_risk = self._latent_risk(features) # includes unobserved confounder
|
|
134
|
+
true_default = (self.rng.random(self.n_applicants) < _sigmoid(latent_risk)).astype(int)
|
|
135
|
+
approved, prior_score = self._apply_selection(latent_risk)
|
|
136
|
+
|
|
137
|
+
return {
|
|
138
|
+
"features": features,
|
|
139
|
+
"true_default": true_default,
|
|
140
|
+
"approved": approved,
|
|
141
|
+
"prior_score": prior_score,
|
|
142
|
+
"ground_truth": {
|
|
143
|
+
"selection_severity": self.selection_severity,
|
|
144
|
+
"base_rate": float(true_default.mean()),
|
|
145
|
+
"approval_rate": float(approved.mean()),
|
|
146
|
+
"seed": self.seed,
|
|
147
|
+
"n_applicants": self.n_applicants,
|
|
148
|
+
},
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
# ------------------------------------------------------------------ #
|
|
152
|
+
# Internals
|
|
153
|
+
# ------------------------------------------------------------------ #
|
|
154
|
+
|
|
155
|
+
def _draw_features(self) -> pd.DataFrame:
|
|
156
|
+
rng = self.rng
|
|
157
|
+
n = self.n_applicants
|
|
158
|
+
|
|
159
|
+
stated_annual_revenue = rng.lognormal(mean=12.4, sigma=0.7, size=n) # ~$240k median
|
|
160
|
+
requested_amount = np.clip(rng.lognormal(mean=9.7, sigma=0.55, size=n), 5_000, 50_000)
|
|
161
|
+
vintage_years = rng.gamma(shape=2.0, scale=2.5, size=n)
|
|
162
|
+
aggregate_credit_utilization = np.clip(rng.beta(2.0, 3.0, size=n), 0.0, 1.0)
|
|
163
|
+
existing_debt_obligations = rng.lognormal(mean=10.5, sigma=0.9, size=n)
|
|
164
|
+
recent_inquiries_count_6mo = rng.poisson(lam=1.8, size=n)
|
|
165
|
+
prior_loans_count = rng.poisson(lam=1.2, size=n)
|
|
166
|
+
# Defaults among prior loans rise with utilization (latent link to risk).
|
|
167
|
+
prior_default_p = np.clip(0.10 + 0.25 * aggregate_credit_utilization, 0.0, 1.0)
|
|
168
|
+
prior_loans_default_count = rng.binomial(np.maximum(prior_loans_count, 0), prior_default_p)
|
|
169
|
+
|
|
170
|
+
has_feed = rng.random(n) < self.bank_feed_rate
|
|
171
|
+
observed_monthly_revenue_avg_3mo = stated_annual_revenue / 12.0 * rng.lognormal(0.0, 0.15, size=n)
|
|
172
|
+
observed_overdraft_count_3mo = rng.poisson(lam=0.8, size=n).astype(float)
|
|
173
|
+
payroll_regularity_score = np.clip(rng.beta(5.0, 2.0, size=n), 0.0, 1.0)
|
|
174
|
+
|
|
175
|
+
df = pd.DataFrame(
|
|
176
|
+
{
|
|
177
|
+
"stated_annual_revenue": stated_annual_revenue,
|
|
178
|
+
"requested_amount": requested_amount,
|
|
179
|
+
"vintage_years": vintage_years,
|
|
180
|
+
"aggregate_credit_utilization": aggregate_credit_utilization,
|
|
181
|
+
"existing_debt_obligations": existing_debt_obligations,
|
|
182
|
+
"recent_inquiries_count_6mo": recent_inquiries_count_6mo.astype(float),
|
|
183
|
+
"prior_loans_count": prior_loans_count.astype(float),
|
|
184
|
+
"prior_loans_default_count": prior_loans_default_count.astype(float),
|
|
185
|
+
"has_linked_bank_feed": has_feed.astype(float),
|
|
186
|
+
"observed_monthly_revenue_avg_3mo": observed_monthly_revenue_avg_3mo,
|
|
187
|
+
"observed_overdraft_count_3mo": observed_overdraft_count_3mo,
|
|
188
|
+
"payroll_regularity_score": payroll_regularity_score,
|
|
189
|
+
},
|
|
190
|
+
columns=FEATURE_COLUMNS,
|
|
191
|
+
)
|
|
192
|
+
|
|
193
|
+
# Structural missingness: bank-feed signals are null without a linked feed.
|
|
194
|
+
df.loc[~has_feed, BANK_FEED_COLUMNS] = np.nan
|
|
195
|
+
return df
|
|
196
|
+
|
|
197
|
+
def _latent_risk(self, df: pd.DataFrame) -> np.ndarray:
|
|
198
|
+
"""Known logistic risk function over observed features + an unobserved confounder."""
|
|
199
|
+
debt_to_revenue = df["existing_debt_obligations"].to_numpy() / np.maximum(
|
|
200
|
+
df["stated_annual_revenue"].to_numpy(), 1.0
|
|
201
|
+
)
|
|
202
|
+
prior_default_rate = df["prior_loans_default_count"].to_numpy() / (
|
|
203
|
+
df["prior_loans_count"].to_numpy() + 1.0
|
|
204
|
+
)
|
|
205
|
+
has_feed = df["has_linked_bank_feed"].to_numpy() > 0.5
|
|
206
|
+
|
|
207
|
+
# Unobserved confounder: drives true risk AND (at high severity) selection,
|
|
208
|
+
# but is never exposed to the PD model — this is what bounds the frontier.
|
|
209
|
+
u = self.rng.standard_normal(self.n_applicants)
|
|
210
|
+
|
|
211
|
+
logit = (
|
|
212
|
+
+0.60 * _zscore(df["aggregate_credit_utilization"].to_numpy())
|
|
213
|
+
+ 0.50 * _zscore(debt_to_revenue)
|
|
214
|
+
+ 0.40 * _zscore(df["recent_inquiries_count_6mo"].to_numpy())
|
|
215
|
+
+ 0.40 * _zscore(df["observed_overdraft_count_3mo"].to_numpy())
|
|
216
|
+
+ 0.50 * _zscore(prior_default_rate)
|
|
217
|
+
- 0.40 * _zscore(df["payroll_regularity_score"].to_numpy())
|
|
218
|
+
- 0.30 * _zscore(np.log(df["stated_annual_revenue"].to_numpy()))
|
|
219
|
+
- 0.30 * _zscore(df["vintage_years"].to_numpy())
|
|
220
|
+
+ 0.20 * (~has_feed).astype(float) # no feed => slightly riskier (signal)
|
|
221
|
+
+ self.unobserved_strength * u
|
|
222
|
+
)
|
|
223
|
+
# Shift the intercept so the planted base rate lands near the target.
|
|
224
|
+
logit += _solve_intercept(logit, self.target_base_rate)
|
|
225
|
+
return logit
|
|
226
|
+
|
|
227
|
+
def _apply_selection(self, latent_risk: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
|
|
228
|
+
"""Fund the lowest-risk ``approval_rate`` fraction by a severity-blended score.
|
|
229
|
+
|
|
230
|
+
``prior_score`` blends standardized true latent risk with independent
|
|
231
|
+
noise: at severity 0 it is pure noise (random selection), at severity 1 it
|
|
232
|
+
is the full latent risk (tightest selection, partly on the unobservable).
|
|
233
|
+
"""
|
|
234
|
+
sev = self.selection_severity
|
|
235
|
+
noise = self.rng.standard_normal(self.n_applicants)
|
|
236
|
+
prior_score = sev * _zscore(latent_risk) + (1.0 - sev) * noise # lower == safer
|
|
237
|
+
cutoff = np.quantile(prior_score, self.approval_rate)
|
|
238
|
+
approved = prior_score <= cutoff
|
|
239
|
+
return approved, prior_score
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def _sigmoid(x: np.ndarray) -> np.ndarray:
|
|
243
|
+
return 1.0 / (1.0 + np.exp(-x))
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def _solve_intercept(logit_no_intercept: np.ndarray, target_rate: float) -> float:
|
|
247
|
+
"""Bisection for the intercept ``c`` s.t. ``mean(sigmoid(logit + c)) == target``."""
|
|
248
|
+
lo, hi = -10.0, 10.0
|
|
249
|
+
for _ in range(60):
|
|
250
|
+
mid = 0.5 * (lo + hi)
|
|
251
|
+
rate = float(np.mean(_sigmoid(logit_no_intercept + mid)))
|
|
252
|
+
if rate < target_rate:
|
|
253
|
+
lo = mid
|
|
254
|
+
else:
|
|
255
|
+
hi = mid
|
|
256
|
+
return 0.5 * (lo + hi)
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: closed-loop-default-detection
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: CLUE-style closed loop that measures selective-labels default detection on synthetic SMB lending cohorts and finds the PD model's operating frontier.
|
|
5
|
+
Author-email: Hossain Pazooki <hossain@pazooki.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/hossainpazooki/closed-loop-default-detection
|
|
8
|
+
Project-URL: Repository, https://github.com/hossainpazooki/closed-loop-default-detection
|
|
9
|
+
Project-URL: Issues, https://github.com/hossainpazooki/closed-loop-default-detection/issues
|
|
10
|
+
Keywords: selective-labels,probability-of-default,reject-inference,causal-inference,calibration,synthetic-data,off-policy-evaluation
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: Intended Audience :: Financial and Insurance Industry
|
|
14
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Typing :: Typed
|
|
18
|
+
Requires-Python: >=3.10
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
License-File: LICENSE
|
|
21
|
+
Requires-Dist: numpy>=2.0
|
|
22
|
+
Requires-Dist: pandas>=2.2
|
|
23
|
+
Requires-Dist: scikit-learn>=1.6
|
|
24
|
+
Requires-Dist: scipy>=1.11
|
|
25
|
+
Requires-Dist: matplotlib>=3.8
|
|
26
|
+
Provides-Extra: dev
|
|
27
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
28
|
+
Requires-Dist: pytest-cov>=5; extra == "dev"
|
|
29
|
+
Provides-Extra: docs
|
|
30
|
+
Requires-Dist: sphinx>=7; extra == "docs"
|
|
31
|
+
Requires-Dist: furo; extra == "docs"
|
|
32
|
+
Requires-Dist: myst-parser>=2; extra == "docs"
|
|
33
|
+
Dynamic: license-file
|
|
34
|
+
|
|
35
|
+
# CLDD — closed-loop default detection
|
|
36
|
+
|
|
37
|
+
[](https://github.com/hossainpazooki/closed-loop-default-detection/actions/workflows/ci.yml)
|
|
38
|
+
[](LICENSE)
|
|
39
|
+
[](docs/)
|
|
40
|
+
[](pyproject.toml)
|
|
41
|
+
|
|
42
|
+
**Stress-test a probability-of-default (PD) model under *selective labels* — and get the
|
|
43
|
+
severity at which it breaks.** Real lending data only labels the loans a prior underwriter
|
|
44
|
+
approved, so you cannot measure calibration on the applicants you *declined* — exactly where a
|
|
45
|
+
new model must still be right. CLDD builds synthetic lending worlds with planted ground truth,
|
|
46
|
+
hides labels the way real approval policies do, and grades every correction against that truth.
|
|
47
|
+
|
|
48
|
+
- **Deterministic** — byte-identical per seed, scikit-learn-only, no services or GPUs.
|
|
49
|
+
- **Pluggable** — correction levers (IPW, retrain, exploration, reject inference) are classes;
|
|
50
|
+
add yours by subclassing `Corrector`.
|
|
51
|
+
- **Honest by construction** — every number below recomputes from committed CSVs; limits are
|
|
52
|
+
reported, not smoothed over.
|
|
53
|
+
|
|
54
|
+
## The result it produces
|
|
55
|
+
|
|
56
|
+
The loop escalates selection severity until correction fails and reports the **operating
|
|
57
|
+
frontier** — the last severity at which declined-cohort calibration still holds (target
|
|
58
|
+
ECE ≤ 0.10). From the committed runs (`artifacts/clue_frontier*.csv`, seed 42):
|
|
59
|
+
|
|
60
|
+
| Selection severity | 0.0 | 0.2 | 0.4 | 0.6 |
|
|
61
|
+
|---|---|---|---|---|
|
|
62
|
+
| Naive declined ECE (flat world) | 0.021 | 0.045 | 0.108 | 0.161 |
|
|
63
|
+
| **IPW-corrected** (flat world) | 0.020 | 0.038 | **0.086 ✓** | **0.154 ✗** |
|
|
64
|
+
| **IPW-corrected** (SCM world) | 0.036 | 0.038 | **0.097 ✓** | **0.244 ✗** |
|
|
65
|
+
|
|
66
|
+
Both worlds land the frontier at **severity 0.4**, and the counterfactual deliverable breaks at
|
|
67
|
+
the same boundary: across 25 seeds, g-computation cuts strong-propagation counterfactual MAE
|
|
68
|
+
from 0.099 to 0.086 (−13.5%, positive on 24/25 seeds, Wilcoxon p = 1.5e-7) *inside* the
|
|
69
|
+
frontier — and collapses to a negligible +0.0017 at full severity, where **no deployable
|
|
70
|
+
advantage is claimed**. One cause explains both: selection through an **unobserved
|
|
71
|
+
confounder**, which backdoor adjustment and IPW cannot fix. That single measured limit — not
|
|
72
|
+
an unverifiable score — is the deliverable.
|
|
73
|
+
|
|
74
|
+
Reproduce the headline from committed evidence: `python scripts/paired_significance.py`.
|
|
75
|
+
The full independent assessment (methodology, all numbers, what didn't hold) is the
|
|
76
|
+
accompanying article, [`FABLE.md`](FABLE.md).
|
|
77
|
+
|
|
78
|
+
## Install
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
pip install closed-loop-default-detection
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
The import name is **`cldd`**. For development (tests, docs, the committed evidence),
|
|
85
|
+
install from source:
|
|
86
|
+
|
|
87
|
+
```bash
|
|
88
|
+
git clone https://github.com/hossainpazooki/closed-loop-default-detection.git
|
|
89
|
+
cd closed-loop-default-detection
|
|
90
|
+
pip install -e ".[dev]"
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
Python ≥ 3.10; dependencies are ranges (`numpy>=2.0`, `pandas>=2.2`, `scikit-learn>=1.6`,
|
|
94
|
+
`scipy>=1.11`, `matplotlib>=3.8`) so `cldd` sits alongside your stack. Exact pins for
|
|
95
|
+
float-exact reproduction: [`requirements-dev.txt`](requirements-dev.txt)
|
|
96
|
+
([details](docs/validation.md)).
|
|
97
|
+
|
|
98
|
+
## 60-second tour
|
|
99
|
+
|
|
100
|
+
```python
|
|
101
|
+
from cldd import SelectiveLabelsLoop
|
|
102
|
+
|
|
103
|
+
result = SelectiveLabelsLoop(improve_mode="both").run() # "reweight" | "retrain" | "both"
|
|
104
|
+
print("Operating frontier:", result.frontier_severity)
|
|
105
|
+
for r in result.rounds:
|
|
106
|
+
print(r.selection_severity, r.naive.declined_ece, r.passed)
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
```mermaid
|
|
110
|
+
flowchart TD
|
|
111
|
+
A["<b>1. Generate</b><br/>synthetic cohort at a given selection severity<br/>plant true default, then hide it via the approval policy"]
|
|
112
|
+
B["<b>2. Measure</b><br/>train the PD model on approved rows only,<br/>score it against planted truth on the declined subpopulation"]
|
|
113
|
+
C["<b>3. Improve</b><br/>apply a correction lever:<br/>IPW reweight · disjoint retrain · exploration"]
|
|
114
|
+
D{"Corrected declined-cohort<br/>ECE ≤ target?"}
|
|
115
|
+
E["<b>Operating frontier</b><br/>report the highest severity<br/>that still passes"]
|
|
116
|
+
|
|
117
|
+
A --> B --> C --> D
|
|
118
|
+
D -->|"yes — raise the severity"| A
|
|
119
|
+
D -->|"no — stop"| E
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
A runnable end-to-end demo (classic + custom-lever paths) is
|
|
123
|
+
[`examples/quickstart.py`](examples/quickstart.py). Full mechanics, diagnostics, and the
|
|
124
|
+
feedback simulation: [docs/how-it-works.md](docs/how-it-works.md).
|
|
125
|
+
|
|
126
|
+
> **Scope.** CLDD is a synthetic **validation harness**, not a production pipeline: retraining
|
|
127
|
+
> and feedback are seeded simulations inside the harness; it never acts on live data or real
|
|
128
|
+
> lending decisions.
|
|
129
|
+
|
|
130
|
+
## What's in the box
|
|
131
|
+
|
|
132
|
+
Everything is importable from top-level `cldd` (full reference: the [Sphinx docs](docs/)):
|
|
133
|
+
|
|
134
|
+
| Import | What it is |
|
|
135
|
+
|---|---|
|
|
136
|
+
| `SelectiveLabelsLoop` | the closed loop; `.run()` → `LoopResult` (frontier + per-round metrics) |
|
|
137
|
+
| `Corrector` + `NaiveCorrector`, `IPWReweightCorrector`, `DisjointRetrainCorrector`, `ExplorationCorrector` | the lever ABC and the four built-ins |
|
|
138
|
+
| `ReclassificationCorrector`, `AugmentationCorrector`, `FuzzyAugmentationCorrector`, `ParcellingCorrector` | four classic reject-inference methods, graded against planted truth ([honest results](docs/reject_inference.md)) |
|
|
139
|
+
| `SyntheticBorrowerGenerator`, `StructuralBorrowerGenerator` | the flat and fitted-SCM synthetic worlds |
|
|
140
|
+
| `run_counterfactual_eval`, `GComputationEstimator` | counterfactual validator (g-computation vs naive conditioning) |
|
|
141
|
+
| `FeedbackLoop` | model-in-the-loop selective-labels simulation |
|
|
142
|
+
| `positivity_diagnostics` | observable regime/drift alarm — needs **no** declined-row labels |
|
|
143
|
+
| `CalibratedPDClassifier` | the calibrated PD detector as a scikit-learn estimator |
|
|
144
|
+
| `cldd.fidelity.run_fidelity_gate` | SCM-vs-real **marginal**-fidelity gate (univariate marginals only) |
|
|
145
|
+
|
|
146
|
+
**Add a lever** by subclassing `Corrector` (`name`, `control_priority`, `apply`) and passing
|
|
147
|
+
`correctors=[NaiveCorrector(), MyCorrector()]` — the legacy `improve_mode` API is unchanged
|
|
148
|
+
and byte-identical. Contract details: [CONTRIBUTING.md](CONTRIBUTING.md).
|
|
149
|
+
|
|
150
|
+
**Use the detector from sklearn tooling** — `CalibratedPDClassifier` is a thin, tested wrapper
|
|
151
|
+
(binary-only; NaN features OK; the full `check_estimator` battery passes with zero failed
|
|
152
|
+
checks on scikit-learn 1.7.2–1.9.0; probabilities byte-identical to the research API):
|
|
153
|
+
|
|
154
|
+
```python
|
|
155
|
+
from sklearn.model_selection import cross_val_score
|
|
156
|
+
from cldd import CalibratedPDClassifier
|
|
157
|
+
|
|
158
|
+
scores = cross_val_score(CalibratedPDClassifier(random_state=42), X, y, scoring="neg_brier_score")
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
## Command-line drivers
|
|
162
|
+
|
|
163
|
+
Each driver runs without install (adds `src/` to the path) and writes to `artifacts/`:
|
|
164
|
+
|
|
165
|
+
```bash
|
|
166
|
+
python scripts/run_clue.py # the closed loop → frontier table + plot (--generator scm for the SCM world)
|
|
167
|
+
python scripts/run_seed_sweep.py --quick # counterfactual certification (drop --quick for all seeds)
|
|
168
|
+
python scripts/run_reject_inference.py # reject-inference levers vs the frontier
|
|
169
|
+
python scripts/run_exploration_sweep.py # frontier vs exploration budget
|
|
170
|
+
python scripts/run_feedback.py # model-in-the-loop feedback simulation
|
|
171
|
+
python scripts/paired_significance.py # recompute the headline stat from committed CSVs
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
## Validation
|
|
175
|
+
|
|
176
|
+
`pytest` — 123 tests, all synthetic, no real data needed. CI runs a pinned-repro job (exact
|
|
177
|
+
pins), a cross-version/OS compat matrix, and a strict docs build. Six float-sensitive tests
|
|
178
|
+
reproduce only under the pins in `requirements-dev.txt`; the optional marginal-fidelity gate
|
|
179
|
+
compares the SCM against a **private** real dataset via `CLDD_DATA_DIR` and is the only thing
|
|
180
|
+
that needs it. Details, reproducibility, and troubleshooting:
|
|
181
|
+
[docs/validation.md](docs/validation.md).
|
|
182
|
+
|
|
183
|
+
## Documentation
|
|
184
|
+
|
|
185
|
+
| Where | What |
|
|
186
|
+
|---|---|
|
|
187
|
+
| [docs/quickstart.md](docs/quickstart.md) | run the loop, the counterfactual eval, the fidelity report |
|
|
188
|
+
| [docs/how-it-works.md](docs/how-it-works.md) | loop mechanics, diagnostics, feedback simulation, repo map |
|
|
189
|
+
| [docs/configuration.md](docs/configuration.md) | every knob (`config.py`) and the one env var |
|
|
190
|
+
| [docs/validation.md](docs/validation.md) | tests, gates, reproducibility, troubleshooting |
|
|
191
|
+
| [docs/reject_inference.md](docs/reject_inference.md) | the four RI methods and their honest (modest) results |
|
|
192
|
+
| [`FABLE.md`](FABLE.md) | **the accompanying article** — independent results & methodology assessment |
|
|
193
|
+
|
|
194
|
+
Build locally: `pip install -e ".[docs]" && sphinx-build -b html -W docs docs/_build/html`.
|
|
195
|
+
|
|
196
|
+
## Status
|
|
197
|
+
|
|
198
|
+
`0.1.0` **alpha** on [PyPI](https://pypi.org/project/closed-loop-default-detection/),
|
|
199
|
+
changelog in [CHANGELOG.md](CHANGELOG.md). Shipped: the loop, both worlds, all levers, the
|
|
200
|
+
fidelity gate, the sklearn estimator, CI on three gates. CLDD began as a validation harness
|
|
201
|
+
for the Intuit TechWeek SMB Underwriting Challenge; it is not a submission and does not
|
|
202
|
+
alter challenge files.
|
|
203
|
+
|
|
204
|
+
## Citation
|
|
205
|
+
|
|
206
|
+
Metadata in [`CITATION.cff`](CITATION.cff) (GitHub's "Cite this repository" reads it):
|
|
207
|
+
|
|
208
|
+
```bibtex
|
|
209
|
+
@software{pazooki_cldd_2026,
|
|
210
|
+
author = {Pazooki, Hossain},
|
|
211
|
+
title = {{closed-loop-default-detection}: measuring selective-labels default
|
|
212
|
+
detection and the PD model's operating frontier},
|
|
213
|
+
year = {2026},
|
|
214
|
+
version = {0.1.0},
|
|
215
|
+
license = {MIT},
|
|
216
|
+
url = {https://github.com/hossainpazooki/closed-loop-default-detection}
|
|
217
|
+
}
|
|
218
|
+
```
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
cldd/__init__.py,sha256=BvY-xpcoGzWKfUs7tpgSYciqT44KpxXp6JlpfemQE6g,2761
|
|
2
|
+
cldd/config.py,sha256=H9wbxMlz7SospB3Q9qZhkTwTWcQzEs_Bm2whyTkDFIw,4632
|
|
3
|
+
cldd/correctors.py,sha256=QpJWK64X8ibDe-dBqD0S2EGcqQKvztKzx4M2nfx_hB4,6075
|
|
4
|
+
cldd/counterfactual.py,sha256=eoseTCms2M01EAheNLJ2KcQS_TnpQZivuvyNOnKfGuE,33364
|
|
5
|
+
cldd/diagnostics.py,sha256=7tvJS8E96FUJemdl18ttiIY2LdwBqd3LJXMAMwITF9M,4448
|
|
6
|
+
cldd/eval_default.py,sha256=SZIPJ8a4rGdHZRiY_oqoLUO9WOv09ty1f2a7BfWnBIY,6294
|
|
7
|
+
cldd/feedback.py,sha256=kTRI2UxRQFlIjwVJEgx2yAxeXd084ClKB8rq2Euz5O8,8253
|
|
8
|
+
cldd/fidelity.py,sha256=oEh2jJGT0Fq-_IZjvEBLwfRtRogmgdvC-xTXeFxpjCQ,16810
|
|
9
|
+
cldd/loop.py,sha256=OO0SOplUSeoK56WN8YlML7Rdzu3XPrFL26NYijzpz_s,17168
|
|
10
|
+
cldd/model_pd.py,sha256=etYU8fy2Lmb_nmGIRP5NmxTNbnuFv8FVWNPeSlI5UPE,8551
|
|
11
|
+
cldd/py.typed,sha256=Vuiuf2j9sxEv9xZUVTk0Bx0nmCWPTxTb5KWXBPDNAZ0,54
|
|
12
|
+
cldd/reject_inference.py,sha256=GeBuiUro8KVngAkdYlsE0L9z4fIAc0jHSFVmuCVOPvo,12535
|
|
13
|
+
cldd/scm.py,sha256=QeMBLQZJpmhGtJ07itLjGExDOgsNT7JzGpva0ojpAbk,50397
|
|
14
|
+
cldd/synthetic.py,sha256=F2wv34yX5y2Shj6Bw97bd6IFemLB_KMTBlp7zUSmyEU,11672
|
|
15
|
+
closed_loop_default_detection-0.1.0.dist-info/licenses/LICENSE,sha256=B3dinmRNljbrbsZv4qToUNiWR-NTzPZ4temLVVSvA8U,1072
|
|
16
|
+
closed_loop_default_detection-0.1.0.dist-info/METADATA,sha256=rKfsfsrqDGb8DgvMtOQfaUeTDQR0Ds6V9v45GCj9dmQ,11089
|
|
17
|
+
closed_loop_default_detection-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
|
|
18
|
+
closed_loop_default_detection-0.1.0.dist-info/top_level.txt,sha256=vsEjy8OVY3hmsf5R_aO7IY9z-k5BowRPd4lSx3Mf4Qs,5
|
|
19
|
+
closed_loop_default_detection-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Hossain Pazooki
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
cldd
|