closed-loop-default-detection 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- closed_loop_default_detection-0.1.0/LICENSE +21 -0
- closed_loop_default_detection-0.1.0/PKG-INFO +218 -0
- closed_loop_default_detection-0.1.0/README.md +184 -0
- closed_loop_default_detection-0.1.0/pyproject.toml +84 -0
- closed_loop_default_detection-0.1.0/setup.cfg +4 -0
- closed_loop_default_detection-0.1.0/src/cldd/__init__.py +96 -0
- closed_loop_default_detection-0.1.0/src/cldd/config.py +94 -0
- closed_loop_default_detection-0.1.0/src/cldd/correctors.py +172 -0
- closed_loop_default_detection-0.1.0/src/cldd/counterfactual.py +767 -0
- closed_loop_default_detection-0.1.0/src/cldd/diagnostics.py +107 -0
- closed_loop_default_detection-0.1.0/src/cldd/eval_default.py +161 -0
- closed_loop_default_detection-0.1.0/src/cldd/feedback.py +191 -0
- closed_loop_default_detection-0.1.0/src/cldd/fidelity.py +430 -0
- closed_loop_default_detection-0.1.0/src/cldd/loop.py +379 -0
- closed_loop_default_detection-0.1.0/src/cldd/model_pd.py +194 -0
- closed_loop_default_detection-0.1.0/src/cldd/py.typed +1 -0
- closed_loop_default_detection-0.1.0/src/cldd/reject_inference.py +264 -0
- closed_loop_default_detection-0.1.0/src/cldd/scm.py +1010 -0
- closed_loop_default_detection-0.1.0/src/cldd/synthetic.py +256 -0
- closed_loop_default_detection-0.1.0/src/closed_loop_default_detection.egg-info/PKG-INFO +218 -0
- closed_loop_default_detection-0.1.0/src/closed_loop_default_detection.egg-info/SOURCES.txt +37 -0
- closed_loop_default_detection-0.1.0/src/closed_loop_default_detection.egg-info/dependency_links.txt +1 -0
- closed_loop_default_detection-0.1.0/src/closed_loop_default_detection.egg-info/requires.txt +14 -0
- closed_loop_default_detection-0.1.0/src/closed_loop_default_detection.egg-info/top_level.txt +1 -0
- closed_loop_default_detection-0.1.0/tests/test_correctors.py +170 -0
- closed_loop_default_detection-0.1.0/tests/test_counterfactual.py +294 -0
- closed_loop_default_detection-0.1.0/tests/test_diagnostics.py +72 -0
- closed_loop_default_detection-0.1.0/tests/test_eval_default.py +89 -0
- closed_loop_default_detection-0.1.0/tests/test_exploration.py +90 -0
- closed_loop_default_detection-0.1.0/tests/test_feedback.py +118 -0
- closed_loop_default_detection-0.1.0/tests/test_fidelity.py +139 -0
- closed_loop_default_detection-0.1.0/tests/test_fidelity_report.py +107 -0
- closed_loop_default_detection-0.1.0/tests/test_loop.py +88 -0
- closed_loop_default_detection-0.1.0/tests/test_loop_scm.py +159 -0
- closed_loop_default_detection-0.1.0/tests/test_model_pd.py +98 -0
- closed_loop_default_detection-0.1.0/tests/test_reject_inference.py +161 -0
- closed_loop_default_detection-0.1.0/tests/test_scm_gating.py +150 -0
- closed_loop_default_detection-0.1.0/tests/test_sklearn_compat.py +191 -0
- closed_loop_default_detection-0.1.0/tests/test_synthetic.py +70 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Hossain Pazooki
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: closed-loop-default-detection
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: CLUE-style closed loop that measures selective-labels default detection on synthetic SMB lending cohorts and finds the PD model's operating frontier.
|
|
5
|
+
Author-email: Hossain Pazooki <hossain@pazooki.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/hossainpazooki/closed-loop-default-detection
|
|
8
|
+
Project-URL: Repository, https://github.com/hossainpazooki/closed-loop-default-detection
|
|
9
|
+
Project-URL: Issues, https://github.com/hossainpazooki/closed-loop-default-detection/issues
|
|
10
|
+
Keywords: selective-labels,probability-of-default,reject-inference,causal-inference,calibration,synthetic-data,off-policy-evaluation
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: Intended Audience :: Financial and Insurance Industry
|
|
14
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Typing :: Typed
|
|
18
|
+
Requires-Python: >=3.10
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
License-File: LICENSE
|
|
21
|
+
Requires-Dist: numpy>=2.0
|
|
22
|
+
Requires-Dist: pandas>=2.2
|
|
23
|
+
Requires-Dist: scikit-learn>=1.6
|
|
24
|
+
Requires-Dist: scipy>=1.11
|
|
25
|
+
Requires-Dist: matplotlib>=3.8
|
|
26
|
+
Provides-Extra: dev
|
|
27
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
28
|
+
Requires-Dist: pytest-cov>=5; extra == "dev"
|
|
29
|
+
Provides-Extra: docs
|
|
30
|
+
Requires-Dist: sphinx>=7; extra == "docs"
|
|
31
|
+
Requires-Dist: furo; extra == "docs"
|
|
32
|
+
Requires-Dist: myst-parser>=2; extra == "docs"
|
|
33
|
+
Dynamic: license-file
|
|
34
|
+
|
|
35
|
+
# CLDD — closed-loop default detection
|
|
36
|
+
|
|
37
|
+
[](https://github.com/hossainpazooki/closed-loop-default-detection/actions/workflows/ci.yml)
|
|
38
|
+
[](LICENSE)
|
|
39
|
+
[](docs/)
|
|
40
|
+
[](pyproject.toml)
|
|
41
|
+
|
|
42
|
+
**Stress-test a probability-of-default (PD) model under *selective labels* — and get the
|
|
43
|
+
severity at which it breaks.** Real lending data only labels the loans a prior underwriter
|
|
44
|
+
approved, so you cannot measure calibration on the applicants you *declined* — exactly where a
|
|
45
|
+
new model must still be right. CLDD builds synthetic lending worlds with planted ground truth,
|
|
46
|
+
hides labels the way real approval policies do, and grades every correction against that truth.
|
|
47
|
+
|
|
48
|
+
- **Deterministic** — byte-identical per seed, scikit-learn-only, no services or GPUs.
|
|
49
|
+
- **Pluggable** — correction levers (IPW, retrain, exploration, reject inference) are classes;
|
|
50
|
+
add yours by subclassing `Corrector`.
|
|
51
|
+
- **Honest by construction** — every number below recomputes from committed CSVs; limits are
|
|
52
|
+
reported, not smoothed over.
|
|
53
|
+
|
|
54
|
+
## The result it produces
|
|
55
|
+
|
|
56
|
+
The loop escalates selection severity until correction fails and reports the **operating
|
|
57
|
+
frontier** — the last severity at which declined-cohort calibration still holds (target
|
|
58
|
+
ECE ≤ 0.10). From the committed runs (`artifacts/clue_frontier*.csv`, seed 42):
|
|
59
|
+
|
|
60
|
+
| Selection severity | 0.0 | 0.2 | 0.4 | 0.6 |
|
|
61
|
+
|---|---|---|---|---|
|
|
62
|
+
| Naive declined ECE (flat world) | 0.021 | 0.045 | 0.108 | 0.161 |
|
|
63
|
+
| **IPW-corrected** (flat world) | 0.020 | 0.038 | **0.086 ✓** | **0.154 ✗** |
|
|
64
|
+
| **IPW-corrected** (SCM world) | 0.036 | 0.038 | **0.097 ✓** | **0.244 ✗** |
|
|
65
|
+
|
|
66
|
+
Both worlds land the frontier at **severity 0.4**, and the counterfactual deliverable breaks at
|
|
67
|
+
the same boundary: across 25 seeds, g-computation cuts strong-propagation counterfactual MAE
|
|
68
|
+
from 0.099 to 0.086 (−13.5%, positive on 24/25 seeds, Wilcoxon p = 1.5e-7) *inside* the
|
|
69
|
+
frontier — and collapses to a negligible +0.0017 at full severity, where **no deployable
|
|
70
|
+
advantage is claimed**. One cause explains both: selection through an **unobserved
|
|
71
|
+
confounder**, which backdoor adjustment and IPW cannot fix. That single measured limit — not
|
|
72
|
+
an unverifiable score — is the deliverable.
|
|
73
|
+
|
|
74
|
+
Reproduce the headline from committed evidence: `python scripts/paired_significance.py`.
|
|
75
|
+
The full independent assessment (methodology, all numbers, what didn't hold) is the
|
|
76
|
+
accompanying article, [`FABLE.md`](FABLE.md).
|
|
77
|
+
|
|
78
|
+
## Install
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
pip install closed-loop-default-detection
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
The import name is **`cldd`**. For development (tests, docs, the committed evidence),
|
|
85
|
+
install from source:
|
|
86
|
+
|
|
87
|
+
```bash
|
|
88
|
+
git clone https://github.com/hossainpazooki/closed-loop-default-detection.git
|
|
89
|
+
cd closed-loop-default-detection
|
|
90
|
+
pip install -e ".[dev]"
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
Python ≥ 3.10; dependencies are ranges (`numpy>=2.0`, `pandas>=2.2`, `scikit-learn>=1.6`,
|
|
94
|
+
`scipy>=1.11`, `matplotlib>=3.8`) so `cldd` sits alongside your stack. Exact pins for
|
|
95
|
+
float-exact reproduction: [`requirements-dev.txt`](requirements-dev.txt)
|
|
96
|
+
([details](docs/validation.md)).
|
|
97
|
+
|
|
98
|
+
## 60-second tour
|
|
99
|
+
|
|
100
|
+
```python
|
|
101
|
+
from cldd import SelectiveLabelsLoop
|
|
102
|
+
|
|
103
|
+
result = SelectiveLabelsLoop(improve_mode="both").run() # "reweight" | "retrain" | "both"
|
|
104
|
+
print("Operating frontier:", result.frontier_severity)
|
|
105
|
+
for r in result.rounds:
|
|
106
|
+
print(r.selection_severity, r.naive.declined_ece, r.passed)
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
```mermaid
|
|
110
|
+
flowchart TD
|
|
111
|
+
A["<b>1. Generate</b><br/>synthetic cohort at a given selection severity<br/>plant true default, then hide it via the approval policy"]
|
|
112
|
+
B["<b>2. Measure</b><br/>train the PD model on approved rows only,<br/>score it against planted truth on the declined subpopulation"]
|
|
113
|
+
C["<b>3. Improve</b><br/>apply a correction lever:<br/>IPW reweight · disjoint retrain · exploration"]
|
|
114
|
+
D{"Corrected declined-cohort<br/>ECE ≤ target?"}
|
|
115
|
+
E["<b>Operating frontier</b><br/>report the highest severity<br/>that still passes"]
|
|
116
|
+
|
|
117
|
+
A --> B --> C --> D
|
|
118
|
+
D -->|"yes — raise the severity"| A
|
|
119
|
+
D -->|"no — stop"| E
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
A runnable end-to-end demo (classic + custom-lever paths) is
|
|
123
|
+
[`examples/quickstart.py`](examples/quickstart.py). Full mechanics, diagnostics, and the
|
|
124
|
+
feedback simulation: [docs/how-it-works.md](docs/how-it-works.md).
|
|
125
|
+
|
|
126
|
+
> **Scope.** CLDD is a synthetic **validation harness**, not a production pipeline: retraining
|
|
127
|
+
> and feedback are seeded simulations inside the harness; it never acts on live data or real
|
|
128
|
+
> lending decisions.
|
|
129
|
+
|
|
130
|
+
## What's in the box
|
|
131
|
+
|
|
132
|
+
Everything is importable from top-level `cldd` (full reference: the [Sphinx docs](docs/)):
|
|
133
|
+
|
|
134
|
+
| Import | What it is |
|
|
135
|
+
|---|---|
|
|
136
|
+
| `SelectiveLabelsLoop` | the closed loop; `.run()` → `LoopResult` (frontier + per-round metrics) |
|
|
137
|
+
| `Corrector` + `NaiveCorrector`, `IPWReweightCorrector`, `DisjointRetrainCorrector`, `ExplorationCorrector` | the lever ABC and the four built-ins |
|
|
138
|
+
| `ReclassificationCorrector`, `AugmentationCorrector`, `FuzzyAugmentationCorrector`, `ParcellingCorrector` | four classic reject-inference methods, graded against planted truth ([honest results](docs/reject_inference.md)) |
|
|
139
|
+
| `SyntheticBorrowerGenerator`, `StructuralBorrowerGenerator` | the flat and fitted-SCM synthetic worlds |
|
|
140
|
+
| `run_counterfactual_eval`, `GComputationEstimator` | counterfactual validator (g-computation vs naive conditioning) |
|
|
141
|
+
| `FeedbackLoop` | model-in-the-loop selective-labels simulation |
|
|
142
|
+
| `positivity_diagnostics` | observable regime/drift alarm — needs **no** declined-row labels |
|
|
143
|
+
| `CalibratedPDClassifier` | the calibrated PD detector as a scikit-learn estimator |
|
|
144
|
+
| `cldd.fidelity.run_fidelity_gate` | SCM-vs-real **marginal**-fidelity gate (univariate marginals only) |
|
|
145
|
+
|
|
146
|
+
**Add a lever** by subclassing `Corrector` (`name`, `control_priority`, `apply`) and passing
|
|
147
|
+
`correctors=[NaiveCorrector(), MyCorrector()]` — the legacy `improve_mode` API is unchanged
|
|
148
|
+
and byte-identical. Contract details: [CONTRIBUTING.md](CONTRIBUTING.md).
|
|
149
|
+
|
|
150
|
+
**Use the detector from sklearn tooling** — `CalibratedPDClassifier` is a thin, tested wrapper
|
|
151
|
+
(binary-only; NaN features OK; the full `check_estimator` battery passes with zero failed
|
|
152
|
+
checks on scikit-learn 1.7.2–1.9.0; probabilities byte-identical to the research API):
|
|
153
|
+
|
|
154
|
+
```python
|
|
155
|
+
from sklearn.model_selection import cross_val_score
|
|
156
|
+
from cldd import CalibratedPDClassifier
|
|
157
|
+
|
|
158
|
+
scores = cross_val_score(CalibratedPDClassifier(random_state=42), X, y, scoring="neg_brier_score")
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
## Command-line drivers
|
|
162
|
+
|
|
163
|
+
Each driver runs without install (adds `src/` to the path) and writes to `artifacts/`:
|
|
164
|
+
|
|
165
|
+
```bash
|
|
166
|
+
python scripts/run_clue.py # the closed loop → frontier table + plot (--generator scm for the SCM world)
|
|
167
|
+
python scripts/run_seed_sweep.py --quick # counterfactual certification (drop --quick for all seeds)
|
|
168
|
+
python scripts/run_reject_inference.py # reject-inference levers vs the frontier
|
|
169
|
+
python scripts/run_exploration_sweep.py # frontier vs exploration budget
|
|
170
|
+
python scripts/run_feedback.py # model-in-the-loop feedback simulation
|
|
171
|
+
python scripts/paired_significance.py # recompute the headline stat from committed CSVs
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
## Validation
|
|
175
|
+
|
|
176
|
+
`pytest` — 123 tests, all synthetic, no real data needed. CI runs a pinned-repro job (exact
|
|
177
|
+
pins), a cross-version/OS compat matrix, and a strict docs build. Six float-sensitive tests
|
|
178
|
+
reproduce only under the pins in `requirements-dev.txt`; the optional marginal-fidelity gate
|
|
179
|
+
compares the SCM against a **private** real dataset via `CLDD_DATA_DIR` and is the only thing
|
|
180
|
+
that needs it. Details, reproducibility, and troubleshooting:
|
|
181
|
+
[docs/validation.md](docs/validation.md).
|
|
182
|
+
|
|
183
|
+
## Documentation
|
|
184
|
+
|
|
185
|
+
| Where | What |
|
|
186
|
+
|---|---|
|
|
187
|
+
| [docs/quickstart.md](docs/quickstart.md) | run the loop, the counterfactual eval, the fidelity report |
|
|
188
|
+
| [docs/how-it-works.md](docs/how-it-works.md) | loop mechanics, diagnostics, feedback simulation, repo map |
|
|
189
|
+
| [docs/configuration.md](docs/configuration.md) | every knob (`config.py`) and the one env var |
|
|
190
|
+
| [docs/validation.md](docs/validation.md) | tests, gates, reproducibility, troubleshooting |
|
|
191
|
+
| [docs/reject_inference.md](docs/reject_inference.md) | the four RI methods and their honest (modest) results |
|
|
192
|
+
| [`FABLE.md`](FABLE.md) | **the accompanying article** — independent results & methodology assessment |
|
|
193
|
+
|
|
194
|
+
Build locally: `pip install -e ".[docs]" && sphinx-build -b html -W docs docs/_build/html`.
|
|
195
|
+
|
|
196
|
+
## Status
|
|
197
|
+
|
|
198
|
+
`0.1.0` **alpha** on [PyPI](https://pypi.org/project/closed-loop-default-detection/),
|
|
199
|
+
changelog in [CHANGELOG.md](CHANGELOG.md). Shipped: the loop, both worlds, all levers, the
|
|
200
|
+
fidelity gate, the sklearn estimator, CI on three gates. CLDD began as a validation harness
|
|
201
|
+
for the Intuit TechWeek SMB Underwriting Challenge; it is not a submission and does not
|
|
202
|
+
alter challenge files.
|
|
203
|
+
|
|
204
|
+
## Citation
|
|
205
|
+
|
|
206
|
+
Metadata in [`CITATION.cff`](CITATION.cff) (GitHub's "Cite this repository" reads it):
|
|
207
|
+
|
|
208
|
+
```bibtex
|
|
209
|
+
@software{pazooki_cldd_2026,
|
|
210
|
+
author = {Pazooki, Hossain},
|
|
211
|
+
title = {{closed-loop-default-detection}: measuring selective-labels default
|
|
212
|
+
detection and the PD model's operating frontier},
|
|
213
|
+
year = {2026},
|
|
214
|
+
version = {0.1.0},
|
|
215
|
+
license = {MIT},
|
|
216
|
+
url = {https://github.com/hossainpazooki/closed-loop-default-detection}
|
|
217
|
+
}
|
|
218
|
+
```
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
# CLDD — closed-loop default detection
|
|
2
|
+
|
|
3
|
+
[](https://github.com/hossainpazooki/closed-loop-default-detection/actions/workflows/ci.yml)
|
|
4
|
+
[](LICENSE)
|
|
5
|
+
[](docs/)
|
|
6
|
+
[](pyproject.toml)
|
|
7
|
+
|
|
8
|
+
**Stress-test a probability-of-default (PD) model under *selective labels* — and get the
|
|
9
|
+
severity at which it breaks.** Real lending data only labels the loans a prior underwriter
|
|
10
|
+
approved, so you cannot measure calibration on the applicants you *declined* — exactly where a
|
|
11
|
+
new model must still be right. CLDD builds synthetic lending worlds with planted ground truth,
|
|
12
|
+
hides labels the way real approval policies do, and grades every correction against that truth.
|
|
13
|
+
|
|
14
|
+
- **Deterministic** — byte-identical per seed, scikit-learn-only, no services or GPUs.
|
|
15
|
+
- **Pluggable** — correction levers (IPW, retrain, exploration, reject inference) are classes;
|
|
16
|
+
add yours by subclassing `Corrector`.
|
|
17
|
+
- **Honest by construction** — every number below recomputes from committed CSVs; limits are
|
|
18
|
+
reported, not smoothed over.
|
|
19
|
+
|
|
20
|
+
## The result it produces
|
|
21
|
+
|
|
22
|
+
The loop escalates selection severity until correction fails and reports the **operating
|
|
23
|
+
frontier** — the last severity at which declined-cohort calibration still holds (target
|
|
24
|
+
ECE ≤ 0.10). From the committed runs (`artifacts/clue_frontier*.csv`, seed 42):
|
|
25
|
+
|
|
26
|
+
| Selection severity | 0.0 | 0.2 | 0.4 | 0.6 |
|
|
27
|
+
|---|---|---|---|---|
|
|
28
|
+
| Naive declined ECE (flat world) | 0.021 | 0.045 | 0.108 | 0.161 |
|
|
29
|
+
| **IPW-corrected** (flat world) | 0.020 | 0.038 | **0.086 ✓** | **0.154 ✗** |
|
|
30
|
+
| **IPW-corrected** (SCM world) | 0.036 | 0.038 | **0.097 ✓** | **0.244 ✗** |
|
|
31
|
+
|
|
32
|
+
Both worlds land the frontier at **severity 0.4**, and the counterfactual deliverable breaks at
|
|
33
|
+
the same boundary: across 25 seeds, g-computation cuts strong-propagation counterfactual MAE
|
|
34
|
+
from 0.099 to 0.086 (−13.5%, positive on 24/25 seeds, Wilcoxon p = 1.5e-7) *inside* the
|
|
35
|
+
frontier — and collapses to a negligible +0.0017 at full severity, where **no deployable
|
|
36
|
+
advantage is claimed**. One cause explains both: selection through an **unobserved
|
|
37
|
+
confounder**, which backdoor adjustment and IPW cannot fix. That single measured limit — not
|
|
38
|
+
an unverifiable score — is the deliverable.
|
|
39
|
+
|
|
40
|
+
Reproduce the headline from committed evidence: `python scripts/paired_significance.py`.
|
|
41
|
+
The full independent assessment (methodology, all numbers, what didn't hold) is the
|
|
42
|
+
accompanying article, [`FABLE.md`](FABLE.md).
|
|
43
|
+
|
|
44
|
+
## Install
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
pip install closed-loop-default-detection
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
The import name is **`cldd`**. For development (tests, docs, the committed evidence),
|
|
51
|
+
install from source:
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
git clone https://github.com/hossainpazooki/closed-loop-default-detection.git
|
|
55
|
+
cd closed-loop-default-detection
|
|
56
|
+
pip install -e ".[dev]"
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
Python ≥ 3.10; dependencies are ranges (`numpy>=2.0`, `pandas>=2.2`, `scikit-learn>=1.6`,
|
|
60
|
+
`scipy>=1.11`, `matplotlib>=3.8`) so `cldd` sits alongside your stack. Exact pins for
|
|
61
|
+
float-exact reproduction: [`requirements-dev.txt`](requirements-dev.txt)
|
|
62
|
+
([details](docs/validation.md)).
|
|
63
|
+
|
|
64
|
+
## 60-second tour
|
|
65
|
+
|
|
66
|
+
```python
|
|
67
|
+
from cldd import SelectiveLabelsLoop
|
|
68
|
+
|
|
69
|
+
result = SelectiveLabelsLoop(improve_mode="both").run() # "reweight" | "retrain" | "both"
|
|
70
|
+
print("Operating frontier:", result.frontier_severity)
|
|
71
|
+
for r in result.rounds:
|
|
72
|
+
print(r.selection_severity, r.naive.declined_ece, r.passed)
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
```mermaid
|
|
76
|
+
flowchart TD
|
|
77
|
+
A["<b>1. Generate</b><br/>synthetic cohort at a given selection severity<br/>plant true default, then hide it via the approval policy"]
|
|
78
|
+
B["<b>2. Measure</b><br/>train the PD model on approved rows only,<br/>score it against planted truth on the declined subpopulation"]
|
|
79
|
+
C["<b>3. Improve</b><br/>apply a correction lever:<br/>IPW reweight · disjoint retrain · exploration"]
|
|
80
|
+
D{"Corrected declined-cohort<br/>ECE ≤ target?"}
|
|
81
|
+
E["<b>Operating frontier</b><br/>report the highest severity<br/>that still passes"]
|
|
82
|
+
|
|
83
|
+
A --> B --> C --> D
|
|
84
|
+
D -->|"yes — raise the severity"| A
|
|
85
|
+
D -->|"no — stop"| E
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
A runnable end-to-end demo (classic + custom-lever paths) is
|
|
89
|
+
[`examples/quickstart.py`](examples/quickstart.py). Full mechanics, diagnostics, and the
|
|
90
|
+
feedback simulation: [docs/how-it-works.md](docs/how-it-works.md).
|
|
91
|
+
|
|
92
|
+
> **Scope.** CLDD is a synthetic **validation harness**, not a production pipeline: retraining
|
|
93
|
+
> and feedback are seeded simulations inside the harness; it never acts on live data or real
|
|
94
|
+
> lending decisions.
|
|
95
|
+
|
|
96
|
+
## What's in the box
|
|
97
|
+
|
|
98
|
+
Everything is importable from top-level `cldd` (full reference: the [Sphinx docs](docs/)):
|
|
99
|
+
|
|
100
|
+
| Import | What it is |
|
|
101
|
+
|---|---|
|
|
102
|
+
| `SelectiveLabelsLoop` | the closed loop; `.run()` → `LoopResult` (frontier + per-round metrics) |
|
|
103
|
+
| `Corrector` + `NaiveCorrector`, `IPWReweightCorrector`, `DisjointRetrainCorrector`, `ExplorationCorrector` | the lever ABC and the four built-ins |
|
|
104
|
+
| `ReclassificationCorrector`, `AugmentationCorrector`, `FuzzyAugmentationCorrector`, `ParcellingCorrector` | four classic reject-inference methods, graded against planted truth ([honest results](docs/reject_inference.md)) |
|
|
105
|
+
| `SyntheticBorrowerGenerator`, `StructuralBorrowerGenerator` | the flat and fitted-SCM synthetic worlds |
|
|
106
|
+
| `run_counterfactual_eval`, `GComputationEstimator` | counterfactual validator (g-computation vs naive conditioning) |
|
|
107
|
+
| `FeedbackLoop` | model-in-the-loop selective-labels simulation |
|
|
108
|
+
| `positivity_diagnostics` | observable regime/drift alarm — needs **no** declined-row labels |
|
|
109
|
+
| `CalibratedPDClassifier` | the calibrated PD detector as a scikit-learn estimator |
|
|
110
|
+
| `cldd.fidelity.run_fidelity_gate` | SCM-vs-real **marginal**-fidelity gate (univariate marginals only) |
|
|
111
|
+
|
|
112
|
+
**Add a lever** by subclassing `Corrector` (`name`, `control_priority`, `apply`) and passing
|
|
113
|
+
`correctors=[NaiveCorrector(), MyCorrector()]` — the legacy `improve_mode` API is unchanged
|
|
114
|
+
and byte-identical. Contract details: [CONTRIBUTING.md](CONTRIBUTING.md).
|
|
115
|
+
|
|
116
|
+
**Use the detector from sklearn tooling** — `CalibratedPDClassifier` is a thin, tested wrapper
|
|
117
|
+
(binary-only; NaN features OK; the full `check_estimator` battery passes with zero failed
|
|
118
|
+
checks on scikit-learn 1.7.2–1.9.0; probabilities byte-identical to the research API):
|
|
119
|
+
|
|
120
|
+
```python
|
|
121
|
+
from sklearn.model_selection import cross_val_score
|
|
122
|
+
from cldd import CalibratedPDClassifier
|
|
123
|
+
|
|
124
|
+
scores = cross_val_score(CalibratedPDClassifier(random_state=42), X, y, scoring="neg_brier_score")
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
## Command-line drivers
|
|
128
|
+
|
|
129
|
+
Each driver runs without install (adds `src/` to the path) and writes to `artifacts/`:
|
|
130
|
+
|
|
131
|
+
```bash
|
|
132
|
+
python scripts/run_clue.py # the closed loop → frontier table + plot (--generator scm for the SCM world)
|
|
133
|
+
python scripts/run_seed_sweep.py --quick # counterfactual certification (drop --quick for all seeds)
|
|
134
|
+
python scripts/run_reject_inference.py # reject-inference levers vs the frontier
|
|
135
|
+
python scripts/run_exploration_sweep.py # frontier vs exploration budget
|
|
136
|
+
python scripts/run_feedback.py # model-in-the-loop feedback simulation
|
|
137
|
+
python scripts/paired_significance.py # recompute the headline stat from committed CSVs
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
## Validation
|
|
141
|
+
|
|
142
|
+
`pytest` — 123 tests, all synthetic, no real data needed. CI runs a pinned-repro job (exact
|
|
143
|
+
pins), a cross-version/OS compat matrix, and a strict docs build. Six float-sensitive tests
|
|
144
|
+
reproduce only under the pins in `requirements-dev.txt`; the optional marginal-fidelity gate
|
|
145
|
+
compares the SCM against a **private** real dataset via `CLDD_DATA_DIR` and is the only thing
|
|
146
|
+
that needs it. Details, reproducibility, and troubleshooting:
|
|
147
|
+
[docs/validation.md](docs/validation.md).
|
|
148
|
+
|
|
149
|
+
## Documentation
|
|
150
|
+
|
|
151
|
+
| Where | What |
|
|
152
|
+
|---|---|
|
|
153
|
+
| [docs/quickstart.md](docs/quickstart.md) | run the loop, the counterfactual eval, the fidelity report |
|
|
154
|
+
| [docs/how-it-works.md](docs/how-it-works.md) | loop mechanics, diagnostics, feedback simulation, repo map |
|
|
155
|
+
| [docs/configuration.md](docs/configuration.md) | every knob (`config.py`) and the one env var |
|
|
156
|
+
| [docs/validation.md](docs/validation.md) | tests, gates, reproducibility, troubleshooting |
|
|
157
|
+
| [docs/reject_inference.md](docs/reject_inference.md) | the four RI methods and their honest (modest) results |
|
|
158
|
+
| [`FABLE.md`](FABLE.md) | **the accompanying article** — independent results & methodology assessment |
|
|
159
|
+
|
|
160
|
+
Build locally: `pip install -e ".[docs]" && sphinx-build -b html -W docs docs/_build/html`.
|
|
161
|
+
|
|
162
|
+
## Status
|
|
163
|
+
|
|
164
|
+
`0.1.0` **alpha** on [PyPI](https://pypi.org/project/closed-loop-default-detection/),
|
|
165
|
+
changelog in [CHANGELOG.md](CHANGELOG.md). Shipped: the loop, both worlds, all levers, the
|
|
166
|
+
fidelity gate, the sklearn estimator, CI on three gates. CLDD began as a validation harness
|
|
167
|
+
for the Intuit TechWeek SMB Underwriting Challenge; it is not a submission and does not
|
|
168
|
+
alter challenge files.
|
|
169
|
+
|
|
170
|
+
## Citation
|
|
171
|
+
|
|
172
|
+
Metadata in [`CITATION.cff`](CITATION.cff) (GitHub's "Cite this repository" reads it):
|
|
173
|
+
|
|
174
|
+
```bibtex
|
|
175
|
+
@software{pazooki_cldd_2026,
|
|
176
|
+
author = {Pazooki, Hossain},
|
|
177
|
+
title = {{closed-loop-default-detection}: measuring selective-labels default
|
|
178
|
+
detection and the PD model's operating frontier},
|
|
179
|
+
year = {2026},
|
|
180
|
+
version = {0.1.0},
|
|
181
|
+
license = {MIT},
|
|
182
|
+
url = {https://github.com/hossainpazooki/closed-loop-default-detection}
|
|
183
|
+
}
|
|
184
|
+
```
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "closed-loop-default-detection"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "CLUE-style closed loop that measures selective-labels default detection on synthetic SMB lending cohorts and finds the PD model's operating frontier."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{ name = "Hossain Pazooki", email = "hossain@pazooki.com" }]
|
|
14
|
+
keywords = [
|
|
15
|
+
"selective-labels",
|
|
16
|
+
"probability-of-default",
|
|
17
|
+
"reject-inference",
|
|
18
|
+
"causal-inference",
|
|
19
|
+
"calibration",
|
|
20
|
+
"synthetic-data",
|
|
21
|
+
"off-policy-evaluation",
|
|
22
|
+
]
|
|
23
|
+
classifiers = [
|
|
24
|
+
"Development Status :: 3 - Alpha",
|
|
25
|
+
"Intended Audience :: Science/Research",
|
|
26
|
+
"Intended Audience :: Financial and Insurance Industry",
|
|
27
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
28
|
+
"Programming Language :: Python :: 3",
|
|
29
|
+
"Operating System :: OS Independent",
|
|
30
|
+
"Typing :: Typed",
|
|
31
|
+
]
|
|
32
|
+
# Library dependencies are declared as ranges so the package installs alongside
|
|
33
|
+
# a practitioner's own stack. The EXACT provenance pins under which the committed
|
|
34
|
+
# artifacts and the frozen byte-identity baseline were captured
|
|
35
|
+
# (scikit-learn 1.9.0 / numpy 2.4.6, Python 3.14.2) live in requirements-dev.txt
|
|
36
|
+
# and are exercised by the CI `pinned-repro` job. HistGradientBoosting float
|
|
37
|
+
# output shifts across scikit-learn releases, so a handful of float-exact tests
|
|
38
|
+
# (marked `pinned`) are reproducible only under those pins.
|
|
39
|
+
dependencies = [
|
|
40
|
+
"numpy>=2.0",
|
|
41
|
+
"pandas>=2.2",
|
|
42
|
+
"scikit-learn>=1.6",
|
|
43
|
+
"scipy>=1.11",
|
|
44
|
+
"matplotlib>=3.8",
|
|
45
|
+
]
|
|
46
|
+
|
|
47
|
+
[project.optional-dependencies]
|
|
48
|
+
dev = ["pytest>=8.0", "pytest-cov>=5"]
|
|
49
|
+
docs = ["sphinx>=7", "furo", "myst-parser>=2"]
|
|
50
|
+
|
|
51
|
+
[project.urls]
|
|
52
|
+
Homepage = "https://github.com/hossainpazooki/closed-loop-default-detection"
|
|
53
|
+
Repository = "https://github.com/hossainpazooki/closed-loop-default-detection"
|
|
54
|
+
Issues = "https://github.com/hossainpazooki/closed-loop-default-detection/issues"
|
|
55
|
+
|
|
56
|
+
[tool.setuptools.packages.find]
|
|
57
|
+
where = ["src"]
|
|
58
|
+
|
|
59
|
+
# Ship the PEP 561 typing marker so downstream type-checkers honor cldd's hints.
|
|
60
|
+
[tool.setuptools.package-data]
|
|
61
|
+
cldd = ["py.typed"]
|
|
62
|
+
|
|
63
|
+
[tool.pytest.ini_options]
|
|
64
|
+
testpaths = ["tests"]
|
|
65
|
+
addopts = "-q"
|
|
66
|
+
markers = [
|
|
67
|
+
"pinned: asserts exact or tight floating-point output of the calibrated PD model; reproducible only under the pinned scikit-learn/numpy in requirements-dev.txt. The CI compat matrix deselects these with `-m 'not pinned'`.",
|
|
68
|
+
]
|
|
69
|
+
|
|
70
|
+
# Coverage is opt-in (run `pytest --cov=cldd`); it is not wired into addopts so a
|
|
71
|
+
# plain `pytest` stays lightweight. The private-data fidelity path (cldd/fidelity.py)
|
|
72
|
+
# is only exercised when the real dataset is present, so its data-loading branch is
|
|
73
|
+
# expected to be uncovered on public CI by design — see README.
|
|
74
|
+
[tool.coverage.run]
|
|
75
|
+
source = ["cldd"]
|
|
76
|
+
branch = true
|
|
77
|
+
|
|
78
|
+
[tool.coverage.report]
|
|
79
|
+
show_missing = true
|
|
80
|
+
exclude_also = [
|
|
81
|
+
"if TYPE_CHECKING:",
|
|
82
|
+
"raise NotImplementedError",
|
|
83
|
+
"if __name__ == .__main__.:",
|
|
84
|
+
]
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
"""Closed-loop default-rate detection — a CLUE-style harness for selective labels.
|
|
2
|
+
|
|
3
|
+
generate (``synthetic``) -> measure (``eval_default``) -> improve/frontier
|
|
4
|
+
(``loop``). See README.md for the CLUE mapping and how to run.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from importlib.metadata import PackageNotFoundError, version as _pkg_version
|
|
10
|
+
|
|
11
|
+
try: # pyproject `version` is the single source of truth (read from installed metadata)
|
|
12
|
+
__version__ = _pkg_version("closed-loop-default-detection")
|
|
13
|
+
except PackageNotFoundError: # running from a source tree with no install
|
|
14
|
+
__version__ = "0.1.0"
|
|
15
|
+
|
|
16
|
+
from .correctors import (
|
|
17
|
+
CorrectionOutcome,
|
|
18
|
+
Corrector,
|
|
19
|
+
CorrectorContext,
|
|
20
|
+
DisjointRetrainCorrector,
|
|
21
|
+
ExplorationCorrector,
|
|
22
|
+
IPWReweightCorrector,
|
|
23
|
+
NaiveCorrector,
|
|
24
|
+
)
|
|
25
|
+
from .counterfactual import (
|
|
26
|
+
CounterfactualResult,
|
|
27
|
+
GComputationEstimator,
|
|
28
|
+
generate_queries,
|
|
29
|
+
run_counterfactual_eval,
|
|
30
|
+
)
|
|
31
|
+
from .diagnostics import PositivityDiagnostics, positivity_diagnostics
|
|
32
|
+
from .reject_inference import (
|
|
33
|
+
AugmentationCorrector,
|
|
34
|
+
FuzzyAugmentationCorrector,
|
|
35
|
+
ParcellingCorrector,
|
|
36
|
+
ReclassificationCorrector,
|
|
37
|
+
RejectInferenceCorrector,
|
|
38
|
+
)
|
|
39
|
+
from .eval_default import PdDetectionResult, fit_observed_model, score_pd_detection
|
|
40
|
+
from .model_pd import CalibratedPDClassifier, CalibratedPDModel
|
|
41
|
+
from .feedback import FeedbackLoop, FeedbackResult, GenerationResult
|
|
42
|
+
from .loop import LeverMetrics, LoopResult, RoundResult, SelectiveLabelsLoop
|
|
43
|
+
from .scm import (
|
|
44
|
+
BANK_FEED_COLUMNS,
|
|
45
|
+
FEATURE_COLUMNS,
|
|
46
|
+
INTERVENABLE_FEATURES,
|
|
47
|
+
InterventionResult,
|
|
48
|
+
SCMState,
|
|
49
|
+
StructuralBorrowerGenerator,
|
|
50
|
+
dag_children,
|
|
51
|
+
dag_parents,
|
|
52
|
+
)
|
|
53
|
+
from .synthetic import SyntheticBorrowerGenerator
|
|
54
|
+
|
|
55
|
+
__all__ = [
|
|
56
|
+
"__version__",
|
|
57
|
+
"SyntheticBorrowerGenerator",
|
|
58
|
+
"StructuralBorrowerGenerator",
|
|
59
|
+
"SCMState",
|
|
60
|
+
"InterventionResult",
|
|
61
|
+
"INTERVENABLE_FEATURES",
|
|
62
|
+
"FEATURE_COLUMNS",
|
|
63
|
+
"BANK_FEED_COLUMNS",
|
|
64
|
+
"dag_children",
|
|
65
|
+
"dag_parents",
|
|
66
|
+
"SelectiveLabelsLoop",
|
|
67
|
+
"LoopResult",
|
|
68
|
+
"RoundResult",
|
|
69
|
+
"LeverMetrics",
|
|
70
|
+
"Corrector",
|
|
71
|
+
"CorrectorContext",
|
|
72
|
+
"CorrectionOutcome",
|
|
73
|
+
"NaiveCorrector",
|
|
74
|
+
"IPWReweightCorrector",
|
|
75
|
+
"DisjointRetrainCorrector",
|
|
76
|
+
"ExplorationCorrector",
|
|
77
|
+
"RejectInferenceCorrector",
|
|
78
|
+
"ReclassificationCorrector",
|
|
79
|
+
"AugmentationCorrector",
|
|
80
|
+
"FuzzyAugmentationCorrector",
|
|
81
|
+
"ParcellingCorrector",
|
|
82
|
+
"FeedbackLoop",
|
|
83
|
+
"FeedbackResult",
|
|
84
|
+
"GenerationResult",
|
|
85
|
+
"PositivityDiagnostics",
|
|
86
|
+
"positivity_diagnostics",
|
|
87
|
+
"PdDetectionResult",
|
|
88
|
+
"fit_observed_model",
|
|
89
|
+
"score_pd_detection",
|
|
90
|
+
"CalibratedPDModel",
|
|
91
|
+
"CalibratedPDClassifier",
|
|
92
|
+
"CounterfactualResult",
|
|
93
|
+
"GComputationEstimator",
|
|
94
|
+
"run_counterfactual_eval",
|
|
95
|
+
"generate_queries",
|
|
96
|
+
]
|